def test_get_vt_iterator_with_filter_no_vt(self): w = DummyDaemon() vthelper = VtHelper(w.nvti) w.nvti.get_nvt_metadata.return_value = None vt = ["1.3.6.1.4.1.25623.1.0.100065"] for _, values in vthelper.get_vt_iterator(vt_selection=vt): self.assertIs(values, None)
def test_get_vt_iterator(self): dummy = DummyDaemon() vthelper = VtHelper(dummy.nvti) vt = ["1.3.6.1.4.1.25623.1.0.100061"] for key, _ in vthelper.get_vt_iterator(): self.assertIn(key, vt)
def test_get_single_vt_severity_cvssv2(self): w = DummyDaemon() w.nvti.get_nvt_metadata.return_value = { 'category': '3', 'creation_date': '1237458156', 'cvss_base_vector': 'AV:N/AC:L/Au:N/C:N/I:N/A:N', 'excluded_keys': 'Settings/disable_cgi_scanning', 'family': 'Product detection', 'filename': 'mantis_detect.nasl', 'last_modification': ('1533906565'), 'name': 'Mantis Detection', 'qod_type': 'remote_banner', 'required_ports': 'Services/www, 80', 'solution': 'some solution', 'solution_type': 'WillNotFix', 'solution_method': 'DebianAPTUpgrade', 'impact': 'some impact', 'insight': 'some insight', 'summary': ('some summary'), 'affected': 'some affection', 'timeout': '0', 'vt_params': { '1': { 'id': '1', 'default': '', 'description': 'Description', 'name': 'Data length :', 'type': 'entry', }, '2': { 'id': '2', 'default': 'no', 'description': 'Description', 'name': 'Do not randomize the order in which ports are scanned', # pylint: disable=line-too-long 'type': 'checkbox', }, }, 'refs': { 'bid': [''], 'cve': [''], 'xref': ['URL:http://www.mantisbt.org/'], }, } vthelper = VtHelper(w.nvti) res = vthelper.get_single_vt("1.3.6.1.4.1.25623.1.0.100061") assert_called_once(w.nvti.get_nvt_metadata) severities = res.get('severities') self.assertEqual( "AV:N/AC:L/Au:N/C:N/I:N/A:N", severities.get('severity_base_vector'), ) self.assertEqual("cvss_base_v2", severities.get('severity_type')) self.assertEqual(None, severities.get('severity_origin')) self.assertEqual("1237458156", severities.get('severity_date'))
def test_get_severity_score_v2(self): dummy = DummyDaemon() vthelper = VtHelper(dummy.nvti) vtaux = { 'severities': { 'severity_type': 'cvss_base_v2', 'severity_base_vector': 'AV:N/AC:L/Au:N/C:P/I:N/A:N', } } self.assertEqual(vthelper.get_severity_score(vtaux), 5.0)
def test_get_vt_iterator_with_filter(self): w = DummyDaemon() vthelper = VtHelper(w.nvti) vt = ["1.3.6.1.4.1.25623.1.0.100061"] vtout = w.VTS["1.3.6.1.4.1.25623.1.0.100061"] for key, vt_dict in vthelper.get_vt_iterator(vt_selection=vt): self.assertIn(key, vt) for key in vtout: self.assertIn(key, vt_dict)
def test_get_severity_score_v3(self): dummy = DummyDaemon() vthelper = VtHelper(dummy.nvti) vtaux = { 'severities': { 'severity_type': 'cvss_base_v3', 'severity_base_vector': ( 'CVSS:3.0/AV:L/AC:H/PR:H/UI:R/S:U/C:N/I:L/A:L' ), } } self.assertEqual(vthelper.get_severity_score(vtaux), 2.9)
def test_calculate_vts_collection_hash_no_params(self): w = DummyDaemon() vthelper = VtHelper(w.nvti) hash_out = vthelper.calculate_vts_collection_hash() vt_hash_str = ('1.3.6.1.4.1.25623.1.0.10006115339065651Data ' + 'length :2Do not randomize the order in which ' + 'ports are scannedno') vt_hash = sha256() vt_hash.update(vt_hash_str.encode('utf-8')) hash_test = vt_hash.hexdigest() self.assertEqual(hash_test, hash_out)
def init(self, server: BaseServer) -> None: server.start(self.handle_client_stream) self.scanner_info['version'] = Openvas.get_version() self.set_params_from_openvas_settings() with self.feed_lock.wait_for_lock(): Openvas.load_vts_into_redis() current_feed = self.nvti.get_feed_version() self.set_vts_version(vts_version=current_feed) vthelper = VtHelper(self.nvti) self.vts.sha256_hash = vthelper.calculate_vts_collection_hash() self.initialized = True
def get_filtered_vts_list(self, vts, vt_filter: str) -> Optional[List[str]]: """Gets a collection of vulnerability test from the redis cache, which match the filter. Arguments: vt_filter: Filter to apply to the vts collection. vts: The complete vts collection. Returns: List with filtered vulnerability tests. The list can be empty. None in case of filter parse failure. """ filters = self.parse_filters(vt_filter) if not filters: return None if not self.nvti: return None vt_oid_list = [vtlist[1] for vtlist in self.nvti.get_oids()] vt_oid_list_temp = copy.copy(vt_oid_list) vthelper = VtHelper(self.nvti) for element, oper, filter_val in filters: for vt_oid in vt_oid_list_temp: if vt_oid not in vt_oid_list: continue vt = vthelper.get_single_vt(vt_oid) if vt is None or not vt.get(element): vt_oid_list.remove(vt_oid) continue elem_val = vt.get(element) val = self.format_filter_value(element, elem_val) if self.filter_operator[oper](val, filter_val): continue else: vt_oid_list.remove(vt_oid) return vt_oid_list
def init(self, server: BaseServer) -> None: notus_handler = NotusResultHandler(self.report_results) if self._mqtt_broker_address: try: client = MQTTClient(self._mqtt_broker_address, self._mqtt_broker_port, "ospd") daemon = MQTTDaemon(client) subscriber = MQTTSubscriber(client) subscriber.subscribe(ResultMessage, notus_handler.result_handler) daemon.run() except (ConnectionRefusedError, gaierror, ValueError) as e: logger.error( "Could not connect to MQTT broker at %s, error was: %s." " Unable to get results from Notus.", self._mqtt_broker_address, e, ) else: logger.info( "MQTT Broker Adress empty. MQTT disabled. Unable to get Notus" " results.") self.scan_collection.init() server.start(self.handle_client_stream) self.scanner_info['version'] = Openvas.get_version() self.set_params_from_openvas_settings() with self.feed_lock.wait_for_lock(): Openvas.load_vts_into_redis() self.set_feed_info() logger.debug("Calculating vts integrity check hash...") vthelper = VtHelper(self.nvti) self.vts.sha256_hash = vthelper.calculate_vts_collection_hash() self.initialized = True
def init(self, server: BaseServer) -> None: self.scan_collection.init() server.start(self.handle_client_stream) self.scanner_info['version'] = Openvas.get_version() self.set_params_from_openvas_settings() with self.feed_lock.wait_for_lock(): Openvas.load_vts_into_redis() notushandler = NotusMetadataHandler(nvti=self.nvti) notushandler.update_metadata() current_feed = self.nvti.get_feed_version() self.set_vts_version(vts_version=current_feed) logger.debug("Calculating vts integrity check hash...") vthelper = VtHelper(self.nvti) self.vts.sha256_hash = vthelper.calculate_vts_collection_hash() self.initialized = True
def report_openvas_results(self, db: BaseDB, scan_id: str) -> bool: """ Get all result entries from redis kb. """ vthelper = VtHelper(self.nvti) # Result messages come in the next form, with optional uri field # type ||| host ip ||| hostname ||| port ||| OID ||| value [|||uri] all_results = db.get_result() res_list = ResultList() total_dead = 0 for res in all_results: if not res: continue msg = res.split('|||') roid = msg[4].strip() rqod = '' rname = '' current_host = msg[1].strip() if msg[1] else '' rhostname = msg[2].strip() if msg[2] else '' host_is_dead = "Host dead" in msg[5] or msg[0] == "DEADHOST" host_deny = "Host access denied" in msg[5] start_end_msg = msg[0] == "HOST_START" or msg[0] == "HOST_END" vt_aux = None # URI is optional and msg list length must be checked ruri = '' if len(msg) > 6: ruri = msg[6] if (roid and not host_is_dead and not host_deny and not start_end_msg): vt_aux = vthelper.get_single_vt(roid) if (not vt_aux and not host_is_dead and not host_deny and not start_end_msg): logger.warning('Invalid VT oid %s for a result', roid) if vt_aux: if vt_aux.get('qod_type'): qod_t = vt_aux.get('qod_type') rqod = self.nvti.QOD_TYPES[qod_t] elif vt_aux.get('qod'): rqod = vt_aux.get('qod') rname = vt_aux.get('name') if msg[0] == 'ERRMSG': res_list.add_scan_error_to_list( host=current_host, hostname=rhostname, name=rname, value=msg[5], port=msg[3], test_id=roid, uri=ruri, ) elif msg[0] == 'HOST_START' or msg[0] == 'HOST_END': res_list.add_scan_log_to_list( host=current_host, name=msg[0], value=msg[5], ) elif msg[0] == 'LOG': res_list.add_scan_log_to_list( host=current_host, hostname=rhostname, name=rname, value=msg[5], port=msg[3], qod=rqod, test_id=roid, uri=ruri, ) elif msg[0] == 'HOST_DETAIL': res_list.add_scan_host_detail_to_list( host=current_host, hostname=rhostname, name=rname, value=msg[5], uri=ruri, ) elif msg[0] == 'ALARM': rseverity = self.get_severity_score(vt_aux) res_list.add_scan_alarm_to_list( host=current_host, hostname=rhostname, name=rname, value=msg[5], port=msg[3], test_id=roid, severity=rseverity, qod=rqod, uri=ruri, ) # To process non-scanned dead hosts when # test_alive_host_only in openvas is enable elif msg[0] == 'DEADHOST': try: total_dead = int(msg[5]) except TypeError: logger.debug('Error processing dead host count') # Insert result batch into the scan collection table. if len(res_list): self.scan_collection.add_result_list(scan_id, res_list) if total_dead: self.scan_collection.set_amount_dead_hosts(scan_id, total_dead=total_dead) return len(res_list) > 0
def report_openvas_results(self, db: BaseDB, scan_id: str, current_host: str): """ Get all result entries from redis kb. """ vthelper = VtHelper(self.nvti) # Result messages come in the next form, with optional uri field # type ||| hostname ||| port ||| OID ||| value [|||uri] res = db.get_result() res_list = ResultList() total_dead = 0 while res: msg = res.split('|||') roid = msg[3].strip() rqod = '' rname = '' rhostname = msg[1].strip() if msg[1] else '' host_is_dead = "Host dead" in msg[4] or msg[0] == "DEADHOST" host_deny = "Host access denied" in msg[4] vt_aux = None # URI is optional and msg list length must be checked ruri = '' if len(msg) > 5: ruri = msg[5] if roid and not host_is_dead and not host_deny: vt_aux = vthelper.get_single_vt(roid) if not vt_aux and not host_is_dead and not host_deny: logger.warning('Invalid VT oid %s for a result', roid) if vt_aux: if vt_aux.get('qod_type'): qod_t = vt_aux.get('qod_type') rqod = self.nvti.QOD_TYPES[qod_t] elif vt_aux.get('qod'): rqod = vt_aux.get('qod') rname = vt_aux.get('name') if msg[0] == 'ERRMSG': # Some errors are generated before a host is scanned # use the hostname passed in the message if # no current host is available. if not current_host and rhostname: current_host = rhostname res_list.add_scan_error_to_list( host=current_host, hostname=rhostname, name=rname, value=msg[4], port=msg[2], test_id=roid, uri=ruri, ) if msg[0] == 'LOG': res_list.add_scan_log_to_list( host=current_host, hostname=rhostname, name=rname, value=msg[4], port=msg[2], qod=rqod, test_id=roid, uri=ruri, ) if msg[0] == 'HOST_DETAIL': res_list.add_scan_host_detail_to_list( host=current_host, hostname=rhostname, name=rname, value=msg[4], uri=ruri, ) if msg[0] == 'ALARM': rseverity = self.get_severity_score(vt_aux) res_list.add_scan_alarm_to_list( host=current_host, hostname=rhostname, name=rname, value=msg[4], port=msg[2], test_id=roid, severity=rseverity, qod=rqod, uri=ruri, ) # To process non-scanned dead hosts when # test_alive_host_only in openvas is enable if msg[0] == 'DEADHOST': try: total_dead = int(msg[4]) except TypeError: logger.debug('Error processing dead host count') res = db.get_result() # Insert result batch into the scan collection table. if len(res_list): self.scan_collection.add_result_list(scan_id, res_list) if total_dead: self.scan_collection.set_amount_dead_hosts(scan_id, total_dead=total_dead)
def report_openvas_results( self, db: BaseDB, scan_id: str, current_host: str ): """ Get all result entries from redis kb. """ vthelper = VtHelper(self.nvti) res = db.get_result() res_list = ResultList() host_progress_batch = dict() finished_host_batch = list() while res: msg = res.split('|||') roid = msg[3].strip() rqod = '' rname = '' rhostname = msg[1].strip() if msg[1] else '' host_is_dead = "Host dead" in msg[4] vt_aux = None if roid and not host_is_dead: vt_aux = vthelper.get_single_vt(roid) if not vt_aux and not host_is_dead: logger.warning('Invalid VT oid %s for a result', roid) if vt_aux: if vt_aux.get('qod_type'): qod_t = vt_aux.get('qod_type') rqod = self.nvti.QOD_TYPES[qod_t] elif vt_aux.get('qod'): rqod = vt_aux.get('qod') rname = vt_aux.get('name') if msg[0] == 'ERRMSG': res_list.add_scan_error_to_list( host=current_host, hostname=rhostname, name=rname, value=msg[4], port=msg[2], test_id=roid, ) if msg[0] == 'LOG': res_list.add_scan_log_to_list( host=current_host, hostname=rhostname, name=rname, value=msg[4], port=msg[2], qod=rqod, test_id=roid, ) if msg[0] == 'HOST_DETAIL': res_list.add_scan_host_detail_to_list( host=current_host, hostname=rhostname, name=rname, value=msg[4], ) if msg[0] == 'ALARM': rseverity = self.get_severity_score(vt_aux) res_list.add_scan_alarm_to_list( host=current_host, hostname=rhostname, name=rname, value=msg[4], port=msg[2], test_id=roid, severity=rseverity, qod=rqod, ) # To process non scanned dead hosts when # test_alive_host_only in openvas is enable if msg[0] == 'DEADHOST': hosts = msg[3].split(',') for _host in hosts: if _host: host_progress_batch[_host] = 100 finished_host_batch.append(_host) res_list.add_scan_log_to_list( host=_host, hostname=rhostname, name=rname, value=msg[4], port=msg[2], qod=rqod, test_id='', ) timestamp = time.ctime(time.time()) res_list.add_scan_log_to_list( host=_host, name='HOST_START', value=timestamp, ) res_list.add_scan_log_to_list( host=_host, name='HOST_END', value=timestamp, ) res = db.get_result() # Insert result batch into the scan collection table. if len(res_list): self.scan_collection.add_result_list(scan_id, res_list) if host_progress_batch: self.set_scan_progress_batch( scan_id, host_progress=host_progress_batch ) if finished_host_batch: self.set_scan_host_finished( scan_id, finished_hosts=finished_host_batch )
def report_results(self, results: list, scan_id: str) -> bool: """Reports all results given in a list. Arguments: results: list of results each list item must contain a dictionary with following fields: result_type, host_ip, host_name, port, oid, value, uri (optional) """ vthelper = VtHelper(self.nvti) res_list = ResultList() total_dead = 0 for res in results: if not res: continue roid = res["oid"].strip() rqod = '' rname = '' current_host = res["host_ip"].strip() if res["host_ip"] else '' rhostname = res["host_name"].strip() if res["host_name"] else '' host_is_dead = ("Host dead" in res["value"] or res["result_type"] == "DEADHOST") host_deny = "Host access denied" in res["value"] start_end_msg = (res["result_type"] == "HOST_START" or res["result_type"] == "HOST_END") host_count = res["result_type"] == "HOSTS_COUNT" vt_aux = None # URI is optional and containing must be checked ruri = res["uri"] if "uri" in res else "" if (not host_is_dead and not host_deny and not start_end_msg and not host_count): if not roid and res["result_type"] != 'ERRMSG': logger.warning('Missing VT oid for a result') vt_aux = vthelper.get_single_vt(roid) if not vt_aux: logger.warning('Invalid VT oid %s for a result', roid) else: if vt_aux.get('qod_type'): qod_t = vt_aux.get('qod_type') rqod = self.nvti.QOD_TYPES[qod_t] elif vt_aux.get('qod'): rqod = vt_aux.get('qod') rname = vt_aux.get('name') if res["result_type"] == 'ERRMSG': res_list.add_scan_error_to_list( host=current_host, hostname=rhostname, name=rname, value=res["value"], port=res["port"], test_id=roid, uri=ruri, ) elif (res["result_type"] == 'HOST_START' or res["result_type"] == 'HOST_END'): res_list.add_scan_log_to_list( host=current_host, name=res["result_type"], value=res["value"], ) elif res["result_type"] == 'LOG': res_list.add_scan_log_to_list( host=current_host, hostname=rhostname, name=rname, value=res["value"], port=res["port"], qod=rqod, test_id=roid, uri=ruri, ) elif res["result_type"] == 'HOST_DETAIL': res_list.add_scan_host_detail_to_list( host=current_host, hostname=rhostname, name=rname, value=res["value"], uri=ruri, ) elif res["result_type"] == 'ALARM': rseverity = vthelper.get_severity_score(vt_aux) res_list.add_scan_alarm_to_list( host=current_host, hostname=rhostname, name=rname, value=res["value"], port=res["port"], test_id=roid, severity=rseverity, qod=rqod, uri=ruri, ) # To process non-scanned dead hosts when # test_alive_host_only in openvas is enable elif res["result_type"] == 'DEADHOST': try: total_dead = total_dead + int(res["value"]) except TypeError: logger.debug('Error processing dead host count') # To update total host count if res["result_type"] == 'HOSTS_COUNT': try: count_total = int(res["value"]) logger.debug( '%s: Set total hosts counted by OpenVAS: %d', scan_id, count_total, ) self.set_scan_total_hosts(scan_id, count_total) except TypeError: logger.debug('Error processing total host count') # Insert result batch into the scan collection table. if len(res_list): self.scan_collection.add_result_list(scan_id, res_list) logger.debug( '%s: Inserting %d results into scan collection table', scan_id, len(res_list), ) if total_dead: logger.debug( '%s: Set dead hosts counted by OpenVAS: %d', scan_id, total_dead, ) self.scan_collection.set_amount_dead_hosts(scan_id, total_dead=total_dead) return len(res_list) > 0
def get_vt_iterator(self, vt_selection: List[str] = None, details: bool = True) -> Iterator[Tuple[str, Dict]]: vthelper = VtHelper(self.nvti) return vthelper.get_vt_iterator(vt_selection, details)
def _process_vts( self, vts: Dict[str, Dict[str, str]], ) -> Tuple[List[str], Dict[str, str]]: """ Add single VTs and their parameters. """ vts_list = [] vts_params = {} vtgroups = vts.pop('vt_groups') vthelper = VtHelper(self.nvti) if vtgroups: vts_list = self._get_vts_in_groups(vtgroups) for vtid, vt_params in vts.items(): vt = vthelper.get_single_vt(vtid) if not vt: logger.warning( 'The VT %s was not found and it will not be added to the ' 'plugin scheduler.', vtid, ) continue vts_list.append(vtid) for vt_param_id, vt_param_value in vt_params.items(): param_type = self._get_vt_param_type(vt, vt_param_id) param_name = self._get_vt_param_name(vt, vt_param_id) if not param_type or not param_name: logger.debug( 'Missing type or name for VT parameter %s of %s. ' 'This VT parameter will not be set.', vt_param_id, vtid, ) continue if vt_param_id == '0': type_aux = 'integer' else: type_aux = param_type if self.check_param_type(vt_param_value, type_aux): logger.debug( 'The VT parameter %s for %s could not be set. ' 'Expected %s type for parameter value %s', vt_param_id, vtid, type_aux, str(vt_param_value), ) continue if type_aux == 'checkbox': vt_param_value = _from_bool_to_str(int(vt_param_value)) vts_params["{0}:{1}:{2}:{3}".format( vtid, vt_param_id, param_type, param_name)] = str(vt_param_value) return vts_list, vts_params
def exec_dry_run_scan(self, scan_id, nvti, ospd_params): options = self._daemon.scan_collection.get_options(scan_id) results_per_host = None if "results_per_host" in options: results_per_host = options.get("results_per_host") if not results_per_host or not isinstance(results_per_host, int): logger.debug("Using default value for results_per_host options") results_per_host = ospd_params["results_per_host"].get("default") # Get the host list target = self._daemon.scan_collection.get_host_list(scan_id) logger.info("The target list %s", target) host_list = target_str_to_list(target) # Get the port list ports = self._daemon.scan_collection.get_ports(scan_id) logger.info("The port list %s", ports) tcp, _ = ports_as_list(ports) # Get exclude hosts list. It must not be scanned exclude_hosts = self._daemon.scan_collection.get_exclude_hosts(scan_id) logger.info("The exclude hosts list %s", exclude_hosts) self._daemon.set_scan_total_hosts( scan_id, count_total=len(host_list), ) self._daemon.scan_collection.set_amount_dead_hosts(scan_id, total_dead=0) # Get list of VTS. Ignore script params vts = list(self._daemon.scan_collection.get_vts(scan_id)) if "vt_groups" in vts: vts.remove("vt_groups") vthelper = VtHelper(nvti) # Run the scan. # Scan simulation for each single host. # Run the scan against the host, and generates results. while host_list: # Get a host from the list current_host = host_list.pop() # Check if the scan was stopped. status = self._daemon.get_scan_status(scan_id) if status == ScanStatus.STOPPED or status == ScanStatus.FINISHED: logger.debug( 'Task %s stopped or finished.', scan_id, ) return res_list = ResultList() res_list.add_scan_log_to_list( host=current_host, name="HOST_START", value=str(int(time.time())), ) # Generate N results per host. Default 10 results res_count = 0 while res_count < results_per_host: res_count += 1 oid = choice(vts) port = choice(tcp) vt = vthelper.get_single_vt(oid) if vt: if vt.get('qod_type'): qod_t = vt.get('qod_type') rqod = nvti.QOD_TYPES[qod_t] elif vt.get('qod'): rqod = vt.get('qod') rname = vt.get('name') else: logger.debug("oid %s not found", oid) res_type = int(uniform(1, 5)) # Error if res_type == 1: res_list.add_scan_error_to_list( host=current_host, hostname=current_host + ".hostname.net", name=rname, value="error running the script " + oid, port=port, test_id=oid, uri="No location", ) # Log elif res_type == 2: res_list.add_scan_log_to_list( host=current_host, hostname=current_host + ".hostname.net", name=rname, value="Log generate from a dry run scan for the script " + oid, port=port, qod=rqod, test_id=oid, uri="No location", ) # Alarm else: r_severity = vthelper.get_severity_score(vt) res_list.add_scan_alarm_to_list( host=current_host, hostname=current_host + ".hostname.net", name=rname, value="Log generate from a dry run scan for the script " + oid, port=port, test_id=oid, severity=r_severity, qod=rqod, uri="No location", ) res_list.add_scan_log_to_list( host=current_host, name="HOST_END", value=str(int(time.time())), ) # Add the result to the scan collection if len(res_list): logger.debug( '%s: Inserting %d results into scan ' 'scan collection table', scan_id, len(res_list), ) self._daemon.scan_collection.add_result_list(scan_id, res_list) # Set the host scan progress as finished host_progress = dict() host_progress[current_host] = ScanProgress.FINISHED self._daemon.set_scan_progress_batch(scan_id, host_progress=host_progress) # Update the host status, Finished host. So ospd can # calculate the scan progress. # This is quite importan, since the final scan status depends on # the progress calculation. finished_host = list() finished_host.append(current_host) self._daemon.sort_host_finished(scan_id, finished_host) time.sleep(1) logger.debug('%s: End task', scan_id)