def report_openvas_results(self, db: BaseDB, scan_id: str) -> bool: """ Get all result entries from redis kb. """ vthelper = VtHelper(self.nvti) # Result messages come in the next form, with optional uri field # type ||| host ip ||| hostname ||| port ||| OID ||| value [|||uri] all_results = db.get_result() res_list = ResultList() total_dead = 0 for res in all_results: if not res: continue msg = res.split('|||') roid = msg[4].strip() rqod = '' rname = '' current_host = msg[1].strip() if msg[1] else '' rhostname = msg[2].strip() if msg[2] else '' host_is_dead = "Host dead" in msg[5] or msg[0] == "DEADHOST" host_deny = "Host access denied" in msg[5] start_end_msg = msg[0] == "HOST_START" or msg[0] == "HOST_END" vt_aux = None # URI is optional and msg list length must be checked ruri = '' if len(msg) > 6: ruri = msg[6] if (roid and not host_is_dead and not host_deny and not start_end_msg): vt_aux = vthelper.get_single_vt(roid) if (not vt_aux and not host_is_dead and not host_deny and not start_end_msg): logger.warning('Invalid VT oid %s for a result', roid) if vt_aux: if vt_aux.get('qod_type'): qod_t = vt_aux.get('qod_type') rqod = self.nvti.QOD_TYPES[qod_t] elif vt_aux.get('qod'): rqod = vt_aux.get('qod') rname = vt_aux.get('name') if msg[0] == 'ERRMSG': res_list.add_scan_error_to_list( host=current_host, hostname=rhostname, name=rname, value=msg[5], port=msg[3], test_id=roid, uri=ruri, ) elif msg[0] == 'HOST_START' or msg[0] == 'HOST_END': res_list.add_scan_log_to_list( host=current_host, name=msg[0], value=msg[5], ) elif msg[0] == 'LOG': res_list.add_scan_log_to_list( host=current_host, hostname=rhostname, name=rname, value=msg[5], port=msg[3], qod=rqod, test_id=roid, uri=ruri, ) elif msg[0] == 'HOST_DETAIL': res_list.add_scan_host_detail_to_list( host=current_host, hostname=rhostname, name=rname, value=msg[5], uri=ruri, ) elif msg[0] == 'ALARM': rseverity = self.get_severity_score(vt_aux) res_list.add_scan_alarm_to_list( host=current_host, hostname=rhostname, name=rname, value=msg[5], port=msg[3], test_id=roid, severity=rseverity, qod=rqod, uri=ruri, ) # To process non-scanned dead hosts when # test_alive_host_only in openvas is enable elif msg[0] == 'DEADHOST': try: total_dead = int(msg[5]) except TypeError: logger.debug('Error processing dead host count') # Insert result batch into the scan collection table. if len(res_list): self.scan_collection.add_result_list(scan_id, res_list) if total_dead: self.scan_collection.set_amount_dead_hosts(scan_id, total_dead=total_dead) return len(res_list) > 0
def report_openvas_results(self, db: BaseDB, scan_id: str, current_host: str): """ Get all result entries from redis kb. """ res = db.get_result() res_list = ResultList() host_progress_batch = dict() finished_host_batch = list() while res: msg = res.split('|||') roid = msg[3].strip() rqod = '' rname = '' rhostname = msg[1].strip() if msg[1] else '' host_is_dead = "Host dead" in msg[4] vt_aux = None if roid and not host_is_dead: vt_aux = copy.deepcopy(self.vts.get(roid)) if not vt_aux and not host_is_dead: logger.warning('Invalid VT oid %s for a result', roid) if vt_aux: if vt_aux.get('qod_type'): qod_t = vt_aux.get('qod_type') rqod = self.nvti.QOD_TYPES[qod_t] elif vt_aux.get('qod'): rqod = vt_aux.get('qod') rname = vt_aux.get('name') if msg[0] == 'ERRMSG': res_list.add_scan_error_to_list( host=current_host, hostname=rhostname, name=rname, value=msg[4], port=msg[2], test_id=roid, ) if msg[0] == 'LOG': res_list.add_scan_log_to_list( host=current_host, hostname=rhostname, name=rname, value=msg[4], port=msg[2], qod=rqod, test_id=roid, ) if msg[0] == 'HOST_DETAIL': res_list.add_scan_host_detail_to_list( host=current_host, hostname=rhostname, name=rname, value=msg[4], ) if msg[0] == 'ALARM': rseverity = self.get_severity_score(vt_aux) res_list.add_scan_alarm_to_list( host=current_host, hostname=rhostname, name=rname, value=msg[4], port=msg[2], test_id=roid, severity=rseverity, qod=rqod, ) # To process non scanned dead hosts when # test_alive_host_only in openvas is enable if msg[0] == 'DEADHOST': hosts = msg[3].split(',') for _host in hosts: if _host: host_progress_batch[_host] = 100 finished_host_batch.append(_host) res_list.add_scan_log_to_list( host=_host, hostname=rhostname, name=rname, value=msg[4], port=msg[2], qod=rqod, test_id='', ) timestamp = time.ctime(time.time()) res_list.add_scan_log_to_list( host=_host, name='HOST_START', value=timestamp, ) res_list.add_scan_log_to_list( host=_host, name='HOST_END', value=timestamp, ) vt_aux = None del vt_aux res = db.get_result() # Insert result batch into the scan collection table. if len(res_list): self.scan_collection.add_result_list(scan_id, res_list) if host_progress_batch: self.set_scan_progress_batch(scan_id, host_progress=host_progress_batch) if finished_host_batch: self.set_scan_host_finished(scan_id, finished_hosts=finished_host_batch)
def exec_dry_run_scan(self, scan_id, nvti, ospd_params): options = self._daemon.scan_collection.get_options(scan_id) results_per_host = None if "results_per_host" in options: results_per_host = options.get("results_per_host") if not results_per_host or not isinstance(results_per_host, int): logger.debug("Using default value for results_per_host options") results_per_host = ospd_params["results_per_host"].get("default") # Get the host list target = self._daemon.scan_collection.get_host_list(scan_id) logger.info("The target list %s", target) host_list = target_str_to_list(target) # Get the port list ports = self._daemon.scan_collection.get_ports(scan_id) logger.info("The port list %s", ports) tcp, _ = ports_as_list(ports) # Get exclude hosts list. It must not be scanned exclude_hosts = self._daemon.scan_collection.get_exclude_hosts(scan_id) logger.info("The exclude hosts list %s", exclude_hosts) self._daemon.set_scan_total_hosts( scan_id, count_total=len(host_list), ) self._daemon.scan_collection.set_amount_dead_hosts(scan_id, total_dead=0) # Get list of VTS. Ignore script params vts = list(self._daemon.scan_collection.get_vts(scan_id)) if "vt_groups" in vts: vts.remove("vt_groups") vthelper = VtHelper(nvti) # Run the scan. # Scan simulation for each single host. # Run the scan against the host, and generates results. while host_list: # Get a host from the list current_host = host_list.pop() # Check if the scan was stopped. status = self._daemon.get_scan_status(scan_id) if status == ScanStatus.STOPPED or status == ScanStatus.FINISHED: logger.debug( 'Task %s stopped or finished.', scan_id, ) return res_list = ResultList() res_list.add_scan_log_to_list( host=current_host, name="HOST_START", value=str(int(time.time())), ) # Generate N results per host. Default 10 results res_count = 0 while res_count < results_per_host: res_count += 1 oid = choice(vts) port = choice(tcp) vt = vthelper.get_single_vt(oid) if vt: if vt.get('qod_type'): qod_t = vt.get('qod_type') rqod = nvti.QOD_TYPES[qod_t] elif vt.get('qod'): rqod = vt.get('qod') rname = vt.get('name') else: logger.debug("oid %s not found", oid) res_type = int(uniform(1, 5)) # Error if res_type == 1: res_list.add_scan_error_to_list( host=current_host, hostname=current_host + ".hostname.net", name=rname, value="error running the script " + oid, port=port, test_id=oid, uri="No location", ) # Log elif res_type == 2: res_list.add_scan_log_to_list( host=current_host, hostname=current_host + ".hostname.net", name=rname, value="Log generate from a dry run scan for the script " + oid, port=port, qod=rqod, test_id=oid, uri="No location", ) # Alarm else: r_severity = vthelper.get_severity_score(vt) res_list.add_scan_alarm_to_list( host=current_host, hostname=current_host + ".hostname.net", name=rname, value="Log generate from a dry run scan for the script " + oid, port=port, test_id=oid, severity=r_severity, qod=rqod, uri="No location", ) res_list.add_scan_log_to_list( host=current_host, name="HOST_END", value=str(int(time.time())), ) # Add the result to the scan collection if len(res_list): logger.debug( '%s: Inserting %d results into scan ' 'scan collection table', scan_id, len(res_list), ) self._daemon.scan_collection.add_result_list(scan_id, res_list) # Set the host scan progress as finished host_progress = dict() host_progress[current_host] = ScanProgress.FINISHED self._daemon.set_scan_progress_batch(scan_id, host_progress=host_progress) # Update the host status, Finished host. So ospd can # calculate the scan progress. # This is quite importan, since the final scan status depends on # the progress calculation. finished_host = list() finished_host.append(current_host) self._daemon.sort_host_finished(scan_id, finished_host) time.sleep(1) logger.debug('%s: End task', scan_id)
def report_results(self, results: list, scan_id: str) -> bool: """Reports all results given in a list. Arguments: results: list of results each list item must contain a dictionary with following fields: result_type, host_ip, host_name, port, oid, value, uri (optional) """ vthelper = VtHelper(self.nvti) res_list = ResultList() total_dead = 0 for res in results: if not res: continue roid = res["oid"].strip() rqod = '' rname = '' current_host = res["host_ip"].strip() if res["host_ip"] else '' rhostname = res["host_name"].strip() if res["host_name"] else '' host_is_dead = ("Host dead" in res["value"] or res["result_type"] == "DEADHOST") host_deny = "Host access denied" in res["value"] start_end_msg = (res["result_type"] == "HOST_START" or res["result_type"] == "HOST_END") host_count = res["result_type"] == "HOSTS_COUNT" vt_aux = None # URI is optional and containing must be checked ruri = res["uri"] if "uri" in res else "" if (not host_is_dead and not host_deny and not start_end_msg and not host_count): if not roid and res["result_type"] != 'ERRMSG': logger.warning('Missing VT oid for a result') vt_aux = vthelper.get_single_vt(roid) if not vt_aux: logger.warning('Invalid VT oid %s for a result', roid) else: if vt_aux.get('qod_type'): qod_t = vt_aux.get('qod_type') rqod = self.nvti.QOD_TYPES[qod_t] elif vt_aux.get('qod'): rqod = vt_aux.get('qod') rname = vt_aux.get('name') if res["result_type"] == 'ERRMSG': res_list.add_scan_error_to_list( host=current_host, hostname=rhostname, name=rname, value=res["value"], port=res["port"], test_id=roid, uri=ruri, ) elif (res["result_type"] == 'HOST_START' or res["result_type"] == 'HOST_END'): res_list.add_scan_log_to_list( host=current_host, name=res["result_type"], value=res["value"], ) elif res["result_type"] == 'LOG': res_list.add_scan_log_to_list( host=current_host, hostname=rhostname, name=rname, value=res["value"], port=res["port"], qod=rqod, test_id=roid, uri=ruri, ) elif res["result_type"] == 'HOST_DETAIL': res_list.add_scan_host_detail_to_list( host=current_host, hostname=rhostname, name=rname, value=res["value"], uri=ruri, ) elif res["result_type"] == 'ALARM': rseverity = vthelper.get_severity_score(vt_aux) res_list.add_scan_alarm_to_list( host=current_host, hostname=rhostname, name=rname, value=res["value"], port=res["port"], test_id=roid, severity=rseverity, qod=rqod, uri=ruri, ) # To process non-scanned dead hosts when # test_alive_host_only in openvas is enable elif res["result_type"] == 'DEADHOST': try: total_dead = total_dead + int(res["value"]) except TypeError: logger.debug('Error processing dead host count') # To update total host count if res["result_type"] == 'HOSTS_COUNT': try: count_total = int(res["value"]) logger.debug( '%s: Set total hosts counted by OpenVAS: %d', scan_id, count_total, ) self.set_scan_total_hosts(scan_id, count_total) except TypeError: logger.debug('Error processing total host count') # Insert result batch into the scan collection table. if len(res_list): self.scan_collection.add_result_list(scan_id, res_list) logger.debug( '%s: Inserting %d results into scan collection table', scan_id, len(res_list), ) if total_dead: logger.debug( '%s: Set dead hosts counted by OpenVAS: %d', scan_id, total_dead, ) self.scan_collection.set_amount_dead_hosts(scan_id, total_dead=total_dead) return len(res_list) > 0
def report_openvas_results(self, db: BaseDB, scan_id: str, current_host: str): """ Get all result entries from redis kb. """ vthelper = VtHelper(self.nvti) # Result messages come in the next form, with optional uri field # type ||| hostname ||| port ||| OID ||| value [|||uri] res = db.get_result() res_list = ResultList() total_dead = 0 while res: msg = res.split('|||') roid = msg[3].strip() rqod = '' rname = '' rhostname = msg[1].strip() if msg[1] else '' host_is_dead = "Host dead" in msg[4] or msg[0] == "DEADHOST" host_deny = "Host access denied" in msg[4] vt_aux = None # URI is optional and msg list length must be checked ruri = '' if len(msg) > 5: ruri = msg[5] if roid and not host_is_dead and not host_deny: vt_aux = vthelper.get_single_vt(roid) if not vt_aux and not host_is_dead and not host_deny: logger.warning('Invalid VT oid %s for a result', roid) if vt_aux: if vt_aux.get('qod_type'): qod_t = vt_aux.get('qod_type') rqod = self.nvti.QOD_TYPES[qod_t] elif vt_aux.get('qod'): rqod = vt_aux.get('qod') rname = vt_aux.get('name') if msg[0] == 'ERRMSG': # Some errors are generated before a host is scanned # use the hostname passed in the message if # no current host is available. if not current_host and rhostname: current_host = rhostname res_list.add_scan_error_to_list( host=current_host, hostname=rhostname, name=rname, value=msg[4], port=msg[2], test_id=roid, uri=ruri, ) if msg[0] == 'LOG': res_list.add_scan_log_to_list( host=current_host, hostname=rhostname, name=rname, value=msg[4], port=msg[2], qod=rqod, test_id=roid, uri=ruri, ) if msg[0] == 'HOST_DETAIL': res_list.add_scan_host_detail_to_list( host=current_host, hostname=rhostname, name=rname, value=msg[4], uri=ruri, ) if msg[0] == 'ALARM': rseverity = self.get_severity_score(vt_aux) res_list.add_scan_alarm_to_list( host=current_host, hostname=rhostname, name=rname, value=msg[4], port=msg[2], test_id=roid, severity=rseverity, qod=rqod, uri=ruri, ) # To process non-scanned dead hosts when # test_alive_host_only in openvas is enable if msg[0] == 'DEADHOST': try: total_dead = int(msg[4]) except TypeError: logger.debug('Error processing dead host count') res = db.get_result() # Insert result batch into the scan collection table. if len(res_list): self.scan_collection.add_result_list(scan_id, res_list) if total_dead: self.scan_collection.set_amount_dead_hosts(scan_id, total_dead=total_dead)
def exec_scan(self, scan_id: str): """ Starts the scanner for scan_id scan. """ # Get the host list target = self.scan_collection.get_host_list(scan_id) logger.info("The target list %s", target) host_list = target_str_to_list(target) # Get the port list ports = self.scan_collection.get_ports(scan_id) logger.info("The port list %s", ports) # Get exclude hosts list. It must not be scanned exclude_hosts = self.scan_collection.get_exclude_hosts(scan_id) logger.info("The exclude hosts list %s", exclude_hosts) # Get credentials for authenticated scans credentials = self.scan_collection.get_credentials(scan_id) for ( service, credential, ) in credentials.items(): cred_type = credential.get('type', '') username = credential.get('username', '') password = credential.get( 'password', '' ) # pylint: disable=unused-variable logger.info( "Credential for %s: user: %s, type: %s", service, username, cred_type, ) # Get the plugin list and its preferences, if the scanner # supports plugins nvts = self.scan_collection.get_vts( scan_id ) # pylint: disable=unused-variable # Ospd calculates the scan progress on fly, but it depends on the # results. Sometimes, the scanner can be more smart and detect # e.g. repeted hosts or hostnames which can not be resolved and # therefore will not be scanned. # In this cases, the amount of hosts scanned can differ with the amount # of hosts in the target list. Please consider to use the following # method if your scanner has the hability and provides these total hosts # and amount of dead hosts. self.set_scan_total_hosts( scan_id, count_total=len(host_list), ) self.scan_collection.set_amount_dead_hosts(scan_id, total_dead=0) # Run the scan. This template does not run any scan, but generates # a fake result for each host in the whole target list. # Also, it doesn't consider the excluded hosts. If the scanner allows # multiple hosts, you can execute the scanner just once and process # the results later. while host_list: # Get a host from the list current_host = host_list.pop() # Example using subprocess.Popen() to exec the scanner # cmd = [] # if self._niceness: # cmd += ['nice', '-n', self._niceness] # logger.debug("Starting scan with niceness %s", self._niceness) # cmd += ['scanner_exec', current_host] # try: # return subprocess.Popen(cmd, shell=False) # except (subprocess.SubprocessError, OSError) as e: # # the command is not available # logger.warning("Could not start scan process. Reason %s", e) # return None # In some point you may want to check if something goes wrong or # if the scan was stopped by the client. # If your check is not successful, you can stop the server # gracefully. status = self.get_scan_status(scan_id) if status == ScanStatus.INTERRUPTED: self.stop_scan_cleanup(scan_id) logger.error( 'Something when wrong for task %s.', scan_id, ) return elif status == ScanStatus.STOPPED or status == ScanStatus.FINISHED: logger.debug( 'Task %s stopped or finished.', scan_id, ) return # Scan simulation for each single host. # Run the scan against the host, and generates results. res_list = ResultList() res_type = int(uniform(1, 5)) # Error if res_type == 1: res_list.add_scan_error_to_list( host=current_host, hostname=current_host + ".hostname.net", name="Some test name", value="error running the script", port=ports, test_id="1234-5678", uri="No location", ) # Log elif res_type == 2: res_list.add_scan_log_to_list( host=current_host, hostname=current_host + ".hostname.net", name="Some test name", value="Some log", port=ports, qod="10", test_id="1234-5678", uri="No location", ) # Host detail elif res_type == 3: res_list.add_scan_host_detail_to_list( host=current_host, hostname=current_host + ".hostname.net", name="Some Test Name", value="Some host detail", uri="No location", ) # Alarm else: res_list.add_scan_alarm_to_list( host=current_host, hostname=current_host + ".hostname.net", name="Some Test Name", value="Some Alarm", port=ports, test_id="1234-5678", severity="10", qod="10", uri="No location", ) # Add the result to the scan collection if len(res_list): logger.debug( '%s: Inserting %d results into scan ' 'scan collection table', scan_id, len(res_list), ) self.scan_collection.add_result_list(scan_id, res_list) # Update the host status, so ospd can calculate the scan progress # This is quite importan, since the final scan status depends on # the progress calculation. finished_host = list() finished_host.append(current_host) self.sort_host_finished(scan_id, finished_host) # If you know exactly the host scan progress, you can use the # the following methods for a more precise progress calculation. host_progress = dict() # host_progress[current_host] = 45 # host_progress[current_host] = ScanProgress.DEAD_HOST host_progress[current_host] = ScanProgress.FINISHED self.set_scan_progress_batch(scan_id, host_progress=host_progress) time.sleep(1) logger.debug('%s: End task', scan_id)