def test_bad_ipv4_cidr(self): addresses = target_str_to_list('195.70.81.0/32') self.assertIsNotNone(addresses) self.assertEqual(len(addresses), 0) addresses = target_str_to_list('195.70.81.0/31') self.assertIsNotNone(addresses) self.assertEqual(len(addresses), 0)
def simplify_exclude_host_count(self, scan_id: str) -> int: """Remove from exclude_hosts the received hosts in the finished_hosts list sent by the client. The finished hosts are sent also as exclude hosts for backward compatibility purposses. Return: Count of excluded host. """ exc_hosts_list = target_str_to_list(self.get_exclude_hosts(scan_id)) logger.debug( '%s: Excluded Hosts: %s', scan_id, pformat(exc_hosts_list), ) finished_hosts_list = target_str_to_list( self.get_finished_hosts(scan_id)) logger.debug( '%s: Finished Hosts: %s', scan_id, pformat(finished_hosts_list), ) # Remove finished hosts from excluded host list if finished_hosts_list and exc_hosts_list: for finished in finished_hosts_list: if finished in exc_hosts_list: exc_hosts_list.remove(finished) # Remove excluded hosts which don't belong to the target list host_list = target_str_to_list(self.get_host_list(scan_id)) excluded_simplified = 0 invalid_exc_hosts = 0 if exc_hosts_list: for exc_host in exc_hosts_list: if exc_host in host_list: excluded_simplified += 1 else: invalid_exc_hosts += 1 if invalid_exc_hosts > 0: logger.warning( "Please check the excluded host list. It contains hosts which " "do not belong to the target. This warning can be ignored if " "this was done on purpose (e.g. to exclude specific hostname)." ) # Set scan_info's excluded simplified to propagate excluded count # to parent process. self.scans_table[scan_id]['excluded_simplified'] = excluded_simplified return excluded_simplified
def test_24_net(self): addresses = target_str_to_list('195.70.81.0/24') self.assertIsNotNone(addresses) self.assertEqual(len(addresses), 254) for i in range(1, 255): self.assertIn(f'195.70.81.{str(i)}', addresses)
def test_range(self): addresses = target_str_to_list('195.70.81.1-10') self.assertIsNotNone(addresses) self.assertEqual(len(addresses), 10) for i in range(1, 10): self.assertIn('195.70.81.%d' % i, addresses)
def process_exclude_hosts(self, scan_id: str, exclude_hosts: str) -> None: """ Process the exclude hosts before launching the scans.""" exc_hosts_list = '' if not exclude_hosts: return exc_hosts_list = target_str_to_list(exclude_hosts) self.remove_scan_hosts_from_target_progress(scan_id, exc_hosts_list)
def test_range(self): addresses = target_str_to_list('195.70.81.0-10') self.assertIsNotNone(addresses) self.assertEqual(len(addresses), 11) for i in range(0, 10): self.assertIn(f'195.70.81.{str(i)}', addresses)
def test_target_str_with_trailing_comma(self): addresses = target_str_to_list(',195.70.81.1,195.70.81.2,') self.assertIsNotNone(addresses) self.assertEqual(len(addresses), 2) for i in range(1, 2): self.assertIn(f'195.70.81.{str(i)}', addresses)
def get_host_count(self, scan_id: str) -> int: """Get total host count in the target.""" host = self.get_host_list(scan_id) total_hosts = 0 if host: total_hosts = len(target_str_to_list(host)) return total_hosts
def process_finished_hosts(self, scan_id: str) -> None: """ Process the finished hosts before launching the scans.""" finished_hosts = self.scan_collection.get_finished_hosts(scan_id) if not finished_hosts: return exc_finished_hosts_list = target_str_to_list(finished_hosts) self.scan_collection.set_host_finished(scan_id, exc_finished_hosts_list)
def simplify_exclude_host_list(self, scan_id: str) -> List[Any]: """ Remove from exclude_hosts the received hosts in the finished_hosts list sent by the client. The finished hosts are sent also as exclude hosts for backward compatibility purposses. """ exc_hosts_list = target_str_to_list(self.get_exclude_hosts(scan_id)) finished_hosts_list = target_str_to_list( self.get_finished_hosts(scan_id)) if finished_hosts_list and exc_hosts_list: for finished in finished_hosts_list: if finished in exc_hosts_list: exc_hosts_list.remove(finished) return exc_hosts_list
def get_hosts_unfinished(self, scan_id): """ Get a list of unfinished hosts.""" unfinished_hosts = list() for target in self.scans_table[scan_id]['finished_hosts']: unfinished_hosts.extend(target_str_to_list(target)) for target in self.scans_table[scan_id]['finished_hosts']: for host in self.scans_table[scan_id]['finished_hosts'][target]: unfinished_hosts.remove(host) return unfinished_hosts
def get_hosts_unfinished(self, scan_id: str) -> List[Any]: """ Get a list of unfinished hosts.""" unfinished_hosts = target_str_to_list(self.get_host_list(scan_id)) finished_hosts = self.get_hosts_finished(scan_id) for host in finished_hosts: unfinished_hosts.remove(host) return unfinished_hosts
def get_target_progress(self, scan_id, target): """ Get a target's current progress value. The value is calculated with the progress of each single host in the target.""" total_hosts = len(target_str_to_list(target)) exc_hosts_list = target_str_to_list( self.get_exclude_hosts(scan_id, target) ) exc_hosts = len(exc_hosts_list) if exc_hosts_list else 0 host_progresses = self.scans_table[scan_id]['target_progress'].get( target ) try: t_prog = sum(host_progresses.values()) / (total_hosts - exc_hosts) except ZeroDivisionError: LOGGER.error( "Zero division error in %s", self.get_target_progress.__name__ ) raise return t_prog
def process_finished_hosts(self, scan_id: str, finished_hosts: str) -> None: """ Process the finished hosts before launching the scans. Set finished hosts as finished with 100% to calculate the scan progress.""" exc_hosts_list = '' if not finished_hosts: return exc_hosts_list = target_str_to_list(finished_hosts) for host in exc_hosts_list: self.set_scan_host_finished(scan_id, host) self.set_scan_host_progress(scan_id, host, 100)
def calculate_target_progress(self, scan_id: str) -> float: """ Get a target's current progress value. The value is calculated with the progress of each single host in the target.""" host = self.get_host_list(scan_id) total_hosts = len(target_str_to_list(host)) exc_hosts_list = self.simplify_exclude_host_list(scan_id) exc_hosts = len(exc_hosts_list) if exc_hosts_list else 0 host_progresses = self.scans_table[scan_id].get('target_progress') try: t_prog = sum(host_progresses.values()) / (total_hosts - exc_hosts ) # type: float except ZeroDivisionError: LOGGER.error( "Zero division error in %s", self.calculate_target_progress.__name__, ) raise return t_prog
def exec_dry_run_scan(self, scan_id, nvti, ospd_params): options = self._daemon.scan_collection.get_options(scan_id) results_per_host = None if "results_per_host" in options: results_per_host = options.get("results_per_host") if not results_per_host or not isinstance(results_per_host, int): logger.debug("Using default value for results_per_host options") results_per_host = ospd_params["results_per_host"].get("default") # Get the host list target = self._daemon.scan_collection.get_host_list(scan_id) logger.info("The target list %s", target) host_list = target_str_to_list(target) # Get the port list ports = self._daemon.scan_collection.get_ports(scan_id) logger.info("The port list %s", ports) tcp, _ = ports_as_list(ports) # Get exclude hosts list. It must not be scanned exclude_hosts = self._daemon.scan_collection.get_exclude_hosts(scan_id) logger.info("The exclude hosts list %s", exclude_hosts) self._daemon.set_scan_total_hosts( scan_id, count_total=len(host_list), ) self._daemon.scan_collection.set_amount_dead_hosts(scan_id, total_dead=0) # Get list of VTS. Ignore script params vts = list(self._daemon.scan_collection.get_vts(scan_id)) if "vt_groups" in vts: vts.remove("vt_groups") vthelper = VtHelper(nvti) # Run the scan. # Scan simulation for each single host. # Run the scan against the host, and generates results. while host_list: # Get a host from the list current_host = host_list.pop() # Check if the scan was stopped. status = self._daemon.get_scan_status(scan_id) if status == ScanStatus.STOPPED or status == ScanStatus.FINISHED: logger.debug( 'Task %s stopped or finished.', scan_id, ) return res_list = ResultList() res_list.add_scan_log_to_list( host=current_host, name="HOST_START", value=str(int(time.time())), ) # Generate N results per host. Default 10 results res_count = 0 while res_count < results_per_host: res_count += 1 oid = choice(vts) port = choice(tcp) vt = vthelper.get_single_vt(oid) if vt: if vt.get('qod_type'): qod_t = vt.get('qod_type') rqod = nvti.QOD_TYPES[qod_t] elif vt.get('qod'): rqod = vt.get('qod') rname = vt.get('name') else: logger.debug("oid %s not found", oid) res_type = int(uniform(1, 5)) # Error if res_type == 1: res_list.add_scan_error_to_list( host=current_host, hostname=current_host + ".hostname.net", name=rname, value="error running the script " + oid, port=port, test_id=oid, uri="No location", ) # Log elif res_type == 2: res_list.add_scan_log_to_list( host=current_host, hostname=current_host + ".hostname.net", name=rname, value="Log generate from a dry run scan for the script " + oid, port=port, qod=rqod, test_id=oid, uri="No location", ) # Alarm else: r_severity = vthelper.get_severity_score(vt) res_list.add_scan_alarm_to_list( host=current_host, hostname=current_host + ".hostname.net", name=rname, value="Log generate from a dry run scan for the script " + oid, port=port, test_id=oid, severity=r_severity, qod=rqod, uri="No location", ) res_list.add_scan_log_to_list( host=current_host, name="HOST_END", value=str(int(time.time())), ) # Add the result to the scan collection if len(res_list): logger.debug( '%s: Inserting %d results into scan ' 'scan collection table', scan_id, len(res_list), ) self._daemon.scan_collection.add_result_list(scan_id, res_list) # Set the host scan progress as finished host_progress = dict() host_progress[current_host] = ScanProgress.FINISHED self._daemon.set_scan_progress_batch(scan_id, host_progress=host_progress) # Update the host status, Finished host. So ospd can # calculate the scan progress. # This is quite importan, since the final scan status depends on # the progress calculation. finished_host = list() finished_host.append(current_host) self._daemon.sort_host_finished(scan_id, finished_host) time.sleep(1) logger.debug('%s: End task', scan_id)
def exec_scan(self, scan_id: str): """ Starts the scanner for scan_id scan. """ # Get the host list target = self.scan_collection.get_host_list(scan_id) logger.info("The target list %s", target) host_list = target_str_to_list(target) # Get the port list ports = self.scan_collection.get_ports(scan_id) logger.info("The port list %s", ports) # Get exclude hosts list. It must not be scanned exclude_hosts = self.scan_collection.get_exclude_hosts(scan_id) logger.info("The exclude hosts list %s", exclude_hosts) # Get credentials for authenticated scans credentials = self.scan_collection.get_credentials(scan_id) for ( service, credential, ) in credentials.items(): cred_type = credential.get('type', '') username = credential.get('username', '') password = credential.get( 'password', '' ) # pylint: disable=unused-variable logger.info( "Credential for %s: user: %s, type: %s", service, username, cred_type, ) # Get the plugin list and its preferences, if the scanner # supports plugins nvts = self.scan_collection.get_vts( scan_id ) # pylint: disable=unused-variable # Ospd calculates the scan progress on fly, but it depends on the # results. Sometimes, the scanner can be more smart and detect # e.g. repeted hosts or hostnames which can not be resolved and # therefore will not be scanned. # In this cases, the amount of hosts scanned can differ with the amount # of hosts in the target list. Please consider to use the following # method if your scanner has the hability and provides these total hosts # and amount of dead hosts. self.set_scan_total_hosts( scan_id, count_total=len(host_list), ) self.scan_collection.set_amount_dead_hosts(scan_id, total_dead=0) # Run the scan. This template does not run any scan, but generates # a fake result for each host in the whole target list. # Also, it doesn't consider the excluded hosts. If the scanner allows # multiple hosts, you can execute the scanner just once and process # the results later. while host_list: # Get a host from the list current_host = host_list.pop() # Example using subprocess.Popen() to exec the scanner # cmd = [] # if self._niceness: # cmd += ['nice', '-n', self._niceness] # logger.debug("Starting scan with niceness %s", self._niceness) # cmd += ['scanner_exec', current_host] # try: # return subprocess.Popen(cmd, shell=False) # except (subprocess.SubprocessError, OSError) as e: # # the command is not available # logger.warning("Could not start scan process. Reason %s", e) # return None # In some point you may want to check if something goes wrong or # if the scan was stopped by the client. # If your check is not successful, you can stop the server # gracefully. status = self.get_scan_status(scan_id) if status == ScanStatus.INTERRUPTED: self.stop_scan_cleanup(scan_id) logger.error( 'Something when wrong for task %s.', scan_id, ) return elif status == ScanStatus.STOPPED or status == ScanStatus.FINISHED: logger.debug( 'Task %s stopped or finished.', scan_id, ) return # Scan simulation for each single host. # Run the scan against the host, and generates results. res_list = ResultList() res_type = int(uniform(1, 5)) # Error if res_type == 1: res_list.add_scan_error_to_list( host=current_host, hostname=current_host + ".hostname.net", name="Some test name", value="error running the script", port=ports, test_id="1234-5678", uri="No location", ) # Log elif res_type == 2: res_list.add_scan_log_to_list( host=current_host, hostname=current_host + ".hostname.net", name="Some test name", value="Some log", port=ports, qod="10", test_id="1234-5678", uri="No location", ) # Host detail elif res_type == 3: res_list.add_scan_host_detail_to_list( host=current_host, hostname=current_host + ".hostname.net", name="Some Test Name", value="Some host detail", uri="No location", ) # Alarm else: res_list.add_scan_alarm_to_list( host=current_host, hostname=current_host + ".hostname.net", name="Some Test Name", value="Some Alarm", port=ports, test_id="1234-5678", severity="10", qod="10", uri="No location", ) # Add the result to the scan collection if len(res_list): logger.debug( '%s: Inserting %d results into scan ' 'scan collection table', scan_id, len(res_list), ) self.scan_collection.add_result_list(scan_id, res_list) # Update the host status, so ospd can calculate the scan progress # This is quite importan, since the final scan status depends on # the progress calculation. finished_host = list() finished_host.append(current_host) self.sort_host_finished(scan_id, finished_host) # If you know exactly the host scan progress, you can use the # the following methods for a more precise progress calculation. host_progress = dict() # host_progress[current_host] = 45 # host_progress[current_host] = ScanProgress.DEAD_HOST host_progress[current_host] = ScanProgress.FINISHED self.set_scan_progress_batch(scan_id, host_progress=host_progress) time.sleep(1) logger.debug('%s: End task', scan_id)