def test_bad_ipv4_cidr(self):
addresses = target_str_to_list('195.70.81.0/32')
self.assertIsNotNone(addresses)
self.assertEqual(len(addresses), 0)
addresses = target_str_to_list('195.70.81.0/31')
self.assertIsNotNone(addresses)
self.assertEqual(len(addresses), 0)
def simplify_exclude_host_count(self, scan_id: str) -> int:
"""Remove from exclude_hosts the received hosts in the finished_hosts
list sent by the client.
The finished hosts are sent also as exclude hosts for backward
compatibility purposses.
Return:
Count of excluded host.
"""
exc_hosts_list = target_str_to_list(self.get_exclude_hosts(scan_id))
logger.debug(
'%s: Excluded Hosts: %s',
scan_id,
pformat(exc_hosts_list),
)
finished_hosts_list = target_str_to_list(
self.get_finished_hosts(scan_id))
logger.debug(
'%s: Finished Hosts: %s',
scan_id,
pformat(finished_hosts_list),
)
# Remove finished hosts from excluded host list
if finished_hosts_list and exc_hosts_list:
for finished in finished_hosts_list:
if finished in exc_hosts_list:
exc_hosts_list.remove(finished)
# Remove excluded hosts which don't belong to the target list
host_list = target_str_to_list(self.get_host_list(scan_id))
excluded_simplified = 0
invalid_exc_hosts = 0
if exc_hosts_list:
for exc_host in exc_hosts_list:
if exc_host in host_list:
excluded_simplified += 1
else:
invalid_exc_hosts += 1
if invalid_exc_hosts > 0:
logger.warning(
"Please check the excluded host list. It contains hosts which "
"do not belong to the target. This warning can be ignored if "
"this was done on purpose (e.g. to exclude specific hostname)."
)
# Set scan_info's excluded simplified to propagate excluded count
# to parent process.
self.scans_table[scan_id]['excluded_simplified'] = excluded_simplified
return excluded_simplified
def test_24_net(self):
addresses = target_str_to_list('195.70.81.0/24')
self.assertIsNotNone(addresses)
self.assertEqual(len(addresses), 254)
for i in range(1, 255):
self.assertIn(f'195.70.81.{str(i)}', addresses)
def test_range(self):
addresses = target_str_to_list('195.70.81.1-10')
self.assertIsNotNone(addresses)
self.assertEqual(len(addresses), 10)
for i in range(1, 10):
self.assertIn('195.70.81.%d' % i, addresses)
def process_exclude_hosts(self, scan_id: str, exclude_hosts: str) -> None:
""" Process the exclude hosts before launching the scans."""
exc_hosts_list = ''
if not exclude_hosts:
return
exc_hosts_list = target_str_to_list(exclude_hosts)
self.remove_scan_hosts_from_target_progress(scan_id, exc_hosts_list)
def test_range(self):
addresses = target_str_to_list('195.70.81.0-10')
self.assertIsNotNone(addresses)
self.assertEqual(len(addresses), 11)
for i in range(0, 10):
self.assertIn(f'195.70.81.{str(i)}', addresses)
def test_target_str_with_trailing_comma(self):
addresses = target_str_to_list(',195.70.81.1,195.70.81.2,')
self.assertIsNotNone(addresses)
self.assertEqual(len(addresses), 2)
for i in range(1, 2):
self.assertIn(f'195.70.81.{str(i)}', addresses)
def get_host_count(self, scan_id: str) -> int:
"""Get total host count in the target."""
host = self.get_host_list(scan_id)
total_hosts = 0
if host:
total_hosts = len(target_str_to_list(host))
return total_hosts
def process_finished_hosts(self, scan_id: str) -> None:
""" Process the finished hosts before launching the scans."""
finished_hosts = self.scan_collection.get_finished_hosts(scan_id)
if not finished_hosts:
return
exc_finished_hosts_list = target_str_to_list(finished_hosts)
self.scan_collection.set_host_finished(scan_id, exc_finished_hosts_list)
def simplify_exclude_host_list(self, scan_id: str) -> List[Any]:
""" Remove from exclude_hosts the received hosts in the finished_hosts
list sent by the client.
The finished hosts are sent also as exclude hosts for backward
compatibility purposses.
"""
exc_hosts_list = target_str_to_list(self.get_exclude_hosts(scan_id))
finished_hosts_list = target_str_to_list(
self.get_finished_hosts(scan_id))
if finished_hosts_list and exc_hosts_list:
for finished in finished_hosts_list:
if finished in exc_hosts_list:
exc_hosts_list.remove(finished)
return exc_hosts_list
def get_hosts_unfinished(self, scan_id):
""" Get a list of unfinished hosts."""
unfinished_hosts = list()
for target in self.scans_table[scan_id]['finished_hosts']:
unfinished_hosts.extend(target_str_to_list(target))
for target in self.scans_table[scan_id]['finished_hosts']:
for host in self.scans_table[scan_id]['finished_hosts'][target]:
unfinished_hosts.remove(host)
return unfinished_hosts
def get_hosts_unfinished(self, scan_id: str) -> List[Any]:
""" Get a list of unfinished hosts."""
unfinished_hosts = target_str_to_list(self.get_host_list(scan_id))
finished_hosts = self.get_hosts_finished(scan_id)
for host in finished_hosts:
unfinished_hosts.remove(host)
return unfinished_hosts
def get_target_progress(self, scan_id, target):
""" Get a target's current progress value.
The value is calculated with the progress of each single host
in the target."""
total_hosts = len(target_str_to_list(target))
exc_hosts_list = target_str_to_list(
self.get_exclude_hosts(scan_id, target)
)
exc_hosts = len(exc_hosts_list) if exc_hosts_list else 0
host_progresses = self.scans_table[scan_id]['target_progress'].get(
target
)
try:
t_prog = sum(host_progresses.values()) / (total_hosts - exc_hosts)
except ZeroDivisionError:
LOGGER.error(
"Zero division error in %s", self.get_target_progress.__name__
)
raise
return t_prog
def process_finished_hosts(self, scan_id: str,
finished_hosts: str) -> None:
""" Process the finished hosts before launching the scans.
Set finished hosts as finished with 100% to calculate
the scan progress."""
exc_hosts_list = ''
if not finished_hosts:
return
exc_hosts_list = target_str_to_list(finished_hosts)
for host in exc_hosts_list:
self.set_scan_host_finished(scan_id, host)
self.set_scan_host_progress(scan_id, host, 100)
def calculate_target_progress(self, scan_id: str) -> float:
""" Get a target's current progress value.
The value is calculated with the progress of each single host
in the target."""
host = self.get_host_list(scan_id)
total_hosts = len(target_str_to_list(host))
exc_hosts_list = self.simplify_exclude_host_list(scan_id)
exc_hosts = len(exc_hosts_list) if exc_hosts_list else 0
host_progresses = self.scans_table[scan_id].get('target_progress')
try:
t_prog = sum(host_progresses.values()) / (total_hosts - exc_hosts
) # type: float
except ZeroDivisionError:
LOGGER.error(
"Zero division error in %s",
self.calculate_target_progress.__name__,
)
raise
return t_prog
def exec_dry_run_scan(self, scan_id, nvti, ospd_params):
options = self._daemon.scan_collection.get_options(scan_id)
results_per_host = None
if "results_per_host" in options:
results_per_host = options.get("results_per_host")
if not results_per_host or not isinstance(results_per_host, int):
logger.debug("Using default value for results_per_host options")
results_per_host = ospd_params["results_per_host"].get("default")
# Get the host list
target = self._daemon.scan_collection.get_host_list(scan_id)
logger.info("The target list %s", target)
host_list = target_str_to_list(target)
# Get the port list
ports = self._daemon.scan_collection.get_ports(scan_id)
logger.info("The port list %s", ports)
tcp, _ = ports_as_list(ports)
# Get exclude hosts list. It must not be scanned
exclude_hosts = self._daemon.scan_collection.get_exclude_hosts(scan_id)
logger.info("The exclude hosts list %s", exclude_hosts)
self._daemon.set_scan_total_hosts(
scan_id,
count_total=len(host_list),
)
self._daemon.scan_collection.set_amount_dead_hosts(scan_id,
total_dead=0)
# Get list of VTS. Ignore script params
vts = list(self._daemon.scan_collection.get_vts(scan_id))
if "vt_groups" in vts:
vts.remove("vt_groups")
vthelper = VtHelper(nvti)
# Run the scan.
# Scan simulation for each single host.
# Run the scan against the host, and generates results.
while host_list:
# Get a host from the list
current_host = host_list.pop()
# Check if the scan was stopped.
status = self._daemon.get_scan_status(scan_id)
if status == ScanStatus.STOPPED or status == ScanStatus.FINISHED:
logger.debug(
'Task %s stopped or finished.',
scan_id,
)
return
res_list = ResultList()
res_list.add_scan_log_to_list(
host=current_host,
name="HOST_START",
value=str(int(time.time())),
)
# Generate N results per host. Default 10 results
res_count = 0
while res_count < results_per_host:
res_count += 1
oid = choice(vts)
port = choice(tcp)
vt = vthelper.get_single_vt(oid)
if vt:
if vt.get('qod_type'):
qod_t = vt.get('qod_type')
rqod = nvti.QOD_TYPES[qod_t]
elif vt.get('qod'):
rqod = vt.get('qod')
rname = vt.get('name')
else:
logger.debug("oid %s not found", oid)
res_type = int(uniform(1, 5))
# Error
if res_type == 1:
res_list.add_scan_error_to_list(
host=current_host,
hostname=current_host + ".hostname.net",
name=rname,
value="error running the script " + oid,
port=port,
test_id=oid,
uri="No location",
)
# Log
elif res_type == 2:
res_list.add_scan_log_to_list(
host=current_host,
hostname=current_host + ".hostname.net",
name=rname,
value="Log generate from a dry run scan for the script "
+ oid,
port=port,
qod=rqod,
test_id=oid,
uri="No location",
)
# Alarm
else:
r_severity = vthelper.get_severity_score(vt)
res_list.add_scan_alarm_to_list(
host=current_host,
hostname=current_host + ".hostname.net",
name=rname,
value="Log generate from a dry run scan for the script "
+ oid,
port=port,
test_id=oid,
severity=r_severity,
qod=rqod,
uri="No location",
)
res_list.add_scan_log_to_list(
host=current_host,
name="HOST_END",
value=str(int(time.time())),
)
# Add the result to the scan collection
if len(res_list):
logger.debug(
'%s: Inserting %d results into scan '
'scan collection table',
scan_id,
len(res_list),
)
self._daemon.scan_collection.add_result_list(scan_id, res_list)
# Set the host scan progress as finished
host_progress = dict()
host_progress[current_host] = ScanProgress.FINISHED
self._daemon.set_scan_progress_batch(scan_id,
host_progress=host_progress)
# Update the host status, Finished host. So ospd can
# calculate the scan progress.
# This is quite importan, since the final scan status depends on
# the progress calculation.
finished_host = list()
finished_host.append(current_host)
self._daemon.sort_host_finished(scan_id, finished_host)
time.sleep(1)
logger.debug('%s: End task', scan_id)
def exec_scan(self, scan_id: str):
""" Starts the scanner for scan_id scan. """
# Get the host list
target = self.scan_collection.get_host_list(scan_id)
logger.info("The target list %s", target)
host_list = target_str_to_list(target)
# Get the port list
ports = self.scan_collection.get_ports(scan_id)
logger.info("The port list %s", ports)
# Get exclude hosts list. It must not be scanned
exclude_hosts = self.scan_collection.get_exclude_hosts(scan_id)
logger.info("The exclude hosts list %s", exclude_hosts)
# Get credentials for authenticated scans
credentials = self.scan_collection.get_credentials(scan_id)
for (
service,
credential,
) in credentials.items():
cred_type = credential.get('type', '')
username = credential.get('username', '')
password = credential.get(
'password', ''
) # pylint: disable=unused-variable
logger.info(
"Credential for %s: user: %s, type: %s",
service,
username,
cred_type,
)
# Get the plugin list and its preferences, if the scanner
# supports plugins
nvts = self.scan_collection.get_vts(
scan_id
) # pylint: disable=unused-variable
# Ospd calculates the scan progress on fly, but it depends on the
# results. Sometimes, the scanner can be more smart and detect
# e.g. repeted hosts or hostnames which can not be resolved and
# therefore will not be scanned.
# In this cases, the amount of hosts scanned can differ with the amount
# of hosts in the target list. Please consider to use the following
# method if your scanner has the hability and provides these total hosts
# and amount of dead hosts.
self.set_scan_total_hosts(
scan_id,
count_total=len(host_list),
)
self.scan_collection.set_amount_dead_hosts(scan_id, total_dead=0)
# Run the scan. This template does not run any scan, but generates
# a fake result for each host in the whole target list.
# Also, it doesn't consider the excluded hosts. If the scanner allows
# multiple hosts, you can execute the scanner just once and process
# the results later.
while host_list:
# Get a host from the list
current_host = host_list.pop()
# Example using subprocess.Popen() to exec the scanner
# cmd = []
# if self._niceness:
# cmd += ['nice', '-n', self._niceness]
# logger.debug("Starting scan with niceness %s", self._niceness)
# cmd += ['scanner_exec', current_host]
# try:
# return subprocess.Popen(cmd, shell=False)
# except (subprocess.SubprocessError, OSError) as e:
# # the command is not available
# logger.warning("Could not start scan process. Reason %s", e)
# return None
# In some point you may want to check if something goes wrong or
# if the scan was stopped by the client.
# If your check is not successful, you can stop the server
# gracefully.
status = self.get_scan_status(scan_id)
if status == ScanStatus.INTERRUPTED:
self.stop_scan_cleanup(scan_id)
logger.error(
'Something when wrong for task %s.',
scan_id,
)
return
elif status == ScanStatus.STOPPED or status == ScanStatus.FINISHED:
logger.debug(
'Task %s stopped or finished.',
scan_id,
)
return
# Scan simulation for each single host.
# Run the scan against the host, and generates results.
res_list = ResultList()
res_type = int(uniform(1, 5))
# Error
if res_type == 1:
res_list.add_scan_error_to_list(
host=current_host,
hostname=current_host + ".hostname.net",
name="Some test name",
value="error running the script",
port=ports,
test_id="1234-5678",
uri="No location",
)
# Log
elif res_type == 2:
res_list.add_scan_log_to_list(
host=current_host,
hostname=current_host + ".hostname.net",
name="Some test name",
value="Some log",
port=ports,
qod="10",
test_id="1234-5678",
uri="No location",
)
# Host detail
elif res_type == 3:
res_list.add_scan_host_detail_to_list(
host=current_host,
hostname=current_host + ".hostname.net",
name="Some Test Name",
value="Some host detail",
uri="No location",
)
# Alarm
else:
res_list.add_scan_alarm_to_list(
host=current_host,
hostname=current_host + ".hostname.net",
name="Some Test Name",
value="Some Alarm",
port=ports,
test_id="1234-5678",
severity="10",
qod="10",
uri="No location",
)
# Add the result to the scan collection
if len(res_list):
logger.debug(
'%s: Inserting %d results into scan '
'scan collection table',
scan_id,
len(res_list),
)
self.scan_collection.add_result_list(scan_id, res_list)
# Update the host status, so ospd can calculate the scan progress
# This is quite importan, since the final scan status depends on
# the progress calculation.
finished_host = list()
finished_host.append(current_host)
self.sort_host_finished(scan_id, finished_host)
# If you know exactly the host scan progress, you can use the
# the following methods for a more precise progress calculation.
host_progress = dict()
# host_progress[current_host] = 45
# host_progress[current_host] = ScanProgress.DEAD_HOST
host_progress[current_host] = ScanProgress.FINISHED
self.set_scan_progress_batch(scan_id, host_progress=host_progress)
time.sleep(1)
logger.debug('%s: End task', scan_id)
