def test_batch_result(self):
daemon = DummyWrapper([])
reslist = ResultList()
fs = FakeStream()
daemon.handle_command(
'<start_scan parallel="1">'
'<scanner_params />'
'<targets><target>'
'<hosts>a</hosts>'
'<ports>22</ports>'
'</target></targets>'
'</start_scan>',
fs,
)
response = fs.get_response()
scan_id = response.findtext('id')
reslist.add_scan_log_to_list(host='a', name='a')
reslist.add_scan_log_to_list(host='c', name='c')
reslist.add_scan_log_to_list(host='b', name='b')
daemon.scan_collection.add_result_list(scan_id, reslist)
hosts = ['a', 'c', 'b']
fs = FakeStream()
daemon.handle_command('<get_scans details="1"/>', fs)
response = fs.get_response()
results = response.findall("scan/results/")
for idx, res in enumerate(results):
att_dict = res.attrib
self.assertEqual(hosts[idx], att_dict['name'])
def report_openvas_results(self, db: BaseDB, scan_id: str) -> bool:
""" Get all result entries from redis kb. """
vthelper = VtHelper(self.nvti)
# Result messages come in the next form, with optional uri field
# type ||| host ip ||| hostname ||| port ||| OID ||| value [|||uri]
all_results = db.get_result()
res_list = ResultList()
total_dead = 0
for res in all_results:
if not res:
continue
msg = res.split('|||')
roid = msg[4].strip()
rqod = ''
rname = ''
current_host = msg[1].strip() if msg[1] else ''
rhostname = msg[2].strip() if msg[2] else ''
host_is_dead = "Host dead" in msg[5] or msg[0] == "DEADHOST"
host_deny = "Host access denied" in msg[5]
start_end_msg = msg[0] == "HOST_START" or msg[0] == "HOST_END"
vt_aux = None
# URI is optional and msg list length must be checked
ruri = ''
if len(msg) > 6:
ruri = msg[6]
if (roid and not host_is_dead and not host_deny
and not start_end_msg):
vt_aux = vthelper.get_single_vt(roid)
if (not vt_aux and not host_is_dead and not host_deny
and not start_end_msg):
logger.warning('Invalid VT oid %s for a result', roid)
if vt_aux:
if vt_aux.get('qod_type'):
qod_t = vt_aux.get('qod_type')
rqod = self.nvti.QOD_TYPES[qod_t]
elif vt_aux.get('qod'):
rqod = vt_aux.get('qod')
rname = vt_aux.get('name')
if msg[0] == 'ERRMSG':
res_list.add_scan_error_to_list(
host=current_host,
hostname=rhostname,
name=rname,
value=msg[5],
port=msg[3],
test_id=roid,
uri=ruri,
)
elif msg[0] == 'HOST_START' or msg[0] == 'HOST_END':
res_list.add_scan_log_to_list(
host=current_host,
name=msg[0],
value=msg[5],
)
elif msg[0] == 'LOG':
res_list.add_scan_log_to_list(
host=current_host,
hostname=rhostname,
name=rname,
value=msg[5],
port=msg[3],
qod=rqod,
test_id=roid,
uri=ruri,
)
elif msg[0] == 'HOST_DETAIL':
res_list.add_scan_host_detail_to_list(
host=current_host,
hostname=rhostname,
name=rname,
value=msg[5],
uri=ruri,
)
elif msg[0] == 'ALARM':
rseverity = self.get_severity_score(vt_aux)
res_list.add_scan_alarm_to_list(
host=current_host,
hostname=rhostname,
name=rname,
value=msg[5],
port=msg[3],
test_id=roid,
severity=rseverity,
qod=rqod,
uri=ruri,
)
# To process non-scanned dead hosts when
# test_alive_host_only in openvas is enable
elif msg[0] == 'DEADHOST':
try:
total_dead = int(msg[5])
except TypeError:
logger.debug('Error processing dead host count')
# Insert result batch into the scan collection table.
if len(res_list):
self.scan_collection.add_result_list(scan_id, res_list)
if total_dead:
self.scan_collection.set_amount_dead_hosts(scan_id,
total_dead=total_dead)
return len(res_list) > 0
def report_openvas_results(self, db: BaseDB, scan_id: str,
current_host: str):
""" Get all result entries from redis kb. """
res = db.get_result()
res_list = ResultList()
host_progress_batch = dict()
finished_host_batch = list()
while res:
msg = res.split('|||')
roid = msg[3].strip()
rqod = ''
rname = ''
rhostname = msg[1].strip() if msg[1] else ''
host_is_dead = "Host dead" in msg[4]
vt_aux = None
if roid and not host_is_dead:
vt_aux = copy.deepcopy(self.vts.get(roid))
if not vt_aux and not host_is_dead:
logger.warning('Invalid VT oid %s for a result', roid)
if vt_aux:
if vt_aux.get('qod_type'):
qod_t = vt_aux.get('qod_type')
rqod = self.nvti.QOD_TYPES[qod_t]
elif vt_aux.get('qod'):
rqod = vt_aux.get('qod')
rname = vt_aux.get('name')
if msg[0] == 'ERRMSG':
res_list.add_scan_error_to_list(
host=current_host,
hostname=rhostname,
name=rname,
value=msg[4],
port=msg[2],
test_id=roid,
)
if msg[0] == 'LOG':
res_list.add_scan_log_to_list(
host=current_host,
hostname=rhostname,
name=rname,
value=msg[4],
port=msg[2],
qod=rqod,
test_id=roid,
)
if msg[0] == 'HOST_DETAIL':
res_list.add_scan_host_detail_to_list(
host=current_host,
hostname=rhostname,
name=rname,
value=msg[4],
)
if msg[0] == 'ALARM':
rseverity = self.get_severity_score(vt_aux)
res_list.add_scan_alarm_to_list(
host=current_host,
hostname=rhostname,
name=rname,
value=msg[4],
port=msg[2],
test_id=roid,
severity=rseverity,
qod=rqod,
)
# To process non scanned dead hosts when
# test_alive_host_only in openvas is enable
if msg[0] == 'DEADHOST':
hosts = msg[3].split(',')
for _host in hosts:
if _host:
host_progress_batch[_host] = 100
finished_host_batch.append(_host)
res_list.add_scan_log_to_list(
host=_host,
hostname=rhostname,
name=rname,
value=msg[4],
port=msg[2],
qod=rqod,
test_id='',
)
timestamp = time.ctime(time.time())
res_list.add_scan_log_to_list(
host=_host,
name='HOST_START',
value=timestamp,
)
res_list.add_scan_log_to_list(
host=_host,
name='HOST_END',
value=timestamp,
)
vt_aux = None
del vt_aux
res = db.get_result()
# Insert result batch into the scan collection table.
if len(res_list):
self.scan_collection.add_result_list(scan_id, res_list)
if host_progress_batch:
self.set_scan_progress_batch(scan_id,
host_progress=host_progress_batch)
if finished_host_batch:
self.set_scan_host_finished(scan_id,
finished_hosts=finished_host_batch)
def exec_dry_run_scan(self, scan_id, nvti, ospd_params):
options = self._daemon.scan_collection.get_options(scan_id)
results_per_host = None
if "results_per_host" in options:
results_per_host = options.get("results_per_host")
if not results_per_host or not isinstance(results_per_host, int):
logger.debug("Using default value for results_per_host options")
results_per_host = ospd_params["results_per_host"].get("default")
# Get the host list
target = self._daemon.scan_collection.get_host_list(scan_id)
logger.info("The target list %s", target)
host_list = target_str_to_list(target)
# Get the port list
ports = self._daemon.scan_collection.get_ports(scan_id)
logger.info("The port list %s", ports)
tcp, _ = ports_as_list(ports)
# Get exclude hosts list. It must not be scanned
exclude_hosts = self._daemon.scan_collection.get_exclude_hosts(scan_id)
logger.info("The exclude hosts list %s", exclude_hosts)
self._daemon.set_scan_total_hosts(
scan_id,
count_total=len(host_list),
)
self._daemon.scan_collection.set_amount_dead_hosts(scan_id,
total_dead=0)
# Get list of VTS. Ignore script params
vts = list(self._daemon.scan_collection.get_vts(scan_id))
if "vt_groups" in vts:
vts.remove("vt_groups")
vthelper = VtHelper(nvti)
# Run the scan.
# Scan simulation for each single host.
# Run the scan against the host, and generates results.
while host_list:
# Get a host from the list
current_host = host_list.pop()
# Check if the scan was stopped.
status = self._daemon.get_scan_status(scan_id)
if status == ScanStatus.STOPPED or status == ScanStatus.FINISHED:
logger.debug(
'Task %s stopped or finished.',
scan_id,
)
return
res_list = ResultList()
res_list.add_scan_log_to_list(
host=current_host,
name="HOST_START",
value=str(int(time.time())),
)
# Generate N results per host. Default 10 results
res_count = 0
while res_count < results_per_host:
res_count += 1
oid = choice(vts)
port = choice(tcp)
vt = vthelper.get_single_vt(oid)
if vt:
if vt.get('qod_type'):
qod_t = vt.get('qod_type')
rqod = nvti.QOD_TYPES[qod_t]
elif vt.get('qod'):
rqod = vt.get('qod')
rname = vt.get('name')
else:
logger.debug("oid %s not found", oid)
res_type = int(uniform(1, 5))
# Error
if res_type == 1:
res_list.add_scan_error_to_list(
host=current_host,
hostname=current_host + ".hostname.net",
name=rname,
value="error running the script " + oid,
port=port,
test_id=oid,
uri="No location",
)
# Log
elif res_type == 2:
res_list.add_scan_log_to_list(
host=current_host,
hostname=current_host + ".hostname.net",
name=rname,
value="Log generate from a dry run scan for the script "
+ oid,
port=port,
qod=rqod,
test_id=oid,
uri="No location",
)
# Alarm
else:
r_severity = vthelper.get_severity_score(vt)
res_list.add_scan_alarm_to_list(
host=current_host,
hostname=current_host + ".hostname.net",
name=rname,
value="Log generate from a dry run scan for the script "
+ oid,
port=port,
test_id=oid,
severity=r_severity,
qod=rqod,
uri="No location",
)
res_list.add_scan_log_to_list(
host=current_host,
name="HOST_END",
value=str(int(time.time())),
)
# Add the result to the scan collection
if len(res_list):
logger.debug(
'%s: Inserting %d results into scan '
'scan collection table',
scan_id,
len(res_list),
)
self._daemon.scan_collection.add_result_list(scan_id, res_list)
# Set the host scan progress as finished
host_progress = dict()
host_progress[current_host] = ScanProgress.FINISHED
self._daemon.set_scan_progress_batch(scan_id,
host_progress=host_progress)
# Update the host status, Finished host. So ospd can
# calculate the scan progress.
# This is quite importan, since the final scan status depends on
# the progress calculation.
finished_host = list()
finished_host.append(current_host)
self._daemon.sort_host_finished(scan_id, finished_host)
time.sleep(1)
logger.debug('%s: End task', scan_id)
def report_results(self, results: list, scan_id: str) -> bool:
"""Reports all results given in a list.
Arguments:
results: list of results each list item must contain a dictionary
with following fields: result_type, host_ip, host_name, port, oid,
value, uri (optional)
"""
vthelper = VtHelper(self.nvti)
res_list = ResultList()
total_dead = 0
for res in results:
if not res:
continue
roid = res["oid"].strip()
rqod = ''
rname = ''
current_host = res["host_ip"].strip() if res["host_ip"] else ''
rhostname = res["host_name"].strip() if res["host_name"] else ''
host_is_dead = ("Host dead" in res["value"]
or res["result_type"] == "DEADHOST")
host_deny = "Host access denied" in res["value"]
start_end_msg = (res["result_type"] == "HOST_START"
or res["result_type"] == "HOST_END")
host_count = res["result_type"] == "HOSTS_COUNT"
vt_aux = None
# URI is optional and containing must be checked
ruri = res["uri"] if "uri" in res else ""
if (not host_is_dead and not host_deny and not start_end_msg
and not host_count):
if not roid and res["result_type"] != 'ERRMSG':
logger.warning('Missing VT oid for a result')
vt_aux = vthelper.get_single_vt(roid)
if not vt_aux:
logger.warning('Invalid VT oid %s for a result', roid)
else:
if vt_aux.get('qod_type'):
qod_t = vt_aux.get('qod_type')
rqod = self.nvti.QOD_TYPES[qod_t]
elif vt_aux.get('qod'):
rqod = vt_aux.get('qod')
rname = vt_aux.get('name')
if res["result_type"] == 'ERRMSG':
res_list.add_scan_error_to_list(
host=current_host,
hostname=rhostname,
name=rname,
value=res["value"],
port=res["port"],
test_id=roid,
uri=ruri,
)
elif (res["result_type"] == 'HOST_START'
or res["result_type"] == 'HOST_END'):
res_list.add_scan_log_to_list(
host=current_host,
name=res["result_type"],
value=res["value"],
)
elif res["result_type"] == 'LOG':
res_list.add_scan_log_to_list(
host=current_host,
hostname=rhostname,
name=rname,
value=res["value"],
port=res["port"],
qod=rqod,
test_id=roid,
uri=ruri,
)
elif res["result_type"] == 'HOST_DETAIL':
res_list.add_scan_host_detail_to_list(
host=current_host,
hostname=rhostname,
name=rname,
value=res["value"],
uri=ruri,
)
elif res["result_type"] == 'ALARM':
rseverity = vthelper.get_severity_score(vt_aux)
res_list.add_scan_alarm_to_list(
host=current_host,
hostname=rhostname,
name=rname,
value=res["value"],
port=res["port"],
test_id=roid,
severity=rseverity,
qod=rqod,
uri=ruri,
)
# To process non-scanned dead hosts when
# test_alive_host_only in openvas is enable
elif res["result_type"] == 'DEADHOST':
try:
total_dead = total_dead + int(res["value"])
except TypeError:
logger.debug('Error processing dead host count')
# To update total host count
if res["result_type"] == 'HOSTS_COUNT':
try:
count_total = int(res["value"])
logger.debug(
'%s: Set total hosts counted by OpenVAS: %d',
scan_id,
count_total,
)
self.set_scan_total_hosts(scan_id, count_total)
except TypeError:
logger.debug('Error processing total host count')
# Insert result batch into the scan collection table.
if len(res_list):
self.scan_collection.add_result_list(scan_id, res_list)
logger.debug(
'%s: Inserting %d results into scan collection table',
scan_id,
len(res_list),
)
if total_dead:
logger.debug(
'%s: Set dead hosts counted by OpenVAS: %d',
scan_id,
total_dead,
)
self.scan_collection.set_amount_dead_hosts(scan_id,
total_dead=total_dead)
return len(res_list) > 0
def report_openvas_results(self, db: BaseDB, scan_id: str,
current_host: str):
""" Get all result entries from redis kb. """
vthelper = VtHelper(self.nvti)
# Result messages come in the next form, with optional uri field
# type ||| hostname ||| port ||| OID ||| value [|||uri]
res = db.get_result()
res_list = ResultList()
total_dead = 0
while res:
msg = res.split('|||')
roid = msg[3].strip()
rqod = ''
rname = ''
rhostname = msg[1].strip() if msg[1] else ''
host_is_dead = "Host dead" in msg[4] or msg[0] == "DEADHOST"
host_deny = "Host access denied" in msg[4]
vt_aux = None
# URI is optional and msg list length must be checked
ruri = ''
if len(msg) > 5:
ruri = msg[5]
if roid and not host_is_dead and not host_deny:
vt_aux = vthelper.get_single_vt(roid)
if not vt_aux and not host_is_dead and not host_deny:
logger.warning('Invalid VT oid %s for a result', roid)
if vt_aux:
if vt_aux.get('qod_type'):
qod_t = vt_aux.get('qod_type')
rqod = self.nvti.QOD_TYPES[qod_t]
elif vt_aux.get('qod'):
rqod = vt_aux.get('qod')
rname = vt_aux.get('name')
if msg[0] == 'ERRMSG':
# Some errors are generated before a host is scanned
# use the hostname passed in the message if
# no current host is available.
if not current_host and rhostname:
current_host = rhostname
res_list.add_scan_error_to_list(
host=current_host,
hostname=rhostname,
name=rname,
value=msg[4],
port=msg[2],
test_id=roid,
uri=ruri,
)
if msg[0] == 'LOG':
res_list.add_scan_log_to_list(
host=current_host,
hostname=rhostname,
name=rname,
value=msg[4],
port=msg[2],
qod=rqod,
test_id=roid,
uri=ruri,
)
if msg[0] == 'HOST_DETAIL':
res_list.add_scan_host_detail_to_list(
host=current_host,
hostname=rhostname,
name=rname,
value=msg[4],
uri=ruri,
)
if msg[0] == 'ALARM':
rseverity = self.get_severity_score(vt_aux)
res_list.add_scan_alarm_to_list(
host=current_host,
hostname=rhostname,
name=rname,
value=msg[4],
port=msg[2],
test_id=roid,
severity=rseverity,
qod=rqod,
uri=ruri,
)
# To process non-scanned dead hosts when
# test_alive_host_only in openvas is enable
if msg[0] == 'DEADHOST':
try:
total_dead = int(msg[4])
except TypeError:
logger.debug('Error processing dead host count')
res = db.get_result()
# Insert result batch into the scan collection table.
if len(res_list):
self.scan_collection.add_result_list(scan_id, res_list)
if total_dead:
self.scan_collection.set_amount_dead_hosts(scan_id,
total_dead=total_dead)
def exec_scan(self, scan_id: str):
""" Starts the scanner for scan_id scan. """
# Get the host list
target = self.scan_collection.get_host_list(scan_id)
logger.info("The target list %s", target)
host_list = target_str_to_list(target)
# Get the port list
ports = self.scan_collection.get_ports(scan_id)
logger.info("The port list %s", ports)
# Get exclude hosts list. It must not be scanned
exclude_hosts = self.scan_collection.get_exclude_hosts(scan_id)
logger.info("The exclude hosts list %s", exclude_hosts)
# Get credentials for authenticated scans
credentials = self.scan_collection.get_credentials(scan_id)
for (
service,
credential,
) in credentials.items():
cred_type = credential.get('type', '')
username = credential.get('username', '')
password = credential.get(
'password', ''
) # pylint: disable=unused-variable
logger.info(
"Credential for %s: user: %s, type: %s",
service,
username,
cred_type,
)
# Get the plugin list and its preferences, if the scanner
# supports plugins
nvts = self.scan_collection.get_vts(
scan_id
) # pylint: disable=unused-variable
# Ospd calculates the scan progress on fly, but it depends on the
# results. Sometimes, the scanner can be more smart and detect
# e.g. repeted hosts or hostnames which can not be resolved and
# therefore will not be scanned.
# In this cases, the amount of hosts scanned can differ with the amount
# of hosts in the target list. Please consider to use the following
# method if your scanner has the hability and provides these total hosts
# and amount of dead hosts.
self.set_scan_total_hosts(
scan_id,
count_total=len(host_list),
)
self.scan_collection.set_amount_dead_hosts(scan_id, total_dead=0)
# Run the scan. This template does not run any scan, but generates
# a fake result for each host in the whole target list.
# Also, it doesn't consider the excluded hosts. If the scanner allows
# multiple hosts, you can execute the scanner just once and process
# the results later.
while host_list:
# Get a host from the list
current_host = host_list.pop()
# Example using subprocess.Popen() to exec the scanner
# cmd = []
# if self._niceness:
# cmd += ['nice', '-n', self._niceness]
# logger.debug("Starting scan with niceness %s", self._niceness)
# cmd += ['scanner_exec', current_host]
# try:
# return subprocess.Popen(cmd, shell=False)
# except (subprocess.SubprocessError, OSError) as e:
# # the command is not available
# logger.warning("Could not start scan process. Reason %s", e)
# return None
# In some point you may want to check if something goes wrong or
# if the scan was stopped by the client.
# If your check is not successful, you can stop the server
# gracefully.
status = self.get_scan_status(scan_id)
if status == ScanStatus.INTERRUPTED:
self.stop_scan_cleanup(scan_id)
logger.error(
'Something when wrong for task %s.',
scan_id,
)
return
elif status == ScanStatus.STOPPED or status == ScanStatus.FINISHED:
logger.debug(
'Task %s stopped or finished.',
scan_id,
)
return
# Scan simulation for each single host.
# Run the scan against the host, and generates results.
res_list = ResultList()
res_type = int(uniform(1, 5))
# Error
if res_type == 1:
res_list.add_scan_error_to_list(
host=current_host,
hostname=current_host + ".hostname.net",
name="Some test name",
value="error running the script",
port=ports,
test_id="1234-5678",
uri="No location",
)
# Log
elif res_type == 2:
res_list.add_scan_log_to_list(
host=current_host,
hostname=current_host + ".hostname.net",
name="Some test name",
value="Some log",
port=ports,
qod="10",
test_id="1234-5678",
uri="No location",
)
# Host detail
elif res_type == 3:
res_list.add_scan_host_detail_to_list(
host=current_host,
hostname=current_host + ".hostname.net",
name="Some Test Name",
value="Some host detail",
uri="No location",
)
# Alarm
else:
res_list.add_scan_alarm_to_list(
host=current_host,
hostname=current_host + ".hostname.net",
name="Some Test Name",
value="Some Alarm",
port=ports,
test_id="1234-5678",
severity="10",
qod="10",
uri="No location",
)
# Add the result to the scan collection
if len(res_list):
logger.debug(
'%s: Inserting %d results into scan '
'scan collection table',
scan_id,
len(res_list),
)
self.scan_collection.add_result_list(scan_id, res_list)
# Update the host status, so ospd can calculate the scan progress
# This is quite importan, since the final scan status depends on
# the progress calculation.
finished_host = list()
finished_host.append(current_host)
self.sort_host_finished(scan_id, finished_host)
# If you know exactly the host scan progress, you can use the
# the following methods for a more precise progress calculation.
host_progress = dict()
# host_progress[current_host] = 45
# host_progress[current_host] = ScanProgress.DEAD_HOST
host_progress[current_host] = ScanProgress.FINISHED
self.set_scan_progress_batch(scan_id, host_progress=host_progress)
time.sleep(1)
logger.debug('%s: End task', scan_id)