def test_already_locked(self, mock_logger):
lock_file_path = self.temp_dir / "test.lock"
lock_file_aux = LockFile(lock_file_path)
lock_file_aux._acquire_lock() # pylint: disable = protected-access
self.assertTrue(lock_file_aux.has_lock())
lock_file = LockFile(lock_file_path)
lock_file._acquire_lock() # pylint: disable = protected-access
self.assertFalse(lock_file.has_lock())
assert_called(mock_logger.debug)
lock_file_aux._release_lock() # pylint: disable = protected-access
def test_context_manager(self):
lock_file_path = self.temp_dir / 'test.lock'
lock_file = LockFile(lock_file_path)
with lock_file:
self.assertTrue(lock_file.is_locked())
self.assertTrue(lock_file.has_lock())
self.assertTrue(lock_file_path.is_file())
self.assertFalse(lock_file.is_locked())
self.assertFalse(lock_file.has_lock())
self.assertFalse(lock_file_path.is_file())
def test_create_paren_dirs_fail(self, mock_logger):
lock_file_path = Path('/root/lock/file/test.lock')
lock_file = LockFile(lock_file_path)
lock_file._acquire_lock()
self.assertFalse(lock_file.has_lock())
assert_called_once(mock_logger.error)
def test_acquire_lock(self):
lock_file_path = self.temp_dir / 'test.lock'
lock_file = LockFile(lock_file_path)
lock_file._acquire_lock()
self.assertTrue(lock_file.has_lock())
self.assertTrue(lock_file_path.exists())
lock_file._release_lock()
def test_acquire_lock(self):
lock_file_path = self.temp_dir / "test.lock"
lock_file = LockFile(lock_file_path)
lock_file._acquire_lock() # pylint: disable = protected-access
self.assertTrue(lock_file.has_lock())
self.assertTrue(lock_file_path.exists())
lock_file._release_lock() # pylint: disable = protected-access
def test_already_locked(self):
lock_file_path = self.temp_dir / 'test.lock'
lock_file_path.touch()
lock_file = LockFile(lock_file_path)
lock_file.acquire_lock()
self.assertFalse(lock_file.has_lock())
self.assertTrue(lock_file.is_locked())
self.assertTrue(lock_file_path.exists())
def test_create_paren_dirs_fail(self, mock_logger):
lock_file_path = MagicMock(spec=Path).return_value
parent = MagicMock(spec=PosixPath)
lock_file_path.parent = parent
parent.mkdir.side_effect = PermissionError
lock_file = LockFile(lock_file_path)
lock_file._acquire_lock()
self.assertFalse(lock_file.has_lock())
assert_called_once(mock_logger.error)
def test_create_parent_dirs(self):
lock_file_path = self.temp_dir / 'foo' / 'bar' / 'test.lock'
lock_file = LockFile(lock_file_path)
lock_file.acquire_lock()
self.assertTrue(lock_file.has_lock())
self.assertTrue(lock_file.is_locked())
self.assertTrue(lock_file_path.exists())
self.assertTrue(lock_file_path.parent.is_dir())
self.assertTrue(lock_file_path.parent.parent.is_dir())
def test_create_parent_dirs(self):
lock_file_path = self.temp_dir / "foo" / "bar" / "test.lock"
lock_file = LockFile(lock_file_path)
lock_file._acquire_lock()
self.assertTrue(lock_file.has_lock())
self.assertTrue(lock_file_path.exists())
self.assertTrue(lock_file_path.parent.is_dir())
self.assertTrue(lock_file_path.parent.parent.is_dir())
lock_file._release_lock()
class OSPDopenvas(OSPDaemon):
"""Class for ospd-openvas daemon."""
def __init__(
self,
*,
niceness=None,
lock_file_dir='/var/lib/openvas',
mqtt_broker_address="localhost",
mqtt_broker_port=1883,
disable_notus_hashsum_verification=False,
**kwargs,
):
"""Initializes the ospd-openvas daemon's internal data."""
self.main_db = MainDB()
notus_dir = kwargs.get('notus_feed_dir')
notus = None
if notus_dir:
ndir = Path(notus_dir)
verifier = hashsum_verificator(ndir,
disable_notus_hashsum_verification)
notus = Notus(ndir, self.main_db.ctx, verifier)
self.nvti = NVTICache(
self.main_db,
notus,
)
super().__init__(
customvtfilter=OpenVasVtsFilter(self.nvti),
storage=dict,
file_storage_dir=lock_file_dir,
**kwargs,
)
self.server_version = __version__
self._niceness = str(niceness)
self.feed_lock = LockFile(Path(lock_file_dir) / 'feed-update.lock')
self.daemon_info['name'] = 'OSPd OpenVAS'
self.scanner_info['name'] = 'openvas'
self.scanner_info['version'] = '' # achieved during self.init()
self.scanner_info['description'] = OSPD_DESC
for name, param in OSPD_PARAMS.items():
self.set_scanner_param(name, param)
self._sudo_available = None
self._is_running_as_root = None
self.scan_only_params = dict()
self._mqtt_broker_address = mqtt_broker_address
self._mqtt_broker_port = mqtt_broker_port
def init(self, server: BaseServer) -> None:
notus_handler = NotusResultHandler(self.report_results)
if self._mqtt_broker_address:
try:
client = MQTTClient(self._mqtt_broker_address,
self._mqtt_broker_port, "ospd")
daemon = MQTTDaemon(client)
subscriber = MQTTSubscriber(client)
subscriber.subscribe(ResultMessage,
notus_handler.result_handler)
daemon.run()
except (ConnectionRefusedError, gaierror, ValueError) as e:
logger.error(
"Could not connect to MQTT broker at %s, error was: %s."
" Unable to get results from Notus.",
self._mqtt_broker_address,
e,
)
else:
logger.info(
"MQTT Broker Adress empty. MQTT disabled. Unable to get Notus"
" results.")
self.scan_collection.init()
server.start(self.handle_client_stream)
self.scanner_info['version'] = Openvas.get_version()
self.set_params_from_openvas_settings()
with self.feed_lock.wait_for_lock():
Openvas.load_vts_into_redis()
self.set_feed_info()
logger.debug("Calculating vts integrity check hash...")
vthelper = VtHelper(self.nvti)
self.vts.sha256_hash = vthelper.calculate_vts_collection_hash()
self.initialized = True
def set_params_from_openvas_settings(self):
"""Set OSPD_PARAMS with the params taken from the openvas executable."""
param_list = Openvas.get_settings()
for elem in param_list: # pylint: disable=consider-using-dict-items
if elem not in OSPD_PARAMS:
self.scan_only_params[elem] = param_list[elem]
else:
OSPD_PARAMS[elem]['default'] = param_list[elem]
def feed_is_outdated(self, current_feed: str) -> Optional[bool]:
"""Compare the current feed with the one in the disk.
Return:
False if there is no new feed.
True if the feed version in disk is newer than the feed in
redis cache.
None if there is no feed on the disk.
"""
current_feed = safe_int(current_feed)
if current_feed is None:
logger.debug(
"Wrong PLUGIN_SET format in plugins feed file "
"'plugin_feed_info.inc'. Format has to"
" be yyyymmddhhmm. For example 'PLUGIN_SET = \"201910251033\"'"
)
feed_date = None
feed_info = self.get_feed_info()
if feed_info:
feed_date = safe_int(feed_info.get("PLUGIN_SET"))
logger.debug("Current feed version: %s", current_feed)
logger.debug("Plugin feed version: %s", feed_date)
return ((not feed_date) or (not current_feed)
or (current_feed < feed_date))
def get_feed_info(self) -> Dict[str, Any]:
"""Parses the current plugin_feed_info.inc file"""
plugins_folder = self.scan_only_params.get('plugins_folder')
if not plugins_folder:
raise OspdOpenvasError("Error: Path to plugins folder not found.")
feed_info_file = Path(plugins_folder) / 'plugin_feed_info.inc'
if not feed_info_file.exists():
self.set_params_from_openvas_settings()
logger.debug('Plugins feed file %s not found.', feed_info_file)
return {}
feed_info = {}
with feed_info_file.open(encoding='utf-8') as fcontent:
for line in fcontent:
try:
key, value = line.split('=', 1)
except ValueError:
continue
key = key.strip()
value = value.strip()
value = value.replace(';', '')
value = value.replace('"', '')
if value:
feed_info[key] = value
return feed_info
def set_feed_info(self):
"""Set feed current information to be included in the response of
<get_version/> command
"""
current_feed = self.nvti.get_feed_version()
self.set_vts_version(vts_version=current_feed)
feed_info = self.get_feed_info()
self.set_feed_vendor(feed_info.get("FEED_VENDOR", "unknown"))
self.set_feed_home(feed_info.get("FEED_HOME", "unknown"))
self.set_feed_name(feed_info.get("PLUGIN_FEED", "unknown"))
def check_feed_self_test(self) -> Dict:
"""Perform a feed sync self tests and check if the feed lock file is
locked.
"""
feed_status = dict()
# It is locked by the current process
if self.feed_lock.has_lock():
feed_status["lockfile_in_use"] = '1'
# Check if we can get the lock
else:
with self.feed_lock as fl:
# It is available
if fl.has_lock():
feed_status["lockfile_in_use"] = '0'
# Locked by another process
else:
feed_status["lockfile_in_use"] = '1'
feed = Feed()
_exit_error, _error_msg = feed.perform_feed_sync_self_test_success()
feed_status["self_test_exit_error"] = str(_exit_error)
feed_status["self_test_error_msg"] = _error_msg
return feed_status
def check_feed(self):
"""Check if there is a feed update.
Wait until all the running scans finished. Set a flag to announce there
is a pending feed update, which avoids to start a new scan.
"""
if not self.vts.is_cache_available:
return
current_feed = self.nvti.get_feed_version()
is_outdated = self.feed_is_outdated(current_feed)
# Check if the nvticache in redis is outdated
if not current_feed or is_outdated:
with self.feed_lock as fl:
if fl.has_lock():
self.initialized = False
Openvas.load_vts_into_redis()
self.set_feed_info()
vthelper = VtHelper(self.nvti)
self.vts.sha256_hash = (
vthelper.calculate_vts_collection_hash())
self.initialized = True
else:
logger.debug("The feed was not upload or it is outdated, "
"but other process is locking the update. "
"Trying again later...")
return
def scheduler(self):
"""This method is called periodically to run tasks."""
self.check_feed()
def get_vt_iterator(self,
vt_selection: List[str] = None,
details: bool = True) -> Iterator[Tuple[str, Dict]]:
vthelper = VtHelper(self.nvti)
return vthelper.get_vt_iterator(vt_selection, details)
@property
def is_running_as_root(self) -> bool:
"""Check if it is running as root user."""
if self._is_running_as_root is not None:
return self._is_running_as_root
self._is_running_as_root = False
if geteuid() == 0:
self._is_running_as_root = True
return self._is_running_as_root
@property
def sudo_available(self) -> bool:
"""Checks that sudo is available"""
if self._sudo_available is not None:
return self._sudo_available
if self.is_running_as_root:
self._sudo_available = False
return self._sudo_available
self._sudo_available = Openvas.check_sudo()
return self._sudo_available
def check(self) -> bool:
"""Checks that openvas command line tool is found and
is executable."""
has_openvas = Openvas.check()
if not has_openvas:
logger.error(
'openvas executable not available. Please install openvas'
' into your PATH.')
return has_openvas
def report_openvas_scan_status(self, kbdb: BaseDB, scan_id: str):
"""Get all status entries from redis kb.
Arguments:
kbdb: KB context where to get the status from.
scan_id: Scan ID to identify the current scan.
"""
all_status = kbdb.get_scan_status()
all_hosts = dict()
finished_hosts = list()
for res in all_status:
try:
current_host, launched, total = res.split('/')
except ValueError:
continue
try:
if float(total) == 0:
continue
elif float(total) == ScanProgress.DEAD_HOST:
host_prog = ScanProgress.DEAD_HOST
else:
host_prog = int((float(launched) / float(total)) * 100)
except TypeError:
continue
all_hosts[current_host] = host_prog
if (host_prog == ScanProgress.DEAD_HOST
or host_prog == ScanProgress.FINISHED):
finished_hosts.append(current_host)
logger.debug('%s: Host %s has progress: %d', scan_id, current_host,
host_prog)
self.set_scan_progress_batch(scan_id, host_progress=all_hosts)
self.sort_host_finished(scan_id, finished_hosts)
def report_openvas_results(self, db: BaseDB, scan_id: str) -> bool:
"""Get all result entries from redis kb.
Arguments:
db: KB context where to get the results from.
scan_id: Scan ID to identify the current scan.
"""
# result_type|||host ip|||hostname|||port|||OID|||value[|||uri]
all_results = db.get_result()
results = []
for res in all_results:
if not res:
continue
msg = res.split('|||')
result = {
"result_type": msg[0],
"host_ip": msg[1],
"host_name": msg[2],
"port": msg[3],
"oid": msg[4],
"value": msg[5],
}
if len(msg) > 6:
result["uri"] = msg[6]
results.append(result)
return self.report_results(results, scan_id)
def report_results(self, results: list, scan_id: str) -> bool:
"""Reports all results given in a list.
Arguments:
results: list of results each list item must contain a dictionary
with following fields: result_type, host_ip, host_name, port, oid,
value, uri (optional)
"""
vthelper = VtHelper(self.nvti)
res_list = ResultList()
total_dead = 0
for res in results:
if not res:
continue
roid = res["oid"].strip()
rqod = ''
rname = ''
current_host = res["host_ip"].strip() if res["host_ip"] else ''
rhostname = res["host_name"].strip() if res["host_name"] else ''
host_is_dead = ("Host dead" in res["value"]
or res["result_type"] == "DEADHOST")
host_deny = "Host access denied" in res["value"]
start_end_msg = (res["result_type"] == "HOST_START"
or res["result_type"] == "HOST_END")
host_count = res["result_type"] == "HOSTS_COUNT"
vt_aux = None
# URI is optional and containing must be checked
ruri = res["uri"] if "uri" in res else ""
if (not host_is_dead and not host_deny and not start_end_msg
and not host_count):
if not roid and res["result_type"] != 'ERRMSG':
logger.warning('Missing VT oid for a result')
vt_aux = vthelper.get_single_vt(roid)
if not vt_aux:
logger.warning('Invalid VT oid %s for a result', roid)
else:
if vt_aux.get('qod_type'):
qod_t = vt_aux.get('qod_type')
rqod = self.nvti.QOD_TYPES[qod_t]
elif vt_aux.get('qod'):
rqod = vt_aux.get('qod')
rname = vt_aux.get('name')
if res["result_type"] == 'ERRMSG':
res_list.add_scan_error_to_list(
host=current_host,
hostname=rhostname,
name=rname,
value=res["value"],
port=res["port"],
test_id=roid,
uri=ruri,
)
elif (res["result_type"] == 'HOST_START'
or res["result_type"] == 'HOST_END'):
res_list.add_scan_log_to_list(
host=current_host,
name=res["result_type"],
value=res["value"],
)
elif res["result_type"] == 'LOG':
res_list.add_scan_log_to_list(
host=current_host,
hostname=rhostname,
name=rname,
value=res["value"],
port=res["port"],
qod=rqod,
test_id=roid,
uri=ruri,
)
elif res["result_type"] == 'HOST_DETAIL':
res_list.add_scan_host_detail_to_list(
host=current_host,
hostname=rhostname,
name=rname,
value=res["value"],
uri=ruri,
)
elif res["result_type"] == 'ALARM':
rseverity = vthelper.get_severity_score(vt_aux)
res_list.add_scan_alarm_to_list(
host=current_host,
hostname=rhostname,
name=rname,
value=res["value"],
port=res["port"],
test_id=roid,
severity=rseverity,
qod=rqod,
uri=ruri,
)
# To process non-scanned dead hosts when
# test_alive_host_only in openvas is enable
elif res["result_type"] == 'DEADHOST':
try:
total_dead = total_dead + int(res["value"])
except TypeError:
logger.debug('Error processing dead host count')
# To update total host count
if res["result_type"] == 'HOSTS_COUNT':
try:
count_total = int(res["value"])
logger.debug(
'%s: Set total hosts counted by OpenVAS: %d',
scan_id,
count_total,
)
self.set_scan_total_hosts(scan_id, count_total)
except TypeError:
logger.debug('Error processing total host count')
# Insert result batch into the scan collection table.
if len(res_list):
self.scan_collection.add_result_list(scan_id, res_list)
logger.debug(
'%s: Inserting %d results into scan collection table',
scan_id,
len(res_list),
)
if total_dead:
logger.debug(
'%s: Set dead hosts counted by OpenVAS: %d',
scan_id,
total_dead,
)
self.scan_collection.set_amount_dead_hosts(scan_id,
total_dead=total_dead)
return len(res_list) > 0
@staticmethod
def is_openvas_process_alive(openvas_process: psutil.Popen) -> bool:
try:
if openvas_process.status() == psutil.STATUS_ZOMBIE:
logger.debug("Process is a Zombie, waiting for it to clean up")
openvas_process.wait()
except psutil.NoSuchProcess:
return False
return openvas_process.is_running()
def stop_scan_cleanup(
self,
kbdb: BaseDB,
scan_id: str,
ovas_process: psutil.Popen, # pylint: disable=arguments-differ
):
"""Set a key in redis to indicate the wrapper is stopped.
It is done through redis because it is a new multiprocess
instance and it is not possible to reach the variables
of the grandchild process.
Indirectly sends SIGUSR1 to the running openvas scan process
via an invocation of openvas with the --scan-stop option to
stop it."""
if kbdb:
# Set stop flag in redis
kbdb.stop_scan(scan_id)
# Check if openvas is running
if ovas_process.is_running():
# Cleaning in case of Zombie Process
if ovas_process.status() == psutil.STATUS_ZOMBIE:
logger.debug(
'%s: Process with PID %s is a Zombie process.'
' Cleaning up...',
scan_id,
ovas_process.pid,
)
ovas_process.wait()
# Stop openvas process and wait until it stopped
else:
can_stop_scan = Openvas.stop_scan(
scan_id,
not self.is_running_as_root and self.sudo_available,
)
if not can_stop_scan:
logger.debug(
'Not possible to stop scan process: %s.',
ovas_process,
)
return
logger.debug('Stopping process: %s', ovas_process)
while ovas_process.is_running():
if ovas_process.status() == psutil.STATUS_ZOMBIE:
ovas_process.wait()
else:
time.sleep(0.1)
else:
logger.debug(
"%s: Process with PID %s already stopped",
scan_id,
ovas_process.pid,
)
# Clean redis db
for scan_db in kbdb.get_scan_databases():
self.main_db.release_database(scan_db)
def exec_scan(self, scan_id: str):
"""Starts the OpenVAS scanner for scan_id scan."""
params = self.scan_collection.get_options(scan_id)
if params.get("dry_run"):
dryrun = DryRun(self)
dryrun.exec_dry_run_scan(scan_id, self.nvti, OSPD_PARAMS)
return
do_not_launch = False
kbdb = self.main_db.get_new_kb_database()
scan_prefs = PreferenceHandler(scan_id, kbdb, self.scan_collection,
self.nvti)
kbdb.add_scan_id(scan_id)
scan_prefs.prepare_target_for_openvas()
if not scan_prefs.prepare_ports_for_openvas():
self.add_scan_error(scan_id,
name='',
host='',
value='Invalid port list.')
do_not_launch = True
# Set credentials
if not scan_prefs.prepare_credentials_for_openvas():
error = ('All authentifications contain errors.' +
'Starting unauthenticated scan instead.')
self.add_scan_error(
scan_id,
name='',
host='',
value=error,
)
logger.error(error)
errors = scan_prefs.get_error_messages()
for e in errors:
error = 'Malformed credential. ' + e
self.add_scan_error(
scan_id,
name='',
host='',
value=error,
)
logger.error(error)
if not scan_prefs.prepare_plugins_for_openvas():
self.add_scan_error(scan_id,
name='',
host='',
value='No VTS to run.')
do_not_launch = True
scan_prefs.prepare_main_kbindex_for_openvas()
scan_prefs.prepare_host_options_for_openvas()
scan_prefs.prepare_scan_params_for_openvas(OSPD_PARAMS)
scan_prefs.prepare_reverse_lookup_opt_for_openvas()
scan_prefs.prepare_alive_test_option_for_openvas()
# VT preferences are stored after all preferences have been processed,
# since alive tests preferences have to be able to overwrite default
# preferences of ping_host.nasl for the classic method.
scan_prefs.prepare_nvt_preferences()
scan_prefs.prepare_boreas_alive_test()
# Release memory used for scan preferences.
del scan_prefs
scan_stopped = self.get_scan_status(scan_id) == ScanStatus.STOPPED
if do_not_launch or kbdb.scan_is_stopped(scan_id) or scan_stopped:
self.main_db.release_database(kbdb)
return
openvas_process = Openvas.start_scan(
scan_id,
not self.is_running_as_root and self.sudo_available,
self._niceness,
)
if openvas_process is None:
self.main_db.release_database(kbdb)
return
kbdb.add_scan_process_id(openvas_process.pid)
logger.debug('pid = %s', openvas_process.pid)
# Wait until the scanner starts and loads all the preferences.
while kbdb.get_status(scan_id) == 'new':
res = openvas_process.poll()
if res and res < 0:
self.stop_scan_cleanup(kbdb, scan_id, openvas_process)
logger.error(
'It was not possible run the task %s, since openvas ended '
'unexpectedly with errors during launching.',
scan_id,
)
return
time.sleep(1)
got_results = False
while True:
openvas_process_is_alive = self.is_openvas_process_alive(
openvas_process)
target_is_finished = kbdb.target_is_finished(scan_id)
scan_stopped = self.get_scan_status(scan_id) == ScanStatus.STOPPED
# Report new Results and update status
got_results = self.report_openvas_results(kbdb, scan_id)
self.report_openvas_scan_status(kbdb, scan_id)
# Check if the client stopped the whole scan
if scan_stopped:
logger.debug('%s: Scan stopped by the client', scan_id)
self.stop_scan_cleanup(kbdb, scan_id, openvas_process)
# clean main_db, but wait for scanner to finish.
while not kbdb.target_is_finished(scan_id):
if not self.is_openvas_process_alive(openvas_process):
break
logger.debug('%s: Waiting for openvas to finish', scan_id)
time.sleep(1)
self.main_db.release_database(kbdb)
return
# Scan end. No kb in use for this scan id
if target_is_finished:
logger.debug('%s: Target is finished', scan_id)
break
if not openvas_process_is_alive:
logger.error(
'Task %s was unexpectedly stopped or killed.',
scan_id,
)
self.add_scan_error(
scan_id,
name='',
host='',
value='Task was unexpectedly stopped or killed.',
)
# check for scanner error messages before leaving.
self.report_openvas_results(kbdb, scan_id)
kbdb.stop_scan(scan_id)
for scan_db in kbdb.get_scan_databases():
self.main_db.release_database(scan_db)
self.main_db.release_database(kbdb)
return
# Wait a second before trying to get result from redis if there
# was no results before.
# Otherwise, wait 50 msec to give access other process to redis.
if not got_results:
time.sleep(1)
else:
time.sleep(0.05)
got_results = False
# Sleep a second to be sure to get all notus results
time.sleep(1)
# Delete keys from KB related to this scan task.
logger.debug('%s: End Target. Release main database', scan_id)
self.main_db.release_database(kbdb)