def _update_scans(config_pk: int):
LOGGER.debug(F'Starting update scans: {config_pk}')
config = Config.objects.filter(pk=config_pk)
if config.exists():
config = config.first()
else:
LOGGER.error(F'Config: {config_pk} not exist!')
return None
try:
config.set_status(Config.Status.IN_PROGRESS)
manager = scanners_registry.get(config)
client = manager.get_client()
parser = manager.get_parser()
now_date = now()
LOGGER.info(F'Trying to download scan lists')
scan_list = client.get_scans()
scan_list = parser.get_scans_ids(scan_list)
LOGGER.info(F'scan list downloaded')
LOGGER.debug(F'Scan list: {scan_list}')
for scan_id in scan_list:
LOGGER.info(F'Trying to download report form {config.name}')
file = client.download_scan(scan_id, client.ReportFormat.XML)
path = _get_save_path(config)
file_name = '{}-{}.zip'.format(config.scanner, now().strftime('%H-%M-%S'))
full_file_path = Path(path) / file_name
LOGGER.info(F"Saving file: {full_file_path}")
thread_pool_executor.submit(save_scan, client, scan_id, file, full_file_path)
saved_scan = Scan.objects.create(config=config, file=str(full_file_path))
file_url = F"{getattr(settings, 'ABSOLUTE_URI', '')}{reverse('download_scan', args=[saved_scan.file_id])}"
targets = copy.deepcopy(file)
LOGGER.info(F'Retrieving discovered assets for {config.name}')
discovered_assets = AssetDocument.get_assets_with_tag(tag=AssetStatus.DISCOVERED, config=config)
LOGGER.info(F'Trying to parse scan file {scan_id}')
vulns, scanned_hosts = parser.parse(file, file_url)
LOGGER.info(F'File parsed: {scan_id}')
LOGGER.info(F'Trying to parse targets from file {scan_id}')
targets = parser.get_targets(targets)
LOGGER.info(F'Targets parsed: {scan_id}')
if targets:
LOGGER.info(F'Attempting to update discovered assets in {config.name}')
AssetDocument.update_gone_discovered_assets(targets=targets, scanned_hosts=scanned_hosts,
discovered_assets=discovered_assets, config=config)
LOGGER.info(F'Attempting to update vulns data in {config.name}')
VulnerabilityDocument.create_or_update(vulns, scanned_hosts, config)
config.last_scans_pull = now_date
config.set_status(Config.Status.SUCCESS)
config.save(update_fields=['last_scans_pull'])
except Exception as e:
config.set_status(status=Config.Status.ERROR, error_description=e)
LOGGER.error(F'Error while loading vulnerability data {e}')
finally:
thread_pool_executor.wait_for_all()
def test_update_gone_discovered_assets(self):
asset = AssetDocument.get_or_create('10.0.0.1')
AssetDocument.get_or_create('10.0.0.2')
self.assertEqual(2, Search().index(AssetDocument.Index.name).count())
discovered_assets = AssetDocument.get_assets_with_tag(
tag=AssetStatus.DISCOVERED, config=AssetConfigMock())
targets = netaddr.IPSet()
targets.add("10.0.0.0/8")
scanned_hosts = [asset]
AssetDocument.update_gone_discovered_assets(
targets=targets,
scanned_hosts=scanned_hosts,
discovered_assets=discovered_assets,
config=AssetConfigMock())
new_assets = Search().index(AssetDocument.Index.name).execute()
self.assertEqual(2, len(new_assets.hits))
for a in map(lambda new_asset: new_asset.to_dict(), new_assets.hits):
if a["id"] == "10.0.0.1":
self.assertEqual(a["tags"], ["DISCOVERED"])
elif a["id"] == "10.0.0.2":
self.assertCountEqual(a["tags"], ["DELETED", "DISCOVERED"])
def _update_scans(config_pk: int):
config = Config.objects.filter(pk=config_pk)
if config.exists():
config = config.first()
try:
config.set_status(Config.Status.IN_PROGRESS)
client, parser = scanners_registry.get(config)
now_date = now()
scan_list = client.get_scans(last_modification_date=config.last_scans_pull)
scan_list = parser.get_scans_ids(scan_list)
for scan_id in scan_list:
LOGGER.info(F'Trying to download report form {config.name}')
file = client.download_scan(scan_id)
targets = copy.deepcopy(file)
LOGGER.info(F'Retrieving discovered assets for {config.name}')
discovered_assets = AssetDocument.get_assets_with_tag(tag=AssetStatus.DISCOVERED, config=config)
LOGGER.info(F'Trying to parse scan file {scan_id}')
vulns, scanned_hosts = parser.parse(file)
LOGGER.info(F'File parsed: {scan_id}')
LOGGER.info(F'Trying to parse targets from file {scan_id}')
if hasattr(parser, "get_targets"):
targets = parser.get_targets(targets)
else:
targets = client.get_targets(targets)
LOGGER.info(F'Targets parsed: {scan_id}')
if targets:
LOGGER.info(F'Attempting to update discovered assets in {config.name}')
AssetDocument.update_gone_discovered_assets(targets=targets, scanned_hosts=scanned_hosts,
discovered_assets=discovered_assets, config=config)
LOGGER.info(F'Attempting to update vulns data in {config.name}')
VulnerabilityDocument.create_or_update(vulns, scanned_hosts, config)
config.last_scans_pull = now_date
config.set_status(Config.Status.SUCCESS)
config.save(update_fields=['last_scans_pull'])
except Exception as e:
config.set_status(status=Config.Status.ERROR, error_description=e)
LOGGER.error(F'Error while loading vulnerability data {e}')
finally:
thread_pool_executor.wait_for_all()
