def _update_scans(config_pk: int):
LOGGER.debug(F'Starting update scans: {config_pk}')
config = Config.objects.filter(pk=config_pk)
if config.exists():
config = config.first()
else:
LOGGER.error(F'Config: {config_pk} not exist!')
return None
try:
config.set_status(Config.Status.IN_PROGRESS)
manager = scanners_registry.get(config)
client = manager.get_client()
parser = manager.get_parser()
now_date = now()
LOGGER.info(F'Trying to download scan lists')
scan_list = client.get_scans()
scan_list = parser.get_scans_ids(scan_list)
LOGGER.info(F'scan list downloaded')
LOGGER.debug(F'Scan list: {scan_list}')
for scan_id in scan_list:
LOGGER.info(F'Trying to download report form {config.name}')
file = client.download_scan(scan_id, client.ReportFormat.XML)
path = _get_save_path(config)
file_name = '{}-{}.zip'.format(config.scanner, now().strftime('%H-%M-%S'))
full_file_path = Path(path) / file_name
LOGGER.info(F"Saving file: {full_file_path}")
thread_pool_executor.submit(save_scan, client, scan_id, file, full_file_path)
saved_scan = Scan.objects.create(config=config, file=str(full_file_path))
file_url = F"{getattr(settings, 'ABSOLUTE_URI', '')}{reverse('download_scan', args=[saved_scan.file_id])}"
targets = copy.deepcopy(file)
LOGGER.info(F'Retrieving discovered assets for {config.name}')
discovered_assets = AssetDocument.get_assets_with_tag(tag=AssetStatus.DISCOVERED, config=config)
LOGGER.info(F'Trying to parse scan file {scan_id}')
vulns, scanned_hosts = parser.parse(file, file_url)
LOGGER.info(F'File parsed: {scan_id}')
LOGGER.info(F'Trying to parse targets from file {scan_id}')
targets = parser.get_targets(targets)
LOGGER.info(F'Targets parsed: {scan_id}')
if targets:
LOGGER.info(F'Attempting to update discovered assets in {config.name}')
AssetDocument.update_gone_discovered_assets(targets=targets, scanned_hosts=scanned_hosts,
discovered_assets=discovered_assets, config=config)
LOGGER.info(F'Attempting to update vulns data in {config.name}')
VulnerabilityDocument.create_or_update(vulns, scanned_hosts, config)
config.last_scans_pull = now_date
config.set_status(Config.Status.SUCCESS)
config.save(update_fields=['last_scans_pull'])
except Exception as e:
config.set_status(status=Config.Status.ERROR, error_description=e)
LOGGER.error(F'Error while loading vulnerability data {e}')
finally:
thread_pool_executor.wait_for_all()
def test_update_discovered_asset(self):
asset_tenant_1 = self.create_asset(self.config_tenant_1.name)
discovered_asset = AssetDocument.get_or_create(
asset_tenant_1.ip_address)
cve = create_cve()
create_vulnerability(discovered_asset, cve)
self.assertEqual(1, Search().index(AssetDocument.Index.name).count())
AssetDocument.create_or_update({asset_tenant_1.id: asset_tenant_1})
thread_pool_executor.wait_for_all()
self.assertEqual(1, Search().index(AssetDocument.Index.name).count())
self.assertEqual(
1,
Search().index(VulnerabilityDocument.Index.name).count())
result = VulnerabilityDocument.search().filter(
'term', cve__id='CVE-2017-0002').execute()
self.assertEqual(result.hits[0].asset.id, asset_tenant_1.id)
self.assertEqual(result.hits[0].asset.ip_address,
asset_tenant_1.ip_address)
self.assertEqual(result.hits[0].asset.confidentiality_requirement,
asset_tenant_1.confidentiality_requirement)
self.assertEqual(result.hits[0].asset.availability_requirement,
asset_tenant_1.availability_requirement)
def _update_assets(config_id: int):
config = Config.objects.filter(pk=config_id)
if config.exists():
config = config.first()
try:
config.set_status(Config.Status.IN_PROGRESS)
client = RalphClient(config)
parser = AssetsParser(config)
LOGGER.info(F'Start loading data from Ralph: {config.name}')
users = client.get_users()
users = OwnerParser.parse(users)
assets = client.get_data_center_assets()
assets = parser.parse(assets, users)
AssetDocument.create_or_update(assets, config)
LOGGER.info(
F'Finish loading data center assets from Ralph: {config.name}')
assets = client.get_virtual_assets()
assets = parser.parse(assets, users)
AssetDocument.create_or_update(assets, config)
LOGGER.info(
F'Finish loading virtual assets from Ralph: {config.name}')
LOGGER.info(F'Finish loading data from Ralph: {config.name}')
config.set_status(Config.Status.SUCCESS)
except Exception as ex:
LOGGER.error(F'Error with loading data from Ralph: {ex}')
config.set_status(status=Config.Status.ERROR, error_description=ex)
finally:
thread_pool_executor.wait_for_all()
def test_update_discovered_asset(self):
asset = AssetDocument.get_or_create('10.0.0.1')
self.assertEqual(asset.tags, [AssetStatus.DISCOVERED])
self.assertEqual(1, Search().index(AssetDocument.Index.name).count())
asset = AssetDocument(ip_address='10.0.0.1',
os='Windows',
id=1,
confidentiality_requirement='NOT_DEFINED',
integrity_requirement='NOT_DEFINED',
availability_requirement='NOT_DEFINED',
hostname='hostname_1')
AssetDocument.create_or_update({asset.id: asset}, AssetConfigMock())
thread_pool_executor.wait_for_all()
self.assertEqual(1, Search().index(AssetDocument.Index.name).count())
result = AssetDocument.search().filter(
'term', ip_address='10.0.0.1').execute()
uut = result.hits[0]
self.assertEqual(uut.os, 'Windows')
self.assertEqual(uut.ip_address, '10.0.0.1')
self.assertEqual(uut.hostname, 'hostname_1')
self.assertEqual(uut.tags, [])
def update_assets(self, mock_api):
mock_api().get_assets.return_value = self.hosts
mock_api().get_users.return_value = self.users
_update_assets(self.config_id)
self.assertEqual(2, Search().index(AssetDocument.Index.name).count())
self.assertEqual(AssetDocument.search().filter('term', ip_address='10.0.0.23').count(), 1)
self.assertEqual(AssetDocument.search().filter('term', ip_address='10.0.0.25').count(), 1)
_update_assets(self.config_id)
self.assertEqual(2, Search().index(AssetDocument.Index.name).count())
def create(self, item: dict, iface: dict):
asset = AssetDocument()
for field in AssetDocument.get_fields_name():
parser = getattr(self, field, None)
try:
if parser:
setattr(asset, field, parser(item, iface))
except Exception as ex:
LOGGER.debug(F'Unable to parse field {field} ex: {ex}')
setattr(asset, field, 'UNKNOWN')
asset.source = self.__config.name
self.__parsed[asset.id] = asset
def start_processing_per_tenant(vulnerability_index: str, asset_index: str):
LOGGER.info(
F'Calculation for {vulnerability_index} and {asset_index} started')
try:
assets_count = AssetDocument.search(index=asset_index).filter(
~Q('match', tags=AssetStatus.DELETED)).count()
vuln_search = VulnerabilityDocument.search(
index=vulnerability_index).filter(
~Q('match', tags=VulnerabilityStatus.FIXED)
& ~Q('match', asset__tags=AssetStatus.DELETED))
prepare(vulnerability_index)
workers_count = get_workers_count()
vuln_count = vuln_search.count()
slices_count = 1
if vuln_count > 500:
slices_count = vuln_count // workers_count
slices_count = slices_count if slices_count <= workers_count else workers_count
(group(
_processing.si(idx, slices_count, assets_count,
vulnerability_index) for idx in range(slices_count))
| _end_processing.si(vulnerability_index, asset_index))()
except Exception as ex:
LOGGER.error(F'Unknown processing exception {ex}')
def create_asset(tags):
return AssetDocument(ip_address='10.10.10.1',
os='Windows',
id=1,
confidentiality_requirement=Impact.LOW,
hostname='test-hostname',
tags=[tags])
def main():
print('Download CVEs')
group(update_cwe.si() | group(
update_cve.si(year) for year in range(START_YEAR,
datetime.now().year + 1))
| update_exploits.si())().get()
print('Update Assets')
_update_assets(RalphConfig.objects.first().pk)
asset_search = AssetDocument.search()
times = 0
while asset_search.count() == 0:
print('Assets count still 0, waiting...')
time.sleep(1)
if times > 4:
print('Unable to count assets, exiting...')
exit(1)
times += 1
asset_count = asset_search.count()
# Draw enough unique CVEs to make them min 4 times more than there are hosts
cve_sets = get_cve_sets(asset_count)
# draw as many as there will be 130 on average per host
generate_vulns(asset_count, asset_search, cve_sets)
start_processing()
def test_document(self):
self.create_asset(ip_address='10.10.10.1')
result = AssetDocument.search().filter(
'term', ip_address='10.10.10.1').execute()
self.assertEqual(len(result.hits), 1)
uut = result.hits[0]
self.assertEqual(uut.os, 'Windows')
self.assertEqual(uut.confidentiality_requirement.name, Impact.LOW.name)
self.assertEqual(uut.integrity_requirement.name, Impact.LOW.name)
self.assertEqual(uut.availability_requirement.name, Impact.LOW.name)
self.assertEqual(uut.confidentiality_requirement.second_value,
Impact.LOW.second_value)
self.assertEqual(uut.integrity_requirement.second_value,
Impact.LOW.second_value)
self.assertEqual(uut.availability_requirement.second_value,
Impact.LOW.second_value)
self.assertEqual(uut.business_owner, [{
'name': 'bo_name',
'email': '*****@*****.**',
'department': 'department',
'team': ['team']
}])
self.assertEqual(uut.technical_owner, [{
'name': 'to_name',
'email': '*****@*****.**',
'department': 'department',
'team': ['team']
}])
self.assertEqual(uut.hostname, 'test-hostname')
self.assertTrue(uut.created_date)
self.assertEqual(uut.tags, [])
self.assertTrue(uut.modified_date)
def parse(self, report) -> [Dict, Dict]:
for r in report.findall('.//results/result'):
if float(r.find('nvt//cvss_base').text) > 0:
ip_address = r.find('./host').text
self.__scanned_host.append(ip_address)
asset = AssetDocument.get_or_create(ip_address, self._config)
tags = self.parse_tags(r.find('./nvt/tags').text)
for cve in r.find('./nvt//cve').text.split(','):
port = r.find('./port').text.split('/')[0]
protocol = r.find('./port').text.split('/')[1]
oid = r.find('./nvt').attrib.get('oid')
cve = self.get_cve(cve, oid, tags)
if port == 'general':
port = None
protocol = None
uid = self._vuln_id(ip_address, port, oid)
self.__parsed[uid] = VulnerabilityDocument(
id=uid,
port=port,
protocol=protocol,
description=r.find('./description').text,
solution=tags['solution'],
cve=cve,
asset=asset,
source='OpenVas'
)
return self.__parsed, self.__scanned_host
def test_update_gone_discovered_assets(self):
AssetDocument.get_or_create('10.0.0.1')
AssetDocument.get_or_create('10.0.0.2')
self.assertEqual(2, Search().index(AssetDocument.Index.name).count())
discovered_assets = AssetDocument.get_assets_with_tag(
tag=AssetStatus.DISCOVERED, config=AssetConfigMock())
targets = netaddr.IPSet()
targets.add("10.0.0.0/8")
scanned_hosts = ["10.0.0.1"]
AssetDocument.update_gone_discovered_assets(
targets=targets,
scanned_hosts=scanned_hosts,
discovered_assets=discovered_assets,
config=AssetConfigMock())
new_assets = Search().index(AssetDocument.Index.name).execute()
self.assertEqual(2, len(new_assets.hits))
for a in map(lambda new_asset: new_asset.to_dict(), new_assets.hits):
if a["id"] == "10.0.0.1":
self.assertEqual(a["tags"], ["DISCOVERED"])
elif a["id"] == "10.0.0.2":
self.assertCountEqual(a["tags"], ["DELETED", "DISCOVERED"])
def create_asset(ip_address='10.10.10.10',
asset_id=None,
hostname='HOSTNAME',
save=True) -> AssetDocument:
if not asset_id:
asset_id = ip_address
asset = AssetDocument(id=asset_id,
ip_address=ip_address,
mac_address='mac_address',
os='Windows',
hostname=hostname,
confidentiality_requirement=Impact.LOW,
integrity_requirement=Impact.LOW,
availability_requirement=Impact.LOW)
if save:
asset.save()
return asset
def test_delete_asset(self):
asset_1 = self.create_asset(asset_id=1,
ip_address='10.0.0.1',
hostname='hostname_1')
asset_2 = self.create_asset(asset_id=2,
ip_address='10.0.0.2',
hostname='hostname_2')
self.assertEqual(2, Search().index(AssetDocument.Index.name).count())
AssetDocument.create_or_update({asset_1.id: asset_1},
AssetConfigMock())
thread_pool_executor.wait_for_all()
result = AssetDocument.search().filter(
Q('match', tags=AssetStatus.DELETED)).execute()
self.assertEqual(1, len(result.hits))
self.assertEqual(result.hits[0].ip_address, asset_2.ip_address)
self.assertEqual(result.hits[0].id, asset_2.id)
def _update_scans(config_pk: int):
config = Config.objects.filter(pk=config_pk)
if config.exists():
config = config.first()
try:
config.set_status(Config.Status.IN_PROGRESS)
client, parser = scanners_registry.get(config)
now_date = now()
scan_list = client.get_scans(last_modification_date=config.last_scans_pull)
scan_list = parser.get_scans_ids(scan_list)
for scan_id in scan_list:
LOGGER.info(F'Trying to download report form {config.name}')
file = client.download_scan(scan_id)
targets = copy.deepcopy(file)
LOGGER.info(F'Retrieving discovered assets for {config.name}')
discovered_assets = AssetDocument.get_assets_with_tag(tag=AssetStatus.DISCOVERED, config=config)
LOGGER.info(F'Trying to parse scan file {scan_id}')
vulns, scanned_hosts = parser.parse(file)
LOGGER.info(F'File parsed: {scan_id}')
LOGGER.info(F'Trying to parse targets from file {scan_id}')
if hasattr(parser, "get_targets"):
targets = parser.get_targets(targets)
else:
targets = client.get_targets(targets)
LOGGER.info(F'Targets parsed: {scan_id}')
if targets:
LOGGER.info(F'Attempting to update discovered assets in {config.name}')
AssetDocument.update_gone_discovered_assets(targets=targets, scanned_hosts=scanned_hosts,
discovered_assets=discovered_assets, config=config)
LOGGER.info(F'Attempting to update vulns data in {config.name}')
VulnerabilityDocument.create_or_update(vulns, scanned_hosts, config)
config.last_scans_pull = now_date
config.set_status(Config.Status.SUCCESS)
config.save(update_fields=['last_scans_pull'])
except Exception as e:
config.set_status(status=Config.Status.ERROR, error_description=e)
LOGGER.error(F'Error while loading vulnerability data {e}')
finally:
thread_pool_executor.wait_for_all()
def test_get_or_create_call_get_existing_asset(self):
asset_1 = self.create_asset(asset_id=1,
ip_address='10.0.0.1',
hostname='hostname_1')
self.create_asset(asset_id=2,
ip_address='10.0.0.2',
hostname='hostname_2')
self.assertEqual(2, Search().index(AssetDocument.Index.name).count())
AssetDocument.create_or_update({asset_1.id: asset_1},
AssetConfigMock())
asset_3 = AssetDocument.get_or_create('10.0.0.1')
self.assertEqual(2, Search().index(AssetDocument.Index.name).count())
self.assertEqual(asset_3.ip_address, asset_1.ip_address)
self.assertEqual(asset_3.hostname, asset_1.hostname)
self.assertEqual(asset_3.id, asset_1.id)
self.assertEqual(asset_3.confidentiality_requirement,
asset_1.confidentiality_requirement)
self.assertEqual(asset_3.integrity_requirement,
asset_1.integrity_requirement)
self.assertEqual(asset_3.availability_requirement,
asset_1.availability_requirement)
def test_update_asset(self):
asset_tenant_1 = self.create_asset(self.config_tenant_1.name)
asset_tenant_2 = self.create_asset(self.config_tenant_2.name)
AssetDocument.create_or_update({asset_tenant_1.id: asset_tenant_1},
self.config_tenant_1)
AssetDocument.create_or_update({asset_tenant_2.id: asset_tenant_2},
self.config_tenant_2)
thread_pool_executor.wait_for_all()
asset_tenant_1 = self.create_asset(self.config_tenant_1.name)
asset_tenant_1.hostname = 'tenant-test'
AssetDocument.create_or_update({asset_tenant_1.id: asset_tenant_1},
self.config_tenant_1)
thread_pool_executor.wait_for_all()
result = AssetDocument.search(index='test.tenant.asset').filter(
'term', ip_address='10.10.10.1').execute()
self.assertEqual(1, len(result.hits))
self.assertEqual(result.hits[0].ip_address, '10.10.10.1')
self.assertEqual(result.hits[0].hostname, 'tenant-test')
def get_asset(self, ip_address):
return AssetDocument.get_or_create(ip_address, self._config)
def create(item: RestrictedElement, config) -> AssetDocument:
ip_address = item.find(".//tag[@name='host-ip']").text
return AssetDocument.get_or_create(ip_address, config)