def test_asset_updated(self):
self.asset_2 = create_asset('10.10.10.11')
create_vulnerability(self.asset, self.cve)
create_vulnerability(self.asset_2, self.cve)
self.cve_2 = create_cve('CVE-2017-0003')
create_vulnerability(self.asset, self.cve_2)
create_vulnerability(self.asset_2, self.cve_2)
self.assertEqual(Search().index(VulnerabilityDocument.Index.name).count(), 4)
self.asset.confidentiality_requirement = AssetImpact.HIGH
self.asset.integrity_requirement = AssetImpact.HIGH
self.asset.save()
thread_pool_executor.wait_for_all()
self.assertEqual(Search().index(VulnerabilityDocument.Index.name).count(), 4)
result_1 = VulnerabilityDocument.search().filter(
'term', asset__ip_address=self.asset.ip_address).execute()
self.assertEqual(len(result_1.hits), 2)
self.assertEqual(result_1.hits[0].asset.confidentiality_requirement, self.asset.confidentiality_requirement)
self.assertEqual(result_1.hits[0].asset.integrity_requirement, self.asset.integrity_requirement)
self.assertEqual(result_1.hits[1].asset.confidentiality_requirement, self.asset.confidentiality_requirement)
self.assertEqual(result_1.hits[1].asset.integrity_requirement, self.asset.integrity_requirement)
result_2 = VulnerabilityDocument.search().filter(
'term', asset__ip_address=self.asset_2.ip_address).execute()
self.assertEqual(len(result_2.hits), 2)
self.assertEqual(result_2.hits[0].asset.confidentiality_requirement, self.asset_2.confidentiality_requirement)
self.assertEqual(result_2.hits[0].asset.integrity_requirement, self.asset_2.integrity_requirement)
self.assertEqual(result_2.hits[1].asset.confidentiality_requirement, self.asset_2.confidentiality_requirement)
self.assertEqual(result_2.hits[1].asset.integrity_requirement, self.asset_2.integrity_requirement)
def test_cve_updated(self):
self.asset_2 = create_asset('10.10.10.11')
self.cve_2 = create_cve('CVE-2017-0003')
create_vulnerability(self.asset, self.cve)
create_vulnerability(self.asset, self.cve_2)
create_vulnerability(self.asset_2, self.cve)
create_vulnerability(self.asset_2, self.cve_2)
self.assertEqual(Search().index(VulnerabilityDocument.Index.name).count(), 4)
self.cve.access_vector_v2 = metrics.AccessVectorV2.LOCAL
self.cve.save()
thread_pool_executor.wait_for_all()
self.assertEqual(Search().index(VulnerabilityDocument.Index.name).count(), 4)
result_1 = VulnerabilityDocument.search().filter('term', cve__id=self.cve.id).execute()
self.assertEqual(len(result_1.hits), 2)
self.assertEqual(result_1.hits[0].cve.access_vector_v2, self.cve.access_vector_v2)
self.assertEqual(result_1.hits[1].cve.access_vector_v2, self.cve.access_vector_v2)
result_2 = VulnerabilityDocument.search().filter('term', cve__id=self.cve_2.id).execute()
self.assertEqual(len(result_2.hits), 2)
self.assertEqual(result_2.hits[0].cve.access_vector_v2, self.cve_2.access_vector_v2)
self.assertEqual(result_2.hits[1].cve.access_vector_v2, self.cve_2.access_vector_v2)
def test_update_discovered_asset(self):
asset_tenant_1 = self.create_asset(self.config_tenant_1.name)
discovered_asset = AssetDocument.get_or_create(
asset_tenant_1.ip_address)
cve = create_cve()
create_vulnerability(discovered_asset, cve)
self.assertEqual(1, Search().index(AssetDocument.Index.name).count())
AssetDocument.create_or_update({asset_tenant_1.id: asset_tenant_1})
thread_pool_executor.wait_for_all()
self.assertEqual(1, Search().index(AssetDocument.Index.name).count())
self.assertEqual(
1,
Search().index(VulnerabilityDocument.Index.name).count())
result = VulnerabilityDocument.search().filter(
'term', cve__id='CVE-2017-0002').execute()
self.assertEqual(result.hits[0].asset.id, asset_tenant_1.id)
self.assertEqual(result.hits[0].asset.ip_address,
asset_tenant_1.ip_address)
self.assertEqual(result.hits[0].asset.confidentiality_requirement,
asset_tenant_1.confidentiality_requirement)
self.assertEqual(result.hits[0].asset.availability_requirement,
asset_tenant_1.availability_requirement)
def test_call_update_exploits(self, get_file):
self.assertEqual(Search().index(CveDocument.Index.name).count(), 2)
get_file.return_value = self.data
update_exploits()
get_file.assert_called_once_with(
'https://www.cve-search.org/feeds/via4.json')
thread_pool_executor.wait_for_all()
self.assertEqual(Search().index(CveDocument.Index.name).count(), 2)
cve = CveDocument.search().filter(
'term',
id='CVE-2017-0008').sort('-modified_date').execute().hits[0]
prev_modified_date = cve.modified_date
self.assertEqual(len(cve.exploits), 1)
self.assertEqual(cve.exploits,
[{
'id': '44904',
'url': 'https://www.exploit-db.com/exploits/44904'
}])
update_exploits()
thread_pool_executor.wait_for_all()
self.assertEqual(Search().index(CveDocument.Index.name).count(), 2)
cve = CveDocument.search().filter(
'term',
id='CVE-2017-0008').sort('-modified_date').execute().hits[0]
self.assertEqual(cve.modified_date, prev_modified_date)
self.assertEqual(len(cve.exploits), 1)
self.assertEqual(cve.exploits,
[{
'id': '44904',
'url': 'https://www.exploit-db.com/exploits/44904'
}])
def test_reopen_vulnerability(self):
vulnerability = create_vulnerability(self.asset, self.cve)
self.assertEqual(VulnerabilityDocument.search().count(), 1)
VulnerabilityDocument.create_or_update({}, [self.asset.ip_address],
ConfigMock())
thread_pool_executor.wait_for_all()
self.assertEqual(VulnerabilityDocument.search().count(), 1)
result = VulnerabilityDocument.search().filter(
'term', asset__ip_address=self.asset.ip_address).execute()
self.assertEqual(result.hits[0].tags,
['test', VulnerabilityStatus.FIXED])
VulnerabilityDocument.create_or_update(
{vulnerability.id: vulnerability}, [self.asset.ip_address],
ConfigMock())
thread_pool_executor.wait_for_all()
self.assertEqual(VulnerabilityDocument.search().count(), 1)
result = VulnerabilityDocument.search().filter(
'term', asset__ip_address=self.asset.ip_address).execute()
self.assertEqual(result.hits[0].tags,
['test', VulnerabilityStatus.REOPEN])
def test_update_discovered_asset(self):
asset = AssetDocument.get_or_create('10.0.0.1')
self.assertEqual(asset.tags, [AssetStatus.DISCOVERED])
self.assertEqual(1, Search().index(AssetDocument.Index.name).count())
asset = AssetDocument(ip_address='10.0.0.1',
os='Windows',
id=1,
confidentiality_requirement='NOT_DEFINED',
integrity_requirement='NOT_DEFINED',
availability_requirement='NOT_DEFINED',
hostname='hostname_1')
AssetDocument.create_or_update({asset.id: asset}, AssetConfigMock())
thread_pool_executor.wait_for_all()
self.assertEqual(1, Search().index(AssetDocument.Index.name).count())
result = AssetDocument.search().filter(
'term', ip_address='10.0.0.1').execute()
uut = result.hits[0]
self.assertEqual(uut.os, 'Windows')
self.assertEqual(uut.ip_address, '10.0.0.1')
self.assertEqual(uut.hostname, 'hostname_1')
self.assertEqual(uut.tags, [])
def _update_scans(config_pk: int):
LOGGER.debug(F'Starting update scans: {config_pk}')
config = Config.objects.filter(pk=config_pk)
if config.exists():
config = config.first()
else:
LOGGER.error(F'Config: {config_pk} not exist!')
return None
try:
config.set_status(Config.Status.IN_PROGRESS)
manager = scanners_registry.get(config)
client = manager.get_client()
parser = manager.get_parser()
now_date = now()
LOGGER.info(F'Trying to download scan lists')
scan_list = client.get_scans()
scan_list = parser.get_scans_ids(scan_list)
LOGGER.info(F'scan list downloaded')
LOGGER.debug(F'Scan list: {scan_list}')
for scan_id in scan_list:
LOGGER.info(F'Trying to download report form {config.name}')
file = client.download_scan(scan_id, client.ReportFormat.XML)
path = _get_save_path(config)
file_name = '{}-{}.zip'.format(config.scanner, now().strftime('%H-%M-%S'))
full_file_path = Path(path) / file_name
LOGGER.info(F"Saving file: {full_file_path}")
thread_pool_executor.submit(save_scan, client, scan_id, file, full_file_path)
saved_scan = Scan.objects.create(config=config, file=str(full_file_path))
file_url = F"{getattr(settings, 'ABSOLUTE_URI', '')}{reverse('download_scan', args=[saved_scan.file_id])}"
targets = copy.deepcopy(file)
LOGGER.info(F'Retrieving discovered assets for {config.name}')
discovered_assets = AssetDocument.get_assets_with_tag(tag=AssetStatus.DISCOVERED, config=config)
LOGGER.info(F'Trying to parse scan file {scan_id}')
vulns, scanned_hosts = parser.parse(file, file_url)
LOGGER.info(F'File parsed: {scan_id}')
LOGGER.info(F'Trying to parse targets from file {scan_id}')
targets = parser.get_targets(targets)
LOGGER.info(F'Targets parsed: {scan_id}')
if targets:
LOGGER.info(F'Attempting to update discovered assets in {config.name}')
AssetDocument.update_gone_discovered_assets(targets=targets, scanned_hosts=scanned_hosts,
discovered_assets=discovered_assets, config=config)
LOGGER.info(F'Attempting to update vulns data in {config.name}')
VulnerabilityDocument.create_or_update(vulns, scanned_hosts, config)
config.last_scans_pull = now_date
config.set_status(Config.Status.SUCCESS)
config.save(update_fields=['last_scans_pull'])
except Exception as e:
config.set_status(status=Config.Status.ERROR, error_description=e)
LOGGER.error(F'Error while loading vulnerability data {e}')
finally:
thread_pool_executor.wait_for_all()
def setUp(self):
super().setUp()
with open(get_fixture_location(__file__,
'nvdcve-1.0-2017.json')) as handle:
CveFactory.process(handle)
thread_pool_executor.wait_for_all()
def _update_assets(config_id: int):
config = Config.objects.filter(pk=config_id)
if config.exists():
config = config.first()
try:
config.set_status(Config.Status.IN_PROGRESS)
client = RalphClient(config)
parser = AssetsParser(config)
LOGGER.info(F'Start loading data from Ralph: {config.name}')
users = client.get_users()
users = OwnerParser.parse(users)
assets = client.get_data_center_assets()
assets = parser.parse(assets, users)
AssetDocument.create_or_update(assets, config)
LOGGER.info(
F'Finish loading data center assets from Ralph: {config.name}')
assets = client.get_virtual_assets()
assets = parser.parse(assets, users)
AssetDocument.create_or_update(assets, config)
LOGGER.info(
F'Finish loading virtual assets from Ralph: {config.name}')
LOGGER.info(F'Finish loading data from Ralph: {config.name}')
config.set_status(Config.Status.SUCCESS)
except Exception as ex:
LOGGER.error(F'Error with loading data from Ralph: {ex}')
config.set_status(status=Config.Status.ERROR, error_description=ex)
finally:
thread_pool_executor.wait_for_all()
def test_should_not_update(self):
self.assertEqual(Search().index(CveDocument.Index.name).count(), 2)
with open(get_fixture_location(__file__,
'nvdcve-1.0-2017.json')) as handle:
CveFactory.process(handle)
thread_pool_executor.wait_for_all()
self.assertEqual(Search().index(CveDocument.Index.name).count(), 2)
def test_cwe_update(self):
cwe = CweDocument.search().filter('term',
id='CWE-200').execute().hits[0]
cwe.name = 'Changed'
cwe.save()
thread_pool_executor.wait_for_all()
self.assertEqual(
CveDocument.search().filter('term', id='CVE-2017-0008').count(), 1)
cve = CveDocument.search().filter('term',
id='CVE-2017-0008').execute().hits
self.assertEqual(len(cve), 1)
self.assertEqual(cve[0].cwe.name, 'Changed')
def test_not_updated_existing_vulnerability(self):
vuln = create_vulnerability(self.asset, self.cve)
self.assertEqual(VulnerabilityDocument.search().count(), 1)
updated_vuln = vuln.clone()
VulnerabilityDocument.create_or_update({updated_vuln.id: updated_vuln}, [], ConfigMock())
thread_pool_executor.wait_for_all()
self.assertEqual(VulnerabilityDocument.search().count(), 1)
result_2 = VulnerabilityDocument.search().filter(
'term', asset__ip_address=self.asset.ip_address).sort('-modified_date').filter(
'term', cve__id=self.cve.id).execute()
self.assertEqual(result_2.hits[0].description, 'description')
def update_exploits():
try:
LOGGER.info(F'Trying to get {VIA4_URL}')
file = get_file(VIA4_URL)
if file:
LOGGER.info('File downloaded, updating database.')
ExploitFactory.process(file)
LOGGER.info('Database updated')
else:
LOGGER.error(F'Unable do download file {VIA4_URL}')
except Exception as ex:
LOGGER.error(ex)
finally:
thread_pool_executor.wait_for_all()
def update_cve(year: int):
try:
LOGGER.info(F'Trying to get file for {year} year')
file = get_file(CVE_NVD_URL.format(year))
if file:
LOGGER.info(F'File downloaded for {year} year, parsing...')
CveFactory.process(file)
file.close()
LOGGER.info(F'Parsing for {year}, done.')
else:
LOGGER.info(F'Unable to download file for {year} year')
except Exception as ex:
LOGGER.error(ex)
finally:
thread_pool_executor.wait_for_all()
def update_cwe():
try:
LOGGER.info('Updating cws, download file')
file = get_file(CWE_MITRE_URL)
if file:
LOGGER.info('File downloaded for cwe year, parsing...')
CWEFactory.process(file)
file.close()
LOGGER.info('CWE file parsing done.')
else:
LOGGER.info('Unable to download CWE file')
except Exception as ex:
LOGGER.error(ex)
finally:
thread_pool_executor.wait_for_all()
def _processing(idx, slices_count, assets_count, vulnerability_index):
docs = []
try:
vuln_search = VulnerabilityDocument.search(
index=vulnerability_index).filter(
~Q('match', tags=VulnerabilityStatus.FIXED)
& ~Q('match', asset__tags=AssetStatus.DELETED))
LOGGER.debug(
F'Calculation for {vulnerability_index} and {idx}, {slices_count} started'
)
if slices_count > 1:
vuln_search = vuln_search.extra(slice={
"id": idx,
"max": slices_count
}).params(scroll="60m")
# List competence used due to better performance
vulns = [vuln for vuln in vuln_search.scan()]
LOGGER.debug(F'all vulns for slice {idx} downloaded')
for vuln in vulns:
score, vector = calculate_environmental_score_v3(vuln)
vuln.environmental_score_vector_v3 = vector
vuln.environmental_score_v3 = score
vuln_count = get_cve_count(vulnerability_index, vuln.cve.id)
score, vector = calculate_environmental_score_v2(
vuln, vuln_count, assets_count)
vuln.environmental_score_vector_v2 = vector
vuln.environmental_score_v2 = score
docs.append(vuln.to_dict(include_meta=True))
if len(docs) > 10000:
async_bulk(docs, vulnerability_index)
docs = []
async_bulk(docs, vulnerability_index)
except Exception as ex:
LOGGER.error(F'Unknown processing exception {ex}')
finally:
thread_pool_executor.wait_for_all()
LOGGER.debug(
F'Calculation for {vulnerability_index} and {idx}, {slices_count} done'
)
def test_update(self):
cve = CveDocument.search().filter('term',
id='CVE-2017-0002').execute().hits[0]
cve.last_modified_date = None
cve.save(refresh=True)
with open(get_fixture_location(__file__,
'nvdcve-1.0-2017.json')) as handle:
CveFactory.process(handle)
thread_pool_executor.wait_for_all()
self.assertEqual(
CveDocument.search().filter('term', id='CVE-2017-0002').count(), 1)
new_cve = CveDocument.search().filter(
'term', id='CVE-2017-0002').execute().hits[0]
self.assertTrue(cve.last_modified_date != new_cve.last_modified_date)
def test_delete_asset(self):
asset_1 = self.create_asset(asset_id=1,
ip_address='10.0.0.1',
hostname='hostname_1')
asset_2 = self.create_asset(asset_id=2,
ip_address='10.0.0.2',
hostname='hostname_2')
self.assertEqual(2, Search().index(AssetDocument.Index.name).count())
AssetDocument.create_or_update({asset_1.id: asset_1},
AssetConfigMock())
thread_pool_executor.wait_for_all()
result = AssetDocument.search().filter(
Q('match', tags=AssetStatus.DELETED)).execute()
self.assertEqual(1, len(result.hits))
self.assertEqual(result.hits[0].ip_address, asset_2.ip_address)
self.assertEqual(result.hits[0].id, asset_2.id)
def test_update_asset(self):
asset_tenant_1 = self.create_asset(self.config_tenant_1.name)
asset_tenant_2 = self.create_asset(self.config_tenant_2.name)
AssetDocument.create_or_update({asset_tenant_1.id: asset_tenant_1},
self.config_tenant_1)
AssetDocument.create_or_update({asset_tenant_2.id: asset_tenant_2},
self.config_tenant_2)
thread_pool_executor.wait_for_all()
asset_tenant_1 = self.create_asset(self.config_tenant_1.name)
asset_tenant_1.hostname = 'tenant-test'
AssetDocument.create_or_update({asset_tenant_1.id: asset_tenant_1},
self.config_tenant_1)
thread_pool_executor.wait_for_all()
result = AssetDocument.search(index='test.tenant.asset').filter(
'term', ip_address='10.10.10.1').execute()
self.assertEqual(1, len(result.hits))
self.assertEqual(result.hits[0].ip_address, '10.10.10.1')
self.assertEqual(result.hits[0].hostname, 'tenant-test')
def _update_scans(config_pk: int):
config = Config.objects.filter(pk=config_pk)
if config.exists():
config = config.first()
try:
config.set_status(Config.Status.IN_PROGRESS)
client, parser = scanners_registry.get(config)
now_date = now()
scan_list = client.get_scans(last_modification_date=config.last_scans_pull)
scan_list = parser.get_scans_ids(scan_list)
for scan_id in scan_list:
LOGGER.info(F'Trying to download report form {config.name}')
file = client.download_scan(scan_id)
targets = copy.deepcopy(file)
LOGGER.info(F'Retrieving discovered assets for {config.name}')
discovered_assets = AssetDocument.get_assets_with_tag(tag=AssetStatus.DISCOVERED, config=config)
LOGGER.info(F'Trying to parse scan file {scan_id}')
vulns, scanned_hosts = parser.parse(file)
LOGGER.info(F'File parsed: {scan_id}')
LOGGER.info(F'Trying to parse targets from file {scan_id}')
if hasattr(parser, "get_targets"):
targets = parser.get_targets(targets)
else:
targets = client.get_targets(targets)
LOGGER.info(F'Targets parsed: {scan_id}')
if targets:
LOGGER.info(F'Attempting to update discovered assets in {config.name}')
AssetDocument.update_gone_discovered_assets(targets=targets, scanned_hosts=scanned_hosts,
discovered_assets=discovered_assets, config=config)
LOGGER.info(F'Attempting to update vulns data in {config.name}')
VulnerabilityDocument.create_or_update(vulns, scanned_hosts, config)
config.last_scans_pull = now_date
config.set_status(Config.Status.SUCCESS)
config.save(update_fields=['last_scans_pull'])
except Exception as e:
config.set_status(status=Config.Status.ERROR, error_description=e)
LOGGER.error(F'Error while loading vulnerability data {e}')
finally:
thread_pool_executor.wait_for_all()
def test_call_call_not_update(self):
with open(get_fixture_location(__file__,
'nvdcve-1.0-2017-2.json')) as handle:
CveFactory.process(handle)
thread_pool_executor.wait_for_all()
result = CveDocument.search().filter('term',
id='CVE-2017-0002').execute()
self.assertEqual(len(result.hits), 1)
cve = result.hits[0]
self.assertEqual(cve.id, 'CVE-2017-0002')
self.assertEqual(cve.access_vector_v2, metrics.AccessVectorV2.NETWORK)
self.assertEqual(cve.access_complexity_v2,
metrics.AccessComplexityV2.MEDIUM)
self.assertEqual(cve.authentication_v2, metrics.AuthenticationV2.NONE)
self.assertEqual(cve.confidentiality_impact_v2,
metrics.ImpactV2.PARTIAL)
self.assertEqual(cve.integrity_impact_v2, metrics.ImpactV2.PARTIAL)
self.assertEqual(cve.availability_impact_v2, metrics.ImpactV2.PARTIAL)
def test_should_not_update(self):
self.assertEqual(Search().index(CweDocument.Index.name).count(), 2)
with open(get_fixture_location(__file__, 'cwec_v2.12.xml')) as handle:
CWEFactory.process(handle)
thread_pool_executor.wait_for_all()
self.assertEqual(Search().index(CweDocument.Index.name).count(), 2)
def setUp(self):
super().setUp()
self.load_data()
thread_pool_executor.wait_for_all()
def setUp(self):
super().setUp()
with open(get_fixture_location(__file__, 'cwec_v2.12.xml')) as handle:
CWEFactory.process(handle)
thread_pool_executor.wait_for_all()
