def test_asset_updated(self): self.asset_2 = create_asset('10.10.10.11') create_vulnerability(self.asset, self.cve) create_vulnerability(self.asset_2, self.cve) self.cve_2 = create_cve('CVE-2017-0003') create_vulnerability(self.asset, self.cve_2) create_vulnerability(self.asset_2, self.cve_2) self.assertEqual(Search().index(VulnerabilityDocument.Index.name).count(), 4) self.asset.confidentiality_requirement = AssetImpact.HIGH self.asset.integrity_requirement = AssetImpact.HIGH self.asset.save() thread_pool_executor.wait_for_all() self.assertEqual(Search().index(VulnerabilityDocument.Index.name).count(), 4) result_1 = VulnerabilityDocument.search().filter( 'term', asset__ip_address=self.asset.ip_address).execute() self.assertEqual(len(result_1.hits), 2) self.assertEqual(result_1.hits[0].asset.confidentiality_requirement, self.asset.confidentiality_requirement) self.assertEqual(result_1.hits[0].asset.integrity_requirement, self.asset.integrity_requirement) self.assertEqual(result_1.hits[1].asset.confidentiality_requirement, self.asset.confidentiality_requirement) self.assertEqual(result_1.hits[1].asset.integrity_requirement, self.asset.integrity_requirement) result_2 = VulnerabilityDocument.search().filter( 'term', asset__ip_address=self.asset_2.ip_address).execute() self.assertEqual(len(result_2.hits), 2) self.assertEqual(result_2.hits[0].asset.confidentiality_requirement, self.asset_2.confidentiality_requirement) self.assertEqual(result_2.hits[0].asset.integrity_requirement, self.asset_2.integrity_requirement) self.assertEqual(result_2.hits[1].asset.confidentiality_requirement, self.asset_2.confidentiality_requirement) self.assertEqual(result_2.hits[1].asset.integrity_requirement, self.asset_2.integrity_requirement)
def test_cve_updated(self): self.asset_2 = create_asset('10.10.10.11') self.cve_2 = create_cve('CVE-2017-0003') create_vulnerability(self.asset, self.cve) create_vulnerability(self.asset, self.cve_2) create_vulnerability(self.asset_2, self.cve) create_vulnerability(self.asset_2, self.cve_2) self.assertEqual(Search().index(VulnerabilityDocument.Index.name).count(), 4) self.cve.access_vector_v2 = metrics.AccessVectorV2.LOCAL self.cve.save() thread_pool_executor.wait_for_all() self.assertEqual(Search().index(VulnerabilityDocument.Index.name).count(), 4) result_1 = VulnerabilityDocument.search().filter('term', cve__id=self.cve.id).execute() self.assertEqual(len(result_1.hits), 2) self.assertEqual(result_1.hits[0].cve.access_vector_v2, self.cve.access_vector_v2) self.assertEqual(result_1.hits[1].cve.access_vector_v2, self.cve.access_vector_v2) result_2 = VulnerabilityDocument.search().filter('term', cve__id=self.cve_2.id).execute() self.assertEqual(len(result_2.hits), 2) self.assertEqual(result_2.hits[0].cve.access_vector_v2, self.cve_2.access_vector_v2) self.assertEqual(result_2.hits[1].cve.access_vector_v2, self.cve_2.access_vector_v2)
def test_update_discovered_asset(self): asset_tenant_1 = self.create_asset(self.config_tenant_1.name) discovered_asset = AssetDocument.get_or_create( asset_tenant_1.ip_address) cve = create_cve() create_vulnerability(discovered_asset, cve) self.assertEqual(1, Search().index(AssetDocument.Index.name).count()) AssetDocument.create_or_update({asset_tenant_1.id: asset_tenant_1}) thread_pool_executor.wait_for_all() self.assertEqual(1, Search().index(AssetDocument.Index.name).count()) self.assertEqual( 1, Search().index(VulnerabilityDocument.Index.name).count()) result = VulnerabilityDocument.search().filter( 'term', cve__id='CVE-2017-0002').execute() self.assertEqual(result.hits[0].asset.id, asset_tenant_1.id) self.assertEqual(result.hits[0].asset.ip_address, asset_tenant_1.ip_address) self.assertEqual(result.hits[0].asset.confidentiality_requirement, asset_tenant_1.confidentiality_requirement) self.assertEqual(result.hits[0].asset.availability_requirement, asset_tenant_1.availability_requirement)
def test_call_update_exploits(self, get_file): self.assertEqual(Search().index(CveDocument.Index.name).count(), 2) get_file.return_value = self.data update_exploits() get_file.assert_called_once_with( 'https://www.cve-search.org/feeds/via4.json') thread_pool_executor.wait_for_all() self.assertEqual(Search().index(CveDocument.Index.name).count(), 2) cve = CveDocument.search().filter( 'term', id='CVE-2017-0008').sort('-modified_date').execute().hits[0] prev_modified_date = cve.modified_date self.assertEqual(len(cve.exploits), 1) self.assertEqual(cve.exploits, [{ 'id': '44904', 'url': 'https://www.exploit-db.com/exploits/44904' }]) update_exploits() thread_pool_executor.wait_for_all() self.assertEqual(Search().index(CveDocument.Index.name).count(), 2) cve = CveDocument.search().filter( 'term', id='CVE-2017-0008').sort('-modified_date').execute().hits[0] self.assertEqual(cve.modified_date, prev_modified_date) self.assertEqual(len(cve.exploits), 1) self.assertEqual(cve.exploits, [{ 'id': '44904', 'url': 'https://www.exploit-db.com/exploits/44904' }])
def test_reopen_vulnerability(self): vulnerability = create_vulnerability(self.asset, self.cve) self.assertEqual(VulnerabilityDocument.search().count(), 1) VulnerabilityDocument.create_or_update({}, [self.asset.ip_address], ConfigMock()) thread_pool_executor.wait_for_all() self.assertEqual(VulnerabilityDocument.search().count(), 1) result = VulnerabilityDocument.search().filter( 'term', asset__ip_address=self.asset.ip_address).execute() self.assertEqual(result.hits[0].tags, ['test', VulnerabilityStatus.FIXED]) VulnerabilityDocument.create_or_update( {vulnerability.id: vulnerability}, [self.asset.ip_address], ConfigMock()) thread_pool_executor.wait_for_all() self.assertEqual(VulnerabilityDocument.search().count(), 1) result = VulnerabilityDocument.search().filter( 'term', asset__ip_address=self.asset.ip_address).execute() self.assertEqual(result.hits[0].tags, ['test', VulnerabilityStatus.REOPEN])
def test_update_discovered_asset(self): asset = AssetDocument.get_or_create('10.0.0.1') self.assertEqual(asset.tags, [AssetStatus.DISCOVERED]) self.assertEqual(1, Search().index(AssetDocument.Index.name).count()) asset = AssetDocument(ip_address='10.0.0.1', os='Windows', id=1, confidentiality_requirement='NOT_DEFINED', integrity_requirement='NOT_DEFINED', availability_requirement='NOT_DEFINED', hostname='hostname_1') AssetDocument.create_or_update({asset.id: asset}, AssetConfigMock()) thread_pool_executor.wait_for_all() self.assertEqual(1, Search().index(AssetDocument.Index.name).count()) result = AssetDocument.search().filter( 'term', ip_address='10.0.0.1').execute() uut = result.hits[0] self.assertEqual(uut.os, 'Windows') self.assertEqual(uut.ip_address, '10.0.0.1') self.assertEqual(uut.hostname, 'hostname_1') self.assertEqual(uut.tags, [])
def _update_scans(config_pk: int): LOGGER.debug(F'Starting update scans: {config_pk}') config = Config.objects.filter(pk=config_pk) if config.exists(): config = config.first() else: LOGGER.error(F'Config: {config_pk} not exist!') return None try: config.set_status(Config.Status.IN_PROGRESS) manager = scanners_registry.get(config) client = manager.get_client() parser = manager.get_parser() now_date = now() LOGGER.info(F'Trying to download scan lists') scan_list = client.get_scans() scan_list = parser.get_scans_ids(scan_list) LOGGER.info(F'scan list downloaded') LOGGER.debug(F'Scan list: {scan_list}') for scan_id in scan_list: LOGGER.info(F'Trying to download report form {config.name}') file = client.download_scan(scan_id, client.ReportFormat.XML) path = _get_save_path(config) file_name = '{}-{}.zip'.format(config.scanner, now().strftime('%H-%M-%S')) full_file_path = Path(path) / file_name LOGGER.info(F"Saving file: {full_file_path}") thread_pool_executor.submit(save_scan, client, scan_id, file, full_file_path) saved_scan = Scan.objects.create(config=config, file=str(full_file_path)) file_url = F"{getattr(settings, 'ABSOLUTE_URI', '')}{reverse('download_scan', args=[saved_scan.file_id])}" targets = copy.deepcopy(file) LOGGER.info(F'Retrieving discovered assets for {config.name}') discovered_assets = AssetDocument.get_assets_with_tag(tag=AssetStatus.DISCOVERED, config=config) LOGGER.info(F'Trying to parse scan file {scan_id}') vulns, scanned_hosts = parser.parse(file, file_url) LOGGER.info(F'File parsed: {scan_id}') LOGGER.info(F'Trying to parse targets from file {scan_id}') targets = parser.get_targets(targets) LOGGER.info(F'Targets parsed: {scan_id}') if targets: LOGGER.info(F'Attempting to update discovered assets in {config.name}') AssetDocument.update_gone_discovered_assets(targets=targets, scanned_hosts=scanned_hosts, discovered_assets=discovered_assets, config=config) LOGGER.info(F'Attempting to update vulns data in {config.name}') VulnerabilityDocument.create_or_update(vulns, scanned_hosts, config) config.last_scans_pull = now_date config.set_status(Config.Status.SUCCESS) config.save(update_fields=['last_scans_pull']) except Exception as e: config.set_status(status=Config.Status.ERROR, error_description=e) LOGGER.error(F'Error while loading vulnerability data {e}') finally: thread_pool_executor.wait_for_all()
def setUp(self): super().setUp() with open(get_fixture_location(__file__, 'nvdcve-1.0-2017.json')) as handle: CveFactory.process(handle) thread_pool_executor.wait_for_all()
def _update_assets(config_id: int): config = Config.objects.filter(pk=config_id) if config.exists(): config = config.first() try: config.set_status(Config.Status.IN_PROGRESS) client = RalphClient(config) parser = AssetsParser(config) LOGGER.info(F'Start loading data from Ralph: {config.name}') users = client.get_users() users = OwnerParser.parse(users) assets = client.get_data_center_assets() assets = parser.parse(assets, users) AssetDocument.create_or_update(assets, config) LOGGER.info( F'Finish loading data center assets from Ralph: {config.name}') assets = client.get_virtual_assets() assets = parser.parse(assets, users) AssetDocument.create_or_update(assets, config) LOGGER.info( F'Finish loading virtual assets from Ralph: {config.name}') LOGGER.info(F'Finish loading data from Ralph: {config.name}') config.set_status(Config.Status.SUCCESS) except Exception as ex: LOGGER.error(F'Error with loading data from Ralph: {ex}') config.set_status(status=Config.Status.ERROR, error_description=ex) finally: thread_pool_executor.wait_for_all()
def test_should_not_update(self): self.assertEqual(Search().index(CveDocument.Index.name).count(), 2) with open(get_fixture_location(__file__, 'nvdcve-1.0-2017.json')) as handle: CveFactory.process(handle) thread_pool_executor.wait_for_all() self.assertEqual(Search().index(CveDocument.Index.name).count(), 2)
def test_cwe_update(self): cwe = CweDocument.search().filter('term', id='CWE-200').execute().hits[0] cwe.name = 'Changed' cwe.save() thread_pool_executor.wait_for_all() self.assertEqual( CveDocument.search().filter('term', id='CVE-2017-0008').count(), 1) cve = CveDocument.search().filter('term', id='CVE-2017-0008').execute().hits self.assertEqual(len(cve), 1) self.assertEqual(cve[0].cwe.name, 'Changed')
def test_not_updated_existing_vulnerability(self): vuln = create_vulnerability(self.asset, self.cve) self.assertEqual(VulnerabilityDocument.search().count(), 1) updated_vuln = vuln.clone() VulnerabilityDocument.create_or_update({updated_vuln.id: updated_vuln}, [], ConfigMock()) thread_pool_executor.wait_for_all() self.assertEqual(VulnerabilityDocument.search().count(), 1) result_2 = VulnerabilityDocument.search().filter( 'term', asset__ip_address=self.asset.ip_address).sort('-modified_date').filter( 'term', cve__id=self.cve.id).execute() self.assertEqual(result_2.hits[0].description, 'description')
def update_exploits(): try: LOGGER.info(F'Trying to get {VIA4_URL}') file = get_file(VIA4_URL) if file: LOGGER.info('File downloaded, updating database.') ExploitFactory.process(file) LOGGER.info('Database updated') else: LOGGER.error(F'Unable do download file {VIA4_URL}') except Exception as ex: LOGGER.error(ex) finally: thread_pool_executor.wait_for_all()
def update_cve(year: int): try: LOGGER.info(F'Trying to get file for {year} year') file = get_file(CVE_NVD_URL.format(year)) if file: LOGGER.info(F'File downloaded for {year} year, parsing...') CveFactory.process(file) file.close() LOGGER.info(F'Parsing for {year}, done.') else: LOGGER.info(F'Unable to download file for {year} year') except Exception as ex: LOGGER.error(ex) finally: thread_pool_executor.wait_for_all()
def update_cwe(): try: LOGGER.info('Updating cws, download file') file = get_file(CWE_MITRE_URL) if file: LOGGER.info('File downloaded for cwe year, parsing...') CWEFactory.process(file) file.close() LOGGER.info('CWE file parsing done.') else: LOGGER.info('Unable to download CWE file') except Exception as ex: LOGGER.error(ex) finally: thread_pool_executor.wait_for_all()
def _processing(idx, slices_count, assets_count, vulnerability_index): docs = [] try: vuln_search = VulnerabilityDocument.search( index=vulnerability_index).filter( ~Q('match', tags=VulnerabilityStatus.FIXED) & ~Q('match', asset__tags=AssetStatus.DELETED)) LOGGER.debug( F'Calculation for {vulnerability_index} and {idx}, {slices_count} started' ) if slices_count > 1: vuln_search = vuln_search.extra(slice={ "id": idx, "max": slices_count }).params(scroll="60m") # List competence used due to better performance vulns = [vuln for vuln in vuln_search.scan()] LOGGER.debug(F'all vulns for slice {idx} downloaded') for vuln in vulns: score, vector = calculate_environmental_score_v3(vuln) vuln.environmental_score_vector_v3 = vector vuln.environmental_score_v3 = score vuln_count = get_cve_count(vulnerability_index, vuln.cve.id) score, vector = calculate_environmental_score_v2( vuln, vuln_count, assets_count) vuln.environmental_score_vector_v2 = vector vuln.environmental_score_v2 = score docs.append(vuln.to_dict(include_meta=True)) if len(docs) > 10000: async_bulk(docs, vulnerability_index) docs = [] async_bulk(docs, vulnerability_index) except Exception as ex: LOGGER.error(F'Unknown processing exception {ex}') finally: thread_pool_executor.wait_for_all() LOGGER.debug( F'Calculation for {vulnerability_index} and {idx}, {slices_count} done' )
def test_update(self): cve = CveDocument.search().filter('term', id='CVE-2017-0002').execute().hits[0] cve.last_modified_date = None cve.save(refresh=True) with open(get_fixture_location(__file__, 'nvdcve-1.0-2017.json')) as handle: CveFactory.process(handle) thread_pool_executor.wait_for_all() self.assertEqual( CveDocument.search().filter('term', id='CVE-2017-0002').count(), 1) new_cve = CveDocument.search().filter( 'term', id='CVE-2017-0002').execute().hits[0] self.assertTrue(cve.last_modified_date != new_cve.last_modified_date)
def test_delete_asset(self): asset_1 = self.create_asset(asset_id=1, ip_address='10.0.0.1', hostname='hostname_1') asset_2 = self.create_asset(asset_id=2, ip_address='10.0.0.2', hostname='hostname_2') self.assertEqual(2, Search().index(AssetDocument.Index.name).count()) AssetDocument.create_or_update({asset_1.id: asset_1}, AssetConfigMock()) thread_pool_executor.wait_for_all() result = AssetDocument.search().filter( Q('match', tags=AssetStatus.DELETED)).execute() self.assertEqual(1, len(result.hits)) self.assertEqual(result.hits[0].ip_address, asset_2.ip_address) self.assertEqual(result.hits[0].id, asset_2.id)
def test_update_asset(self): asset_tenant_1 = self.create_asset(self.config_tenant_1.name) asset_tenant_2 = self.create_asset(self.config_tenant_2.name) AssetDocument.create_or_update({asset_tenant_1.id: asset_tenant_1}, self.config_tenant_1) AssetDocument.create_or_update({asset_tenant_2.id: asset_tenant_2}, self.config_tenant_2) thread_pool_executor.wait_for_all() asset_tenant_1 = self.create_asset(self.config_tenant_1.name) asset_tenant_1.hostname = 'tenant-test' AssetDocument.create_or_update({asset_tenant_1.id: asset_tenant_1}, self.config_tenant_1) thread_pool_executor.wait_for_all() result = AssetDocument.search(index='test.tenant.asset').filter( 'term', ip_address='10.10.10.1').execute() self.assertEqual(1, len(result.hits)) self.assertEqual(result.hits[0].ip_address, '10.10.10.1') self.assertEqual(result.hits[0].hostname, 'tenant-test')
def _update_scans(config_pk: int): config = Config.objects.filter(pk=config_pk) if config.exists(): config = config.first() try: config.set_status(Config.Status.IN_PROGRESS) client, parser = scanners_registry.get(config) now_date = now() scan_list = client.get_scans(last_modification_date=config.last_scans_pull) scan_list = parser.get_scans_ids(scan_list) for scan_id in scan_list: LOGGER.info(F'Trying to download report form {config.name}') file = client.download_scan(scan_id) targets = copy.deepcopy(file) LOGGER.info(F'Retrieving discovered assets for {config.name}') discovered_assets = AssetDocument.get_assets_with_tag(tag=AssetStatus.DISCOVERED, config=config) LOGGER.info(F'Trying to parse scan file {scan_id}') vulns, scanned_hosts = parser.parse(file) LOGGER.info(F'File parsed: {scan_id}') LOGGER.info(F'Trying to parse targets from file {scan_id}') if hasattr(parser, "get_targets"): targets = parser.get_targets(targets) else: targets = client.get_targets(targets) LOGGER.info(F'Targets parsed: {scan_id}') if targets: LOGGER.info(F'Attempting to update discovered assets in {config.name}') AssetDocument.update_gone_discovered_assets(targets=targets, scanned_hosts=scanned_hosts, discovered_assets=discovered_assets, config=config) LOGGER.info(F'Attempting to update vulns data in {config.name}') VulnerabilityDocument.create_or_update(vulns, scanned_hosts, config) config.last_scans_pull = now_date config.set_status(Config.Status.SUCCESS) config.save(update_fields=['last_scans_pull']) except Exception as e: config.set_status(status=Config.Status.ERROR, error_description=e) LOGGER.error(F'Error while loading vulnerability data {e}') finally: thread_pool_executor.wait_for_all()
def test_call_call_not_update(self): with open(get_fixture_location(__file__, 'nvdcve-1.0-2017-2.json')) as handle: CveFactory.process(handle) thread_pool_executor.wait_for_all() result = CveDocument.search().filter('term', id='CVE-2017-0002').execute() self.assertEqual(len(result.hits), 1) cve = result.hits[0] self.assertEqual(cve.id, 'CVE-2017-0002') self.assertEqual(cve.access_vector_v2, metrics.AccessVectorV2.NETWORK) self.assertEqual(cve.access_complexity_v2, metrics.AccessComplexityV2.MEDIUM) self.assertEqual(cve.authentication_v2, metrics.AuthenticationV2.NONE) self.assertEqual(cve.confidentiality_impact_v2, metrics.ImpactV2.PARTIAL) self.assertEqual(cve.integrity_impact_v2, metrics.ImpactV2.PARTIAL) self.assertEqual(cve.availability_impact_v2, metrics.ImpactV2.PARTIAL)
def test_should_not_update(self): self.assertEqual(Search().index(CweDocument.Index.name).count(), 2) with open(get_fixture_location(__file__, 'cwec_v2.12.xml')) as handle: CWEFactory.process(handle) thread_pool_executor.wait_for_all() self.assertEqual(Search().index(CweDocument.Index.name).count(), 2)
def setUp(self): super().setUp() self.load_data() thread_pool_executor.wait_for_all()
def setUp(self): super().setUp() with open(get_fixture_location(__file__, 'cwec_v2.12.xml')) as handle: CWEFactory.process(handle) thread_pool_executor.wait_for_all()