def test_call_update_exploits(self, get_file): self.assertEqual(Search().index(CveDocument.Index.name).count(), 2) get_file.return_value = self.data update_exploits() get_file.assert_called_once_with( 'https://www.cve-search.org/feeds/via4.json') thread_pool_executor.wait_for_all() self.assertEqual(Search().index(CveDocument.Index.name).count(), 2) cve = CveDocument.search().filter( 'term', id='CVE-2017-0008').sort('-modified_date').execute().hits[0] prev_modified_date = cve.modified_date self.assertEqual(len(cve.exploits), 1) self.assertEqual(cve.exploits, [{ 'id': '44904', 'url': 'https://www.exploit-db.com/exploits/44904' }]) update_exploits() thread_pool_executor.wait_for_all() self.assertEqual(Search().index(CveDocument.Index.name).count(), 2) cve = CveDocument.search().filter( 'term', id='CVE-2017-0008').sort('-modified_date').execute().hits[0] self.assertEqual(cve.modified_date, prev_modified_date) self.assertEqual(len(cve.exploits), 1) self.assertEqual(cve.exploits, [{ 'id': '44904', 'url': 'https://www.exploit-db.com/exploits/44904' }])
def test_cwe_update(self): cwe = CweDocument.search().filter('term', id='CWE-200').execute().hits[0] cwe.name = 'Changed' cwe.save() thread_pool_executor.wait_for_all() self.assertEqual( CveDocument.search().filter('term', id='CVE-2017-0008').count(), 1) cve = CveDocument.search().filter('term', id='CVE-2017-0008').execute().hits self.assertEqual(len(cve), 1) self.assertEqual(cve[0].cwe.name, 'Changed')
def _create_nessus_cve(self, item): cve = CveDocument() cve.id = 'NESSUS-{}'.format(item.get('pluginID')) base_score_v2 = get_value(item.find('cvss_base_score')) if base_score_v2: cve.base_score_v2 = float(base_score_v2) base_score_v3 = get_value(item.find('cvss3_base_score')) if base_score_v3: cve.base_score_v3 = float(base_score_v3) cve = self._create_nessus_cve_cvss3_vector(item, cve) cve = self._create_nessus_cve_cvss_vector(item, cve) return cve
def test_update(self): cve = CveDocument.search().filter('term', id='CVE-2017-0002').execute().hits[0] cve.last_modified_date = None cve.save(refresh=True) with open(get_fixture_location(__file__, 'nvdcve-1.0-2017.json')) as handle: CveFactory.process(handle) thread_pool_executor.wait_for_all() self.assertEqual( CveDocument.search().filter('term', id='CVE-2017-0002').count(), 1) new_cve = CveDocument.search().filter( 'term', id='CVE-2017-0002').execute().hits[0] self.assertTrue(cve.last_modified_date != new_cve.last_modified_date)
def test_exploitability_v3(self, av, ac, scope, pr, ui, expected): cve = CveDocument(attack_vector_v3=av, attack_complexity_v3=ac, scope_v3=scope, privileges_required_v3=pr, user_interaction_v3=ui) self.assertEqual(round(tasks.exploitability_v3(cve), 2), expected)
def parse(self, report) -> [Dict, Dict]: for host in iter_elements_by_name(report, "ReportHost"): self.__scanned_hosts.append(host.get('name')) for item in host.iter('ReportItem'): if item.get('severity') != NessusReportParser.INFO: vuln = dict() vuln['asset'] = AssetFactory.create(host, self.__config) vuln['plugin_id'] = item.get('pluginID') vuln['port'] = item.get('port') if vuln['port'] != '0': vuln['svc_name'] = item.get('svc_name') vuln['protocol'] = item.get('protocol') else: vuln['port'] = None vuln['svc_name'] = None vuln['protocol'] = None vuln['description'] = get_value(item.find('description')) vuln['solution'] = get_value(item.find('solution')) vuln['exploit_available'] = True if get_value(item.find('exploit_available')) == 'true' else False vuln['id'] = self._vuln_id(vuln['asset'].ip_address, vuln['protocol'], vuln['plugin_id']) cves = item.findall('cve') if cves: for cve in cves: vuln['cve_id'] = get_value(cve) vuln['cve'] = CveDocument.get_or_create(cve_id=vuln['cve_id']) self._create(vuln) else: vuln['cve'] = self._create_nessus_cve(item) self._create(vuln) return self.__parsed, self.__scanned_hosts
def create_cve(cve_id='CVE-2017-0002', save=True) -> CveDocument: cve = CveDocument(id=cve_id, base_score_v2=6.8, access_vector_v2=metrics.AccessVectorV2.NETWORK, access_complexity_v2=metrics.AccessComplexityV2.MEDIUM, authentication_v2=metrics.AuthenticationV2.NONE, confidentiality_impact_v2=metrics.ImpactV2.PARTIAL, integrity_impact_v2=metrics.ImpactV2.PARTIAL, availability_impact_v2=metrics.ImpactV2.PARTIAL, base_score_v3=8.8, attack_vector_v3=metrics.AttackVectorV3.NETWORK, attack_complexity_v3=metrics.AttackComplexityV3.LOW, privileges_required_v3=metrics.PrivilegesRequiredV3.NONE, user_interaction_v3=metrics.UserInteractionV3.REQUIRED, scope_v3=metrics.ScopeV3.UNCHANGED, confidentiality_impact_v3=metrics.ImpactV3.HIGH, integrity_impact_v3=metrics.ImpactV3.HIGH, availability_impact_v3=metrics.ImpactV3.HIGH) if save: cve.save() return cve
def create(item: dict): if '** REJECT **' not in CveFactory.summary(item): old = CveDocument.search().filter( 'term', id=CveFactory.get_id(item)).sort( '-last_modified_date')[0].execute() if old.hits: last_modified_date = old.hits[0].last_modified_date else: last_modified_date = None if not last_modified_date or last_modified_date < CveFactory.last_modified_date( item): cve = CveDocument(id=CveFactory.get_id(item)) for field in CveDocument.get_fields_name(): parser = getattr(CveFactory, field, None) if parser: try: setattr(cve, field, parser(item)) except Exception as err: logging.debug( F'cve id {cve.id}, field {field}, err {err}') for cpe in CveFactory.get_cpe(item): cve.cpe.append(cpe) if old.hits and cve.has_changed(old.hits[0]): return old.hits[0].update(cve, weak=True) else: return cve.save(weak=True) return None logging.info(F'cve id {CveFactory.get_id(item)} is rejected') return None
def get_cve_sets(asset_count): cve_sets = {name: set() for name in CPE_LIST.keys()} cve_cvss_3 = CveDocument.search().filter('exists', field='base_score_v3') cve_cvss_2 = CveDocument.search() cve_count = 0 while cve_count < asset_count * 4: cve_count = 0 for os in CPE_LIST: max_cve = cve_cvss_3.filter( 'wildcard', cpe__name=F'*{CPE_LIST[os]}*').query(RAND_QUERY).count() count = random.randint(1, max_cve) if max_cve > 0: cves = cve_cvss_3.filter( 'wildcard', cpe__name=F'*{CPE_LIST[os]}*').query(RAND_QUERY)[0:count] else: cves = cve_cvss_2.filter( 'wildcard', cpe__name=F'*{CPE_LIST[os]}*').query(RAND_QUERY)[0:count] cve_sets[os].update(cve for cve in cves) for cve in cve_sets: cve_count += len(cve_sets[cve]) print(F'The amount of cve drawn: {cve_count}, ' F'assets count: {asset_count}, ' F'ratio: {round(cve_count / asset_count, 2)}') for os_name in cve_sets: print( F'System name: {os_name}, cve pool count: {len(cve_sets[os_name])}' ) return {k: list(v) for k, v in cve_sets.items()}
def test_call_create(self): cve = CveDocument.search().filter('term', id='CVE-2017-0008').execute().hits[0] self.assertEqual(cve.base_score_v2, 4.3) self.assertEqual(cve.base_score_v3, 4.3) self.assertEqual( cve.summary, 'Microsoft Internet Explorer 9 through 11 allow remote attackers to obtain sensitive ' 'information from process memory via a crafted web site, aka "Internet Explorer Information ' 'Disclosure Vulnerability." This vulnerability is different from those described in ' 'CVE-2017-0009 and CVE-2017-0059.') self.assertEquals(str(cve.published_date), '2017-03-17 00:59:00+00:00') self.assertEquals(str(cve.last_modified_date), '2017-07-12 01:29:00+00:00') self.assertEqual(cve.cwe.id, 'CWE-200') self.assertEqual(len(cve.cpe), 3) self.assertEqual(cve.cpe, [{ 'name': 'cpe:2.3:a:microsoft:internet_explorer:9:*:*:*:*:*:*:*', 'vendor': 'microsoft' }, { 'name': 'cpe:2.3:a:microsoft:internet_explorer:10:*:*:*:*:*:*:*', 'vendor': 'microsoft' }, { 'name': 'cpe:2.3:a:microsoft:internet_explorer:11:*:*:*:*:*:*:*', 'vendor': 'microsoft' }]) self.assertEqual(cve.access_vector_v2, metrics.AccessVectorV2.NETWORK) self.assertEqual(cve.access_complexity_v2, metrics.AccessComplexityV2.MEDIUM) self.assertEqual(cve.authentication_v2, metrics.AuthenticationV2.NONE) self.assertEqual(cve.confidentiality_impact_v2, metrics.ImpactV2.PARTIAL) self.assertEqual(cve.integrity_impact_v2, metrics.ImpactV2.NONE) self.assertEqual(cve.availability_impact_v2, metrics.ImpactV2.NONE) self.assertEqual(cve.attack_vector_v3, metrics.AttackVectorV3.NETWORK) self.assertEqual(cve.attack_complexity_v3, metrics.AttackComplexityV3.LOW) self.assertEqual(cve.privileges_required_v3, metrics.PrivilegesRequiredV3.NONE) self.assertEqual(cve.user_interaction_v3, metrics.UserInteractionV3.REQUIRED) self.assertEqual(cve.scope_v3, metrics.ScopeV3.UNCHANGED) self.assertEqual(cve.confidentiality_impact_v3, metrics.ImpactV3.LOW) self.assertEqual(cve.integrity_impact_v3, metrics.ImpactV3.NONE) self.assertEqual(cve.availability_impact_v3, metrics.ImpactV3.NONE) self.assertEqual(cve.get_privileges_required_v3_value(), 0.85)
def parse(self, report, file_url) -> [Dict, Dict]: for host in iter_elements_by_name(report, "ReportHost"): scan_date = host.find( 'HostProperties/tag[@name="HOST_START_TIMESTAMP"]').text scan_date = datetime.fromtimestamp(int(scan_date)) asset = AssetFactory.create(host, self.__config) asset.last_scan_date = scan_date self.__scanned_hosts.append(asset) for item in host.iter('ReportItem'): if item.get('severity') != NessusReportParser.INFO: vuln = dict() vuln['scan_date'] = scan_date vuln['scan_file_url'] = file_url vuln['asset'] = asset vuln['plugin_id'] = item.get('pluginID') vuln['name'] = item.get('pluginName') vuln['port'] = item.get('port') if vuln['port'] != '0': vuln['svc_name'] = item.get('svc_name') vuln['protocol'] = item.get('protocol') else: vuln['port'] = None vuln['svc_name'] = None vuln['protocol'] = None vuln['description'] = get_value(item.find('description')) vuln['solution'] = get_value(item.find('solution')) vuln['exploit_available'] = True if get_value( item.find('exploit_available')) == 'true' else False vuln[ 'tenant'] = self.__config.tenant.name if self.__config.tenant else None cves = item.findall('cve') if cves: for cve in cves: vuln['cve_id'] = get_value(cve) vuln['cve'] = CveDocument.get_or_create( cve_id=vuln['cve_id']) vuln['id'] = self._vuln_id(vuln, vuln['cve_id']) self._create(vuln) else: vuln['cve'] = self._create_nessus_cve(item) vuln['id'] = self._vuln_id(vuln, vuln['cve'].id) self._create(vuln) return self.__parsed, self.__scanned_hosts
def create(key: str, value: dict) -> [CveDocument, None]: try: exploits = [] for exp_id in value['refmap']['exploit-db']: exploits.append(ExploitInnerDoc.create(exp_id=exp_id)) except KeyError: pass else: result = CveDocument.search().filter('term', id=key)[0].execute() if result.hits and result.hits[0].exploits != exploits: result.hits[0].exploits = exploits return result.hits[0] return None
def test_call_call_not_update(self): with open(get_fixture_location(__file__, 'nvdcve-1.0-2017-2.json')) as handle: CveFactory.process(handle) thread_pool_executor.wait_for_all() result = CveDocument.search().filter('term', id='CVE-2017-0002').execute() self.assertEqual(len(result.hits), 1) cve = result.hits[0] self.assertEqual(cve.id, 'CVE-2017-0002') self.assertEqual(cve.access_vector_v2, metrics.AccessVectorV2.NETWORK) self.assertEqual(cve.access_complexity_v2, metrics.AccessComplexityV2.MEDIUM) self.assertEqual(cve.authentication_v2, metrics.AuthenticationV2.NONE) self.assertEqual(cve.confidentiality_impact_v2, metrics.ImpactV2.PARTIAL) self.assertEqual(cve.integrity_impact_v2, metrics.ImpactV2.PARTIAL) self.assertEqual(cve.availability_impact_v2, metrics.ImpactV2.PARTIAL)
def get_cve(cve_id, oid, tags): if cve_id == 'NOCVE': cve_id = F'NOCVE-{oid}' cve = CveDocument.get_or_create(cve_id=cve_id) vector = tags['cvss_base_vector'] vector = dict(x.split(':') for x in vector.split('/')) new_cve = CveDocument(id=cve_id) new_cve.access_vector_v2 = metrics.AccessVectorV2(vector['AV']) new_cve.access_complexity_v2 = metrics.AccessComplexityV2(vector['AC']) new_cve.authentication_v2 = metrics.AuthenticationV2(vector['Au']) new_cve.confidentiality_impact_v2 = metrics.ImpactV2(vector['C']) new_cve.integrity_impact_v2 = metrics.ImpactV2(vector['I']) new_cve.availability_impact_v2 = metrics.ImpactV2(vector['A']) new_cve.base_score_v2 = calculate_base_score_v2(new_cve) if cve.has_changed(new_cve): return cve.update(new_cve, refresh=True) return cve return CveDocument.get_or_create(cve_id=cve_id)
def exploitability_v3(cve: CveDocument) -> float: return 8.22 * \ cve.attack_vector_v3.second_value * \ cve.attack_complexity_v3.second_value * \ cve.get_privileges_required_v3_value() * \ cve.user_interaction_v3.second_value
def test_calculate_base_score(self): for cve in CveDocument.search(): self.assertEqual(cve.base_score_v2, calculate_base_score_v2(cve), cve.id) self.assertEqual(cve.base_score_v3, calculate_base_score_v3(cve), cve.id)
def get_value(value): return value if value else '' def summary(d_cwe, d_cve): text = ' '.join([ get_value(d_cwe.name), get_value(d_cwe.description), get_value(d_cwe.extended_description), get_value(d_cve) ]) return text.translate(str.maketrans('', '', '\n\t\r')).replace('|', ' ') if __name__ == 'main': all_count = CveDocument.search().count() print('All CVE count', all_count) search = CveDocument.search().filter('exists', field='base_score_v3') count = search.count() print('CVE with 3.1 score:', count) print('CVE CVSSv3/CVSSv2', count / all_count * 100) print('Downloading data') cves = [c for c in search.scan()] print('Downloaded data') with open(OUTPUT_FILE, 'w', encoding='utf-8') as out: for cve in cves: k = '|'.join([ str(cve.base_score_v2),
class CveFactory: FIELDS = [i for i in CveDocument.get_fields_name()] @staticmethod def process(handle): data = json.load(handle) docs = [] for obj in data['CVE_Items']: cve = CveFactory.create(obj) if cve: docs.append(cve.to_dict(include_meta=True)) if len(docs) > 500: async_bulk(docs, CveDocument.Index.name) docs = [] async_bulk(docs, CveDocument.Index.name) @staticmethod def create(item: dict): if '** REJECT **' not in CveFactory.summary(item): old = CveDocument.search().filter( 'term', id=CveFactory.get_id(item)).sort( '-last_modified_date')[0].execute() if old.hits: last_modified_date = old.hits[0].last_modified_date else: last_modified_date = None if not last_modified_date or last_modified_date < CveFactory.last_modified_date( item): cve = CveDocument(id=CveFactory.get_id(item)) for field in CveDocument.get_fields_name(): parser = getattr(CveFactory, field, None) if parser: try: setattr(cve, field, parser(item)) except Exception as err: logging.debug( F'cve id {cve.id}, field {field}, err {err}') for cpe in CveFactory.get_cpe(item): cve.cpe.append(cpe) if old.hits and cve.has_changed(old.hits[0]): return old.hits[0].update(cve, weak=True) else: return cve.save(weak=True) return None logging.info(F'cve id {CveFactory.get_id(item)} is rejected') return None @staticmethod def get_id(item: dict) -> str: return item['cve']['CVE_data_meta']['ID'] @staticmethod def base_score_v2(item: dict) -> [float, None]: return CveFactory.base_score('cvssV2', item) @staticmethod def base_score_v3(item: dict) -> [float, None]: return CveFactory.base_score('cvssV3', item) @staticmethod def base_score(version: str, item: dict) -> [float, None]: score = CveFactory.value_from_base_metrics(version, 'baseScore', item) return float(score) if score else None @staticmethod def summary(item: dict) -> str: for desc in item['cve']['description']['description_data']: if desc['lang'] == 'en': return desc['value'] return str() @staticmethod def references(item: RestrictedElement) -> str: objs = [] for ref in item['cve']['references']['reference_data']: objs.append({'source': ref['refsource'], 'url': ref['url']}) return json.dumps(objs) @staticmethod def cwe(item: dict) -> CweDocument: for problemtype_data in item['cve']['problemtype']['problemtype_data']: for desc in problemtype_data['description']: if desc['lang'] == 'en': return CWEFactory.get(desc['value']) @staticmethod def get_cpe(item: dict) -> list: cpes = [] try: for conf in item['configurations']['nodes']: for cpe_match in conf['cpe_match']: cpes.append(CpeFactory.get(cpe_match['cpe23Uri'])) except (KeyError, IndexError): pass return cpes @staticmethod def published_date(item: dict) -> datetime: return parse_datetime(item['publishedDate']) @staticmethod def last_modified_date(item: dict) -> datetime: return parse_datetime(item['lastModifiedDate']) @staticmethod def access_vector_v2(item: dict) -> metrics.AccessVectorV2: av = CveFactory.value_from_base_metrics('cvssV2', 'accessVector', item) return metrics.AccessVectorV2(av).value @staticmethod def access_complexity_v2(item: dict) -> metrics.AccessComplexityV2: ac = CveFactory.value_from_base_metrics('cvssV2', 'accessComplexity', item) return metrics.AccessComplexityV2(ac).value @staticmethod def authentication_v2(item: dict) -> metrics.AuthenticationV2: auth = CveFactory.value_from_base_metrics('cvssV2', 'authentication', item) return metrics.AuthenticationV2(auth).value @staticmethod def confidentiality_impact_v2(item: dict) -> metrics.ImpactV2: imp = CveFactory.value_from_base_metrics('cvssV2', 'confidentialityImpact', item) return metrics.ImpactV2(imp).value @staticmethod def integrity_impact_v2(item: dict) -> metrics.ImpactV2: imp = CveFactory.value_from_base_metrics('cvssV2', 'integrityImpact', item) return metrics.ImpactV2(imp).value @staticmethod def availability_impact_v2(item: dict) -> metrics.ImpactV2: imp = CveFactory.value_from_base_metrics('cvssV2', 'availabilityImpact', item) return metrics.ImpactV2(imp).value @staticmethod def attack_vector_v3(item: dict) -> metrics.AttackVectorV3: av = CveFactory.value_from_base_metrics('cvssV3', 'attackVector', item) return metrics.AttackVectorV3(av).value @staticmethod def attack_complexity_v3(item: dict) -> metrics.AttackComplexityV3: ac = CveFactory.value_from_base_metrics('cvssV3', 'attackComplexity', item) return metrics.AttackComplexityV3(ac).value @staticmethod def privileges_required_v3(item: dict) -> metrics.PrivilegesRequiredV3: pr = CveFactory.value_from_base_metrics('cvssV3', 'privilegesRequired', item) return metrics.PrivilegesRequiredV3(pr).value @staticmethod def user_interaction_v3(item: dict) -> metrics.UserInteractionV3: us = CveFactory.value_from_base_metrics('cvssV3', 'userInteraction', item) return metrics.UserInteractionV3(us).value @staticmethod def scope_v3(item: dict) -> metrics.ScopeV3: sc = CveFactory.value_from_base_metrics('cvssV3', 'scope', item) return metrics.ScopeV3(sc).value @staticmethod def confidentiality_impact_v3(item: dict) -> metrics.ImpactV3: ci = CveFactory.value_from_base_metrics('cvssV3', 'confidentialityImpact', item) return metrics.ImpactV3(ci).value @staticmethod def integrity_impact_v3(item: dict) -> metrics.ImpactV3: ii = CveFactory.value_from_base_metrics('cvssV3', 'integrityImpact', item) return metrics.ImpactV3(ii).value @staticmethod def availability_impact_v3(item: dict) -> metrics.ImpactV3: ai = CveFactory.value_from_base_metrics('cvssV3', 'availabilityImpact', item) return metrics.ImpactV3(ai).value @staticmethod def value_from_base_metrics(version: str, value: str, item: dict) -> [str, None]: return item['impact']['baseMetricV2' if version == 'cvssV2' else 'baseMetricV3'][version][value]
def test_call_call_not_create_rejected(self): self.assertFalse(CveDocument.search().filter( 'term', id='CVE-2017-0605').execute())