def test_call_update_exploits(self, get_file):
self.assertEqual(Search().index(CveDocument.Index.name).count(), 2)
get_file.return_value = self.data
update_exploits()
get_file.assert_called_once_with(
'https://www.cve-search.org/feeds/via4.json')
thread_pool_executor.wait_for_all()
self.assertEqual(Search().index(CveDocument.Index.name).count(), 2)
cve = CveDocument.search().filter(
'term',
id='CVE-2017-0008').sort('-modified_date').execute().hits[0]
prev_modified_date = cve.modified_date
self.assertEqual(len(cve.exploits), 1)
self.assertEqual(cve.exploits,
[{
'id': '44904',
'url': 'https://www.exploit-db.com/exploits/44904'
}])
update_exploits()
thread_pool_executor.wait_for_all()
self.assertEqual(Search().index(CveDocument.Index.name).count(), 2)
cve = CveDocument.search().filter(
'term',
id='CVE-2017-0008').sort('-modified_date').execute().hits[0]
self.assertEqual(cve.modified_date, prev_modified_date)
self.assertEqual(len(cve.exploits), 1)
self.assertEqual(cve.exploits,
[{
'id': '44904',
'url': 'https://www.exploit-db.com/exploits/44904'
}])
def test_cwe_update(self):
cwe = CweDocument.search().filter('term',
id='CWE-200').execute().hits[0]
cwe.name = 'Changed'
cwe.save()
thread_pool_executor.wait_for_all()
self.assertEqual(
CveDocument.search().filter('term', id='CVE-2017-0008').count(), 1)
cve = CveDocument.search().filter('term',
id='CVE-2017-0008').execute().hits
self.assertEqual(len(cve), 1)
self.assertEqual(cve[0].cwe.name, 'Changed')
def _create_nessus_cve(self, item):
cve = CveDocument()
cve.id = 'NESSUS-{}'.format(item.get('pluginID'))
base_score_v2 = get_value(item.find('cvss_base_score'))
if base_score_v2:
cve.base_score_v2 = float(base_score_v2)
base_score_v3 = get_value(item.find('cvss3_base_score'))
if base_score_v3:
cve.base_score_v3 = float(base_score_v3)
cve = self._create_nessus_cve_cvss3_vector(item, cve)
cve = self._create_nessus_cve_cvss_vector(item, cve)
return cve
def test_update(self):
cve = CveDocument.search().filter('term',
id='CVE-2017-0002').execute().hits[0]
cve.last_modified_date = None
cve.save(refresh=True)
with open(get_fixture_location(__file__,
'nvdcve-1.0-2017.json')) as handle:
CveFactory.process(handle)
thread_pool_executor.wait_for_all()
self.assertEqual(
CveDocument.search().filter('term', id='CVE-2017-0002').count(), 1)
new_cve = CveDocument.search().filter(
'term', id='CVE-2017-0002').execute().hits[0]
self.assertTrue(cve.last_modified_date != new_cve.last_modified_date)
def test_exploitability_v3(self, av, ac, scope, pr, ui, expected):
cve = CveDocument(attack_vector_v3=av,
attack_complexity_v3=ac,
scope_v3=scope,
privileges_required_v3=pr,
user_interaction_v3=ui)
self.assertEqual(round(tasks.exploitability_v3(cve), 2), expected)
def parse(self, report) -> [Dict, Dict]:
for host in iter_elements_by_name(report, "ReportHost"):
self.__scanned_hosts.append(host.get('name'))
for item in host.iter('ReportItem'):
if item.get('severity') != NessusReportParser.INFO:
vuln = dict()
vuln['asset'] = AssetFactory.create(host, self.__config)
vuln['plugin_id'] = item.get('pluginID')
vuln['port'] = item.get('port')
if vuln['port'] != '0':
vuln['svc_name'] = item.get('svc_name')
vuln['protocol'] = item.get('protocol')
else:
vuln['port'] = None
vuln['svc_name'] = None
vuln['protocol'] = None
vuln['description'] = get_value(item.find('description'))
vuln['solution'] = get_value(item.find('solution'))
vuln['exploit_available'] = True if get_value(item.find('exploit_available')) == 'true' else False
vuln['id'] = self._vuln_id(vuln['asset'].ip_address, vuln['protocol'], vuln['plugin_id'])
cves = item.findall('cve')
if cves:
for cve in cves:
vuln['cve_id'] = get_value(cve)
vuln['cve'] = CveDocument.get_or_create(cve_id=vuln['cve_id'])
self._create(vuln)
else:
vuln['cve'] = self._create_nessus_cve(item)
self._create(vuln)
return self.__parsed, self.__scanned_hosts
def create_cve(cve_id='CVE-2017-0002', save=True) -> CveDocument:
cve = CveDocument(id=cve_id,
base_score_v2=6.8,
access_vector_v2=metrics.AccessVectorV2.NETWORK,
access_complexity_v2=metrics.AccessComplexityV2.MEDIUM,
authentication_v2=metrics.AuthenticationV2.NONE,
confidentiality_impact_v2=metrics.ImpactV2.PARTIAL,
integrity_impact_v2=metrics.ImpactV2.PARTIAL,
availability_impact_v2=metrics.ImpactV2.PARTIAL,
base_score_v3=8.8,
attack_vector_v3=metrics.AttackVectorV3.NETWORK,
attack_complexity_v3=metrics.AttackComplexityV3.LOW,
privileges_required_v3=metrics.PrivilegesRequiredV3.NONE,
user_interaction_v3=metrics.UserInteractionV3.REQUIRED,
scope_v3=metrics.ScopeV3.UNCHANGED,
confidentiality_impact_v3=metrics.ImpactV3.HIGH,
integrity_impact_v3=metrics.ImpactV3.HIGH,
availability_impact_v3=metrics.ImpactV3.HIGH)
if save:
cve.save()
return cve
def create(item: dict):
if '** REJECT **' not in CveFactory.summary(item):
old = CveDocument.search().filter(
'term', id=CveFactory.get_id(item)).sort(
'-last_modified_date')[0].execute()
if old.hits:
last_modified_date = old.hits[0].last_modified_date
else:
last_modified_date = None
if not last_modified_date or last_modified_date < CveFactory.last_modified_date(
item):
cve = CveDocument(id=CveFactory.get_id(item))
for field in CveDocument.get_fields_name():
parser = getattr(CveFactory, field, None)
if parser:
try:
setattr(cve, field, parser(item))
except Exception as err:
logging.debug(
F'cve id {cve.id}, field {field}, err {err}')
for cpe in CveFactory.get_cpe(item):
cve.cpe.append(cpe)
if old.hits and cve.has_changed(old.hits[0]):
return old.hits[0].update(cve, weak=True)
else:
return cve.save(weak=True)
return None
logging.info(F'cve id {CveFactory.get_id(item)} is rejected')
return None
def get_cve_sets(asset_count):
cve_sets = {name: set() for name in CPE_LIST.keys()}
cve_cvss_3 = CveDocument.search().filter('exists', field='base_score_v3')
cve_cvss_2 = CveDocument.search()
cve_count = 0
while cve_count < asset_count * 4:
cve_count = 0
for os in CPE_LIST:
max_cve = cve_cvss_3.filter(
'wildcard',
cpe__name=F'*{CPE_LIST[os]}*').query(RAND_QUERY).count()
count = random.randint(1, max_cve)
if max_cve > 0:
cves = cve_cvss_3.filter(
'wildcard',
cpe__name=F'*{CPE_LIST[os]}*').query(RAND_QUERY)[0:count]
else:
cves = cve_cvss_2.filter(
'wildcard',
cpe__name=F'*{CPE_LIST[os]}*').query(RAND_QUERY)[0:count]
cve_sets[os].update(cve for cve in cves)
for cve in cve_sets:
cve_count += len(cve_sets[cve])
print(F'The amount of cve drawn: {cve_count}, '
F'assets count: {asset_count}, '
F'ratio: {round(cve_count / asset_count, 2)}')
for os_name in cve_sets:
print(
F'System name: {os_name}, cve pool count: {len(cve_sets[os_name])}'
)
return {k: list(v) for k, v in cve_sets.items()}
def test_call_create(self):
cve = CveDocument.search().filter('term',
id='CVE-2017-0008').execute().hits[0]
self.assertEqual(cve.base_score_v2, 4.3)
self.assertEqual(cve.base_score_v3, 4.3)
self.assertEqual(
cve.summary,
'Microsoft Internet Explorer 9 through 11 allow remote attackers to obtain sensitive '
'information from process memory via a crafted web site, aka "Internet Explorer Information '
'Disclosure Vulnerability." This vulnerability is different from those described in '
'CVE-2017-0009 and CVE-2017-0059.')
self.assertEquals(str(cve.published_date), '2017-03-17 00:59:00+00:00')
self.assertEquals(str(cve.last_modified_date),
'2017-07-12 01:29:00+00:00')
self.assertEqual(cve.cwe.id, 'CWE-200')
self.assertEqual(len(cve.cpe), 3)
self.assertEqual(cve.cpe, [{
'name': 'cpe:2.3:a:microsoft:internet_explorer:9:*:*:*:*:*:*:*',
'vendor': 'microsoft'
}, {
'name': 'cpe:2.3:a:microsoft:internet_explorer:10:*:*:*:*:*:*:*',
'vendor': 'microsoft'
}, {
'name': 'cpe:2.3:a:microsoft:internet_explorer:11:*:*:*:*:*:*:*',
'vendor': 'microsoft'
}])
self.assertEqual(cve.access_vector_v2, metrics.AccessVectorV2.NETWORK)
self.assertEqual(cve.access_complexity_v2,
metrics.AccessComplexityV2.MEDIUM)
self.assertEqual(cve.authentication_v2, metrics.AuthenticationV2.NONE)
self.assertEqual(cve.confidentiality_impact_v2,
metrics.ImpactV2.PARTIAL)
self.assertEqual(cve.integrity_impact_v2, metrics.ImpactV2.NONE)
self.assertEqual(cve.availability_impact_v2, metrics.ImpactV2.NONE)
self.assertEqual(cve.attack_vector_v3, metrics.AttackVectorV3.NETWORK)
self.assertEqual(cve.attack_complexity_v3,
metrics.AttackComplexityV3.LOW)
self.assertEqual(cve.privileges_required_v3,
metrics.PrivilegesRequiredV3.NONE)
self.assertEqual(cve.user_interaction_v3,
metrics.UserInteractionV3.REQUIRED)
self.assertEqual(cve.scope_v3, metrics.ScopeV3.UNCHANGED)
self.assertEqual(cve.confidentiality_impact_v3, metrics.ImpactV3.LOW)
self.assertEqual(cve.integrity_impact_v3, metrics.ImpactV3.NONE)
self.assertEqual(cve.availability_impact_v3, metrics.ImpactV3.NONE)
self.assertEqual(cve.get_privileges_required_v3_value(), 0.85)
def parse(self, report, file_url) -> [Dict, Dict]:
for host in iter_elements_by_name(report, "ReportHost"):
scan_date = host.find(
'HostProperties/tag[@name="HOST_START_TIMESTAMP"]').text
scan_date = datetime.fromtimestamp(int(scan_date))
asset = AssetFactory.create(host, self.__config)
asset.last_scan_date = scan_date
self.__scanned_hosts.append(asset)
for item in host.iter('ReportItem'):
if item.get('severity') != NessusReportParser.INFO:
vuln = dict()
vuln['scan_date'] = scan_date
vuln['scan_file_url'] = file_url
vuln['asset'] = asset
vuln['plugin_id'] = item.get('pluginID')
vuln['name'] = item.get('pluginName')
vuln['port'] = item.get('port')
if vuln['port'] != '0':
vuln['svc_name'] = item.get('svc_name')
vuln['protocol'] = item.get('protocol')
else:
vuln['port'] = None
vuln['svc_name'] = None
vuln['protocol'] = None
vuln['description'] = get_value(item.find('description'))
vuln['solution'] = get_value(item.find('solution'))
vuln['exploit_available'] = True if get_value(
item.find('exploit_available')) == 'true' else False
vuln[
'tenant'] = self.__config.tenant.name if self.__config.tenant else None
cves = item.findall('cve')
if cves:
for cve in cves:
vuln['cve_id'] = get_value(cve)
vuln['cve'] = CveDocument.get_or_create(
cve_id=vuln['cve_id'])
vuln['id'] = self._vuln_id(vuln, vuln['cve_id'])
self._create(vuln)
else:
vuln['cve'] = self._create_nessus_cve(item)
vuln['id'] = self._vuln_id(vuln, vuln['cve'].id)
self._create(vuln)
return self.__parsed, self.__scanned_hosts
def create(key: str, value: dict) -> [CveDocument, None]:
try:
exploits = []
for exp_id in value['refmap']['exploit-db']:
exploits.append(ExploitInnerDoc.create(exp_id=exp_id))
except KeyError:
pass
else:
result = CveDocument.search().filter('term', id=key)[0].execute()
if result.hits and result.hits[0].exploits != exploits:
result.hits[0].exploits = exploits
return result.hits[0]
return None
def test_call_call_not_update(self):
with open(get_fixture_location(__file__,
'nvdcve-1.0-2017-2.json')) as handle:
CveFactory.process(handle)
thread_pool_executor.wait_for_all()
result = CveDocument.search().filter('term',
id='CVE-2017-0002').execute()
self.assertEqual(len(result.hits), 1)
cve = result.hits[0]
self.assertEqual(cve.id, 'CVE-2017-0002')
self.assertEqual(cve.access_vector_v2, metrics.AccessVectorV2.NETWORK)
self.assertEqual(cve.access_complexity_v2,
metrics.AccessComplexityV2.MEDIUM)
self.assertEqual(cve.authentication_v2, metrics.AuthenticationV2.NONE)
self.assertEqual(cve.confidentiality_impact_v2,
metrics.ImpactV2.PARTIAL)
self.assertEqual(cve.integrity_impact_v2, metrics.ImpactV2.PARTIAL)
self.assertEqual(cve.availability_impact_v2, metrics.ImpactV2.PARTIAL)
def get_cve(cve_id, oid, tags):
if cve_id == 'NOCVE':
cve_id = F'NOCVE-{oid}'
cve = CveDocument.get_or_create(cve_id=cve_id)
vector = tags['cvss_base_vector']
vector = dict(x.split(':') for x in vector.split('/'))
new_cve = CveDocument(id=cve_id)
new_cve.access_vector_v2 = metrics.AccessVectorV2(vector['AV'])
new_cve.access_complexity_v2 = metrics.AccessComplexityV2(vector['AC'])
new_cve.authentication_v2 = metrics.AuthenticationV2(vector['Au'])
new_cve.confidentiality_impact_v2 = metrics.ImpactV2(vector['C'])
new_cve.integrity_impact_v2 = metrics.ImpactV2(vector['I'])
new_cve.availability_impact_v2 = metrics.ImpactV2(vector['A'])
new_cve.base_score_v2 = calculate_base_score_v2(new_cve)
if cve.has_changed(new_cve):
return cve.update(new_cve, refresh=True)
return cve
return CveDocument.get_or_create(cve_id=cve_id)
def exploitability_v3(cve: CveDocument) -> float:
return 8.22 * \
cve.attack_vector_v3.second_value * \
cve.attack_complexity_v3.second_value * \
cve.get_privileges_required_v3_value() * \
cve.user_interaction_v3.second_value
def test_calculate_base_score(self):
for cve in CveDocument.search():
self.assertEqual(cve.base_score_v2, calculate_base_score_v2(cve),
cve.id)
self.assertEqual(cve.base_score_v3, calculate_base_score_v3(cve),
cve.id)
def get_value(value):
return value if value else ''
def summary(d_cwe, d_cve):
text = ' '.join([
get_value(d_cwe.name),
get_value(d_cwe.description),
get_value(d_cwe.extended_description),
get_value(d_cve)
])
return text.translate(str.maketrans('', '', '\n\t\r')).replace('|', ' ')
if __name__ == 'main':
all_count = CveDocument.search().count()
print('All CVE count', all_count)
search = CveDocument.search().filter('exists', field='base_score_v3')
count = search.count()
print('CVE with 3.1 score:', count)
print('CVE CVSSv3/CVSSv2', count / all_count * 100)
print('Downloading data')
cves = [c for c in search.scan()]
print('Downloaded data')
with open(OUTPUT_FILE, 'w', encoding='utf-8') as out:
for cve in cves:
k = '|'.join([
str(cve.base_score_v2),
class CveFactory:
FIELDS = [i for i in CveDocument.get_fields_name()]
@staticmethod
def process(handle):
data = json.load(handle)
docs = []
for obj in data['CVE_Items']:
cve = CveFactory.create(obj)
if cve:
docs.append(cve.to_dict(include_meta=True))
if len(docs) > 500:
async_bulk(docs, CveDocument.Index.name)
docs = []
async_bulk(docs, CveDocument.Index.name)
@staticmethod
def create(item: dict):
if '** REJECT **' not in CveFactory.summary(item):
old = CveDocument.search().filter(
'term', id=CveFactory.get_id(item)).sort(
'-last_modified_date')[0].execute()
if old.hits:
last_modified_date = old.hits[0].last_modified_date
else:
last_modified_date = None
if not last_modified_date or last_modified_date < CveFactory.last_modified_date(
item):
cve = CveDocument(id=CveFactory.get_id(item))
for field in CveDocument.get_fields_name():
parser = getattr(CveFactory, field, None)
if parser:
try:
setattr(cve, field, parser(item))
except Exception as err:
logging.debug(
F'cve id {cve.id}, field {field}, err {err}')
for cpe in CveFactory.get_cpe(item):
cve.cpe.append(cpe)
if old.hits and cve.has_changed(old.hits[0]):
return old.hits[0].update(cve, weak=True)
else:
return cve.save(weak=True)
return None
logging.info(F'cve id {CveFactory.get_id(item)} is rejected')
return None
@staticmethod
def get_id(item: dict) -> str:
return item['cve']['CVE_data_meta']['ID']
@staticmethod
def base_score_v2(item: dict) -> [float, None]:
return CveFactory.base_score('cvssV2', item)
@staticmethod
def base_score_v3(item: dict) -> [float, None]:
return CveFactory.base_score('cvssV3', item)
@staticmethod
def base_score(version: str, item: dict) -> [float, None]:
score = CveFactory.value_from_base_metrics(version, 'baseScore', item)
return float(score) if score else None
@staticmethod
def summary(item: dict) -> str:
for desc in item['cve']['description']['description_data']:
if desc['lang'] == 'en':
return desc['value']
return str()
@staticmethod
def references(item: RestrictedElement) -> str:
objs = []
for ref in item['cve']['references']['reference_data']:
objs.append({'source': ref['refsource'], 'url': ref['url']})
return json.dumps(objs)
@staticmethod
def cwe(item: dict) -> CweDocument:
for problemtype_data in item['cve']['problemtype']['problemtype_data']:
for desc in problemtype_data['description']:
if desc['lang'] == 'en':
return CWEFactory.get(desc['value'])
@staticmethod
def get_cpe(item: dict) -> list:
cpes = []
try:
for conf in item['configurations']['nodes']:
for cpe_match in conf['cpe_match']:
cpes.append(CpeFactory.get(cpe_match['cpe23Uri']))
except (KeyError, IndexError):
pass
return cpes
@staticmethod
def published_date(item: dict) -> datetime:
return parse_datetime(item['publishedDate'])
@staticmethod
def last_modified_date(item: dict) -> datetime:
return parse_datetime(item['lastModifiedDate'])
@staticmethod
def access_vector_v2(item: dict) -> metrics.AccessVectorV2:
av = CveFactory.value_from_base_metrics('cvssV2', 'accessVector', item)
return metrics.AccessVectorV2(av).value
@staticmethod
def access_complexity_v2(item: dict) -> metrics.AccessComplexityV2:
ac = CveFactory.value_from_base_metrics('cvssV2', 'accessComplexity',
item)
return metrics.AccessComplexityV2(ac).value
@staticmethod
def authentication_v2(item: dict) -> metrics.AuthenticationV2:
auth = CveFactory.value_from_base_metrics('cvssV2', 'authentication',
item)
return metrics.AuthenticationV2(auth).value
@staticmethod
def confidentiality_impact_v2(item: dict) -> metrics.ImpactV2:
imp = CveFactory.value_from_base_metrics('cvssV2',
'confidentialityImpact', item)
return metrics.ImpactV2(imp).value
@staticmethod
def integrity_impact_v2(item: dict) -> metrics.ImpactV2:
imp = CveFactory.value_from_base_metrics('cvssV2', 'integrityImpact',
item)
return metrics.ImpactV2(imp).value
@staticmethod
def availability_impact_v2(item: dict) -> metrics.ImpactV2:
imp = CveFactory.value_from_base_metrics('cvssV2',
'availabilityImpact', item)
return metrics.ImpactV2(imp).value
@staticmethod
def attack_vector_v3(item: dict) -> metrics.AttackVectorV3:
av = CveFactory.value_from_base_metrics('cvssV3', 'attackVector', item)
return metrics.AttackVectorV3(av).value
@staticmethod
def attack_complexity_v3(item: dict) -> metrics.AttackComplexityV3:
ac = CveFactory.value_from_base_metrics('cvssV3', 'attackComplexity',
item)
return metrics.AttackComplexityV3(ac).value
@staticmethod
def privileges_required_v3(item: dict) -> metrics.PrivilegesRequiredV3:
pr = CveFactory.value_from_base_metrics('cvssV3', 'privilegesRequired',
item)
return metrics.PrivilegesRequiredV3(pr).value
@staticmethod
def user_interaction_v3(item: dict) -> metrics.UserInteractionV3:
us = CveFactory.value_from_base_metrics('cvssV3', 'userInteraction',
item)
return metrics.UserInteractionV3(us).value
@staticmethod
def scope_v3(item: dict) -> metrics.ScopeV3:
sc = CveFactory.value_from_base_metrics('cvssV3', 'scope', item)
return metrics.ScopeV3(sc).value
@staticmethod
def confidentiality_impact_v3(item: dict) -> metrics.ImpactV3:
ci = CveFactory.value_from_base_metrics('cvssV3',
'confidentialityImpact', item)
return metrics.ImpactV3(ci).value
@staticmethod
def integrity_impact_v3(item: dict) -> metrics.ImpactV3:
ii = CveFactory.value_from_base_metrics('cvssV3', 'integrityImpact',
item)
return metrics.ImpactV3(ii).value
@staticmethod
def availability_impact_v3(item: dict) -> metrics.ImpactV3:
ai = CveFactory.value_from_base_metrics('cvssV3', 'availabilityImpact',
item)
return metrics.ImpactV3(ai).value
@staticmethod
def value_from_base_metrics(version: str, value: str,
item: dict) -> [str, None]:
return item['impact']['baseMetricV2' if version ==
'cvssV2' else 'baseMetricV3'][version][value]
def test_call_call_not_create_rejected(self):
self.assertFalse(CveDocument.search().filter(
'term', id='CVE-2017-0605').execute())