def create_or_update(vulnerabilities: dict,
scanned_hosts: list,
config=None) -> None:
index = VulnerabilityDocument.get_index(config)
docs = []
all_vulnerability_docs = VulnerabilityDocument.search(
index=index).filter(~Q('match', tags=VulnerabilityStatus.FIXED))
for current_vuln in all_vulnerability_docs.scan():
vuln_id = current_vuln.id
if vuln_id in vulnerabilities:
if current_vuln.has_changed(vulnerabilities[vuln_id]):
c = current_vuln.update(vulnerabilities[vuln_id],
index=index,
weak=True)
docs.append(c.to_dict(include_meta=True))
del vulnerabilities[vuln_id]
elif vuln_id not in vulnerabilities and current_vuln.asset.ip_address in scanned_hosts:
current_vuln.tags.append(VulnerabilityStatus.FIXED)
c = current_vuln.save(index=index, weak=True)
docs.append(c.to_dict(include_meta=True))
if len(docs) > 500:
async_bulk(docs, index=index)
docs = []
docs.extend(
list(
map(lambda x: x.save(weak=True).to_dict(),
vulnerabilities.values())))
async_bulk(docs, index=index)
def _update_existing_assets(assets: dict, index):
if assets:
updated = []
assets_search = AssetDocument.search(
index=index).filter(~Q('match', tags=AssetStatus.DISCOVERED))
current_assets = [a for a in assets_search.scan()]
for current_asset in current_assets:
asset_id = current_asset.id
if asset_id in assets:
if current_asset.has_changed(assets[asset_id]):
current_asset.update(assets[asset_id],
index=index,
weak=True)
updated.append(
current_asset.to_dict(include_meta=True))
del assets[asset_id]
elif asset_id not in assets and AssetStatus.DELETED not in current_asset.tags:
current_asset.tags.append(AssetStatus.DELETED)
updated.append(
current_asset.save(
index=index, weak=True).to_dict(include_meta=True))
if len(updated) > 500:
async_bulk(updated)
updated = []
async_bulk(updated)
return assets
def create_or_update(assets: dict, config=None) -> None:
index = AssetDocument.get_index(config)
assets = AssetDocument._update_existing_assets(assets, index)
assets = AssetDocument._update_discovered_assets(assets, index)
async_bulk(
list(map(lambda x: x.save(weak=True).to_dict(), assets.values())),
index)
def process(handle):
docs = []
data = json.loads(handle)
for key, value in data['cves'].items():
doc = ExploitFactory.create(key, value)
if doc:
docs.append(doc.to_dict(include_meta=True))
async_bulk(docs, index=CveDocument.Index.name)
def update_gone_discovered_assets(targets, scanned_hosts, discovered_assets, config=None):
index = AssetDocument.get_index(config)
# FIXME: update by query
scanned_ips = [x.ip_address for x in scanned_hosts]
for asset in discovered_assets.scan():
if asset.ip_address in targets and asset.ip_address not in scanned_ips:
asset.tags.append(AssetStatus.DELETED)
asset.save(index=index)
async_bulk(list(map(lambda x: x.save(weak=True).to_dict(include_meta=True), scanned_hosts)), index)
def process(handle):
docs = []
for obj in iter_elements_by_name(handle, 'Weakness'):
cwe = CWEFactory.create(obj)
if cwe:
docs.append(cwe.to_dict(include_meta=True))
if len(docs) > 500:
async_bulk(docs, CweDocument.Index.name)
docs = []
async_bulk(docs, CweDocument.Index.name)
def process(handle):
data = json.load(handle)
docs = []
for obj in data['CVE_Items']:
cve = CveFactory.create(obj)
if cve:
docs.append(cve.to_dict(include_meta=True))
if len(docs) > 500:
async_bulk(docs, CveDocument.Index.name)
docs = []
async_bulk(docs, CveDocument.Index.name)
def _processing(idx, slices_count, assets_count, vulnerability_index):
docs = []
try:
vuln_search = VulnerabilityDocument.search(
index=vulnerability_index).filter(
~Q('match', tags=VulnerabilityStatus.FIXED)
& ~Q('match', asset__tags=AssetStatus.DELETED))
LOGGER.debug(
F'Calculation for {vulnerability_index} and {idx}, {slices_count} started'
)
if slices_count > 1:
vuln_search = vuln_search.extra(slice={
"id": idx,
"max": slices_count
}).params(scroll="60m")
# List competence used due to better performance
vulns = [vuln for vuln in vuln_search.scan()]
LOGGER.debug(F'all vulns for slice {idx} downloaded')
for vuln in vulns:
score, vector = calculate_environmental_score_v3(vuln)
vuln.environmental_score_vector_v3 = vector
vuln.environmental_score_v3 = score
vuln_count = get_cve_count(vulnerability_index, vuln.cve.id)
score, vector = calculate_environmental_score_v2(
vuln, vuln_count, assets_count)
vuln.environmental_score_vector_v2 = vector
vuln.environmental_score_v2 = score
docs.append(vuln.to_dict(include_meta=True))
if len(docs) > 10000:
async_bulk(docs, vulnerability_index)
docs = []
async_bulk(docs, vulnerability_index)
except Exception as ex:
LOGGER.error(F'Unknown processing exception {ex}')
finally:
thread_pool_executor.wait_for_all()
LOGGER.debug(
F'Calculation for {vulnerability_index} and {idx}, {slices_count} done'
)
def update_related(self, sender, new_version, old_version, **kwargs):
if old_version:
updates = []
for document in self._get_related_doc(sender):
for field_name in document.get_fields_name():
field_type = document._doc_type.mapping[field_name]
if isinstance(field_type, Object) and issubclass(
sender, field_type._doc_class):
for index in self._get_indexes(new_version,
field_type._doc_class):
search = document.search(index=index).filter(
'term',
**{F'{field_name}__id': old_version.id})
count = search.count()
if count > 0:
has_related_documents = getattr(
document, 'related_documents', False)
if not has_related_documents:
thread_pool_executor.submit(
self._update_by_query, index
if index else document.Index.name,
field_name, old_version, new_version)
else:
if count > 10000:
result = search.scan()
elif count > 0:
result = search[0:count].execute()
else:
result = []
for hit in result:
setattr(hit, field_name, new_version)
updates.append(
hit.save(index=index,
weak=True).to_dict(
include_meta=True))
if len(updates) > 500:
async_bulk(updates)
updates = []
async_bulk(updates)
def _update_discovered_assets(assets: dict, index):
if assets:
updated = []
assets = {a.ip_address: a for a in assets.values()}
assets_search = AssetDocument.search(index=index).filter(Q('match', tags=AssetStatus.DISCOVERED))
discovered_assets = [a for a in assets_search.scan()]
for discovered_asset in discovered_assets:
if discovered_asset.ip_address in assets:
discovered_asset.update(assets[discovered_asset.ip_address], index=index, weak=True)
updated.append(discovered_asset.to_dict(include_meta=True))
del assets[discovered_asset.ip_address]
if len(updated) > 500:
async_bulk(updated)
updated = []
async_bulk(updated)
return assets
def _reopen(vulnerabilities: dict, index: str):
prev_fixed = VulnerabilityDocument.search(index=index).filter(
Q('match', tags=VulnerabilityStatus.FIXED))
docs = []
for prev_fixed_vuln in prev_fixed.scan():
vuln_id = prev_fixed_vuln.id
if vuln_id in vulnerabilities:
c = prev_fixed_vuln.update(vulnerabilities[vuln_id],
index=index,
weak=True)
c.tags.append(VulnerabilityStatus.REOPEN)
if VulnerabilityStatus.FIXED in c.tags:
c.tags.remove(VulnerabilityStatus.FIXED)
docs.append(c.to_dict(include_meta=True))
del vulnerabilities[vuln_id]
if len(docs) > 500:
async_bulk(docs, index=index)
docs = []
async_bulk(docs, index=index)
return vulnerabilities
