def add_module_status_log(info):
log_to_add = {
'log_module_keyword': info['module_keyword'],
'log_module_state': info['state'],
'log_module_domain': info['domain'],
#Found is just used for recon modules
'log_module_found': info['found'],
'log_module_arguments': info['arguments'],
'log_module_timestamp': datetime.now()
}
log_id = logs.insert_one(log_to_add)
log_to_add['log_id'] = str(log_to_add.pop('_id'))
ELASTIC_CLIENT.index(index='log_module',
doc_type='_doc',
id=log_to_add['log_id'],
body=log_to_add)
def update_elasticsearch_logs():
print('Synchronizing log files')
logs_found = logs.find()
for log in logs_found:
log['log_id'] = str(log.pop('_id'))
try:
something = log['log_module_keyword']
ELASTIC_CLIENT.index(index='log_module',
doc_type='_doc',
id=log['log_id'],
body=log)
except KeyError:
pass
try:
something = log['log_vulnerability_module_keyword']
ELASTIC_CLIENT.index(index='log_vuln',
doc_type='_doc',
id=log['log_id'],
body=log)
except KeyError:
pass
try:
something = log['log_resource_module_keyword']
ELASTIC_CLIENT.index(index='log_resource',
doc_type='_doc',
id=log['log_id'],
body=log)
except KeyError:
pass
def add_resource_found_log(resource, module_keyword):
log_to_add = {
"log_resource_module_keyword": module_keyword,
"log_resource_domain": resource['domain'],
"log_resource_subdomain": resource['subdomain'],
"log_resource_id": str(resource['_id']),
"log_resource_timestamp": datetime.now()
}
log_id = logs.insert_one(log_to_add)
log_to_add['log_id'] = str(log_to_add.pop('_id'))
res = ELASTIC_CLIENT.index(index='log_resource',
doc_type='_doc',
id=log_to_add['log_id'],
body=log_to_add)
def add_found_vulnerability_log(vulnerability, vuln_obj):
log_to_add = {
"log_vulnerability_module_keyword": vuln_obj.module_identifier,
"log_vulnerability_found": True,
"log_vulnerability_id": str(vulnerability['_id']),
"log_vulnerability_name": vulnerability['vulnerability_name'],
"log_vulnerability_timestamp": datetime.now()
}
log_id = logs.insert_one(log_to_add)
log_to_add['log_id'] = str(log_to_add.pop('_id'))
res = ELASTIC_CLIENT.index(index='log_vuln',
doc_type='_doc',
id=log_to_add['log_id'],
body=log_to_add)
def add_resource_to_elastic(resource):
if ELASTIC_CLIENT is None:
return
resource_url = None
if resource['url'] is not None:
for url in resource['url']:
resource_url = url['url']
if 'https' in url['url']:
break
resource_to_add = {
'resource_id': str(resource['_id']),
'resource_domain': resource['domain'],
'resource_subdomain': resource['subdomain'],
'resource_ip': resource['ip'],
'resource_is_alive':
False if resource['is_alive'] == "False" else True,
'resource_additional_info': {
'resource_isp':
resource['additional_info']['isp'],
'resource_asn':
resource['additional_info']['asn'],
'resource_country':
resource['additional_info']['country'],
'resource_region':
resource['additional_info']['region'],
'resource_city':
resource['additional_info']['city'],
'resource_org':
resource['additional_info']['org'],
'resource_geoloc':
'0 , 0' if resource['additional_info']['geoloc'] == 'None , None'
else resource['additional_info']['geoloc']
},
'resource_first_seen': resource['first_seen'],
'resource_last_seen': resource['last_seen'],
'resource_scanned': bool(resource['scanned']),
'resource_type': resource['type'],
'resource_priority': resource['priority'],
'resource_exposition': resource['exposition'],
'resource_asset_value': resource['asset_value'],
'resource_has_urls': bool(resource['has_urls']),
'resource_responsive_urls': resource_url,
'resource_nmap_information': resource['nmap_information']
}
res = ELASTIC_CLIENT.index(index='resources',
doc_type='_doc',
id=resource_to_add['resource_id'],
body=resource_to_add)
return
def add_vuln_to_elastic(vuln):
if ELASTIC_CLIENT is None:
return
if not vuln['observation']:
observation_data = {
'vulnerability_title': None,
'vulnerability_observation_title': None,
'vulnerability_observation_note': None,
'vulnerability_implication': None,
'vulnerability_recommendation_title': None,
'vulnerability_recommendation_note': None,
'vulnerability_severity': None
}
else:
observation_data = {
'vulnerability_title':
vuln['observation']['title'],
'vulnerability_observation_title':
vuln['observation']['observation_title'],
'vulnerability_observation_note':
vuln['observation']['observation_note'],
'vulnerability_implication':
vuln['observation']['implication'],
'vulnerability_recommendation_title':
vuln['observation']['recommendation_title'],
'vulnerability_recommendation_note':
vuln['observation']['recommendation_note'],
'vulnerability_severity':
vuln['observation']['severity']
}
vulnerability_to_add = {
'vulnerability_id': str(vuln['_id']),
'vulnerability_domain': vuln['domain'],
'vulnerability_subdomain': vuln['resource'],
'vulnerability_vulnerability_name': vuln['vulnerability_name'],
'vulnerability_observation': observation_data,
'vulnerability_extra_info': vuln['extra_info'],
'vulnerability_date_found': vuln['date_found'],
'vulnerability_last_seen': vuln['last_seen'],
'vulnerability_language': vuln['language'],
'vulnerability_cvss_score': vuln['cvss_score'],
'vulnerability_vuln_type': vuln['vuln_type'],
'vulnerability_state': vuln['state']
}
res = ELASTIC_CLIENT.index(index='vulnerabilities',
doc_type='_doc',
id=vulnerability_to_add['vulnerability_id'],
body=vulnerability_to_add)
return
def update_elasticsearch():
new_resources = resources.find()
resources_list = list()
for resource in new_resources:
resource_url = None
if resource['url'] is not None:
for url in resource['url']:
resource_url = url['url']
if 'https' in url['url']:
break
resources_list.append({
'resource_id':
str(resource['_id']),
'resource_domain':
resource['domain'],
'resource_subdomain':
resource['subdomain'],
'resource_ip':
resource['ip'],
'resource_is_alive':
False if resource['is_alive'] == "False" else True,
'resource_additional_info': {
'resource_isp':
resource['additional_info']['isp'],
'resource_asn':
resource['additional_info']['asn'],
'resource_country':
resource['additional_info']['country'],
'resource_region':
resource['additional_info']['region'],
'resource_city':
resource['additional_info']['city'],
'resource_org':
resource['additional_info']['org'],
'resource_geoloc':
'0 , 0' if resource['additional_info']['geoloc']
== 'None , None' else resource['additional_info']['geoloc']
},
'resource_first_seen':
resource['first_seen'],
'resource_last_seen':
resource['last_seen'],
'resource_scanned':
bool(resource['scanned']),
'resource_type':
resource['type'],
'resource_priority':
resource['priority'],
'resource_exposition':
resource['exposition'],
'resource_asset_value':
resource['asset_value'],
'resource_has_urls':
bool(resource['has_urls']),
'resource_responsive_urls':
resource_url,
'resource_nmap_information':
resource['nmap_information']
})
### VULNS ###
new_vulnerabilities = vulnerabilities.find()
vulnerabilities_list = list()
for vuln in new_vulnerabilities:
if not vuln['observation']:
observation_data = {
'vulnerability_title': None,
'vulnerability_observation_title': None,
'vulnerability_observation_note': None,
'vulnerability_implication': None,
'vulnerability_recommendation_title': None,
'vulnerability_recommendation_note': None,
'vulnerability_severity': None
}
else:
observation_data = {
'vulnerability_title':
vuln['observation']['title'],
'vulnerability_observation_title':
vuln['observation']['observation_title'],
'vulnerability_observation_note':
vuln['observation']['observation_note'],
'vulnerability_implication':
vuln['observation']['implication'],
'vulnerability_recommendation_title':
vuln['observation']['recommendation_title'],
'vulnerability_recommendation_note':
vuln['observation']['recommendation_note'],
'vulnerability_severity':
vuln['observation']['severity']
}
vulnerabilities_list.append({
'vulnerability_id':
str(vuln['_id']),
'vulnerability_domain':
vuln['domain'],
'vulnerability_subdomain':
vuln['resource'],
'vulnerability_vulnerability_name':
vuln['vulnerability_name'],
'vulnerability_observation':
observation_data,
'vulnerability_extra_info':
vuln['extra_info'],
'vulnerability_date_found':
vuln['date_found'],
'vulnerability_last_seen':
vuln['last_seen'],
'vulnerability_language':
vuln['language'],
'vulnerability_cvss_score':
vuln['cvss_score'],
'vulnerability_cvss3_severity':
resolve_severity(vuln['cvss_score']),
'vulnerability_vuln_type':
vuln['vuln_type'],
'vulnerability_state':
vuln['state']
})
if ELASTIC_CLIENT is None:
return
# Connect to the elastic cluster
print('Adding resources to elasticsearch')
for resource in resources_list:
res = ELASTIC_CLIENT.index(index='resources',
doc_type='_doc',
id=resource['resource_id'],
body=resource)
print('Adding vulnerabilities to elasticsearch')
for vuln in vulnerabilities_list:
res = ELASTIC_CLIENT.index(index='vulnerabilities',
doc_type='_doc',
id=vuln['vulnerability_id'],
body=vuln)