def _threaded_cve_gathering(mongoDbDriver, i):
if InternalServer.is_debug_logging_enabled():
DagdaLogger.get_logger().debug('... Including CVEs - ' + str(i))
compressed_content = get_http_resource_content(
"https://nvd.nist.gov/feeds/json/cve/1.1/nvdcve-1.1-" + str(i) + ".json.gz")
cve_list, cve_ext_info_list = get_cve_list_from_file(compressed_content, i)
if len(cve_list) > 0:
mongoDbDriver.bulk_insert_cves(cve_list)
if len(cve_ext_info_list) > 0:
mongoDbDriver.bulk_insert_cves_info(cve_ext_info_list)
def compose_vuln_db(self):
# -- CVE
# Adding or updating CVEs
first_year = self.mongoDbDriver.remove_only_cve_for_update()
for i in range(first_year, next_year):
compressed_content = get_http_resource_content(
"https://static.nvd.nist.gov/feeds/xml/cve/nvdcve-2.0-" +
str(i) + ".xml.gz")
cve_list = get_cve_list_from_file(compressed_content, i)
if len(cve_list) > 0:
self.mongoDbDriver.bulk_insert_cves(cve_list)
# -- Exploit DB
# Adding or updating Exploit_db
self.mongoDbDriver.delete_exploit_db_collection()
csv_content = get_http_resource_content(
'https://github.com/offensive-security/exploit-database/raw/master/files.csv'
)
self.mongoDbDriver.bulk_insert_exploit_db_ids(
get_exploit_db_list_from_csv(csv_content.decode("utf-8")))
# -- BID
# Adding BugTraqs from 20161118_sf_db.json.gz, where 94417 is the max bid in the gz file
max_bid = self.mongoDbDriver.get_max_bid_inserted()
if max_bid < 94417:
# Clean
if max_bid != 0:
self.mongoDbDriver.delete_bid_collection()
# Adding BIDs
compressed_file = io.BytesIO(
get_http_resource_content(
"https://github.com/eliasgranderubio/bidDB_downloader/raw/master/bonus_track/20161118_sf_db.json.gz"
))
bid_items_array = get_bug_traqs_lists_from_file(compressed_file)
for bid_items_list in bid_items_array:
self.mongoDbDriver.bulk_insert_bids(bid_items_list)
bid_items_list.clear()
# Set the new max bid
max_bid = 94417
# Updating BugTraqs from http://www.securityfocus.com/
bid_items_array = get_bug_traqs_lists_from_online_mode(
bid_downloader(first_bid=max_bid + 1, last_bid=95500))
for bid_items_list in bid_items_array:
self.mongoDbDriver.bulk_insert_bids(bid_items_list)
bid_items_list.clear()
def _threaded_cve_gathering(mongoDbDriver, i):
if InternalServer.is_debug_logging_enabled():
DagdaLogger.get_logger().debug('... Including CVEs - ' + str(i))
compressed_content = get_http_resource_content(
"https://static.nvd.nist.gov/feeds/xml/cve/nvdcve-2.0-" + str(i) +
".xml.gz")
cve_list = get_cve_list_from_file(compressed_content, i)
if len(cve_list) > 0:
mongoDbDriver.bulk_insert_cves(cve_list)
# Add CVE info collection with additional info like score
compressed_content_info = get_http_resource_content(
"https://nvd.nist.gov/download/nvdcve-" + str(i) + ".xml.zip")
cve_info_list = get_cve_description_from_file(compressed_content_info)
compressed_ext_content_info = \
get_http_resource_content("https://static.nvd.nist.gov/feeds/xml/cve/nvdcve-2.0-"
+ str(i) + ".xml.zip")
cve_ext_info_list = get_cve_cweid_from_file(
compressed_ext_content_info, cve_info_list)
if len(cve_ext_info_list) > 0:
mongoDbDriver.bulk_insert_cves_info(cve_ext_info_list)
def compose_vuln_db(self):
if InternalServer.is_debug_logging_enabled():
DagdaLogger.get_logger().debug('ENTRY to the method for composing VulnDB')
# -- CVE
# Adding or updating CVEs
if InternalServer.is_debug_logging_enabled():
DagdaLogger.get_logger().debug('Updating CVE collection ...')
first_year = self.mongoDbDriver.remove_only_cve_for_update()
for i in range(first_year, next_year):
if InternalServer.is_debug_logging_enabled():
DagdaLogger.get_logger().debug('... Including CVEs - ' + str(i))
compressed_content = get_http_resource_content(
"https://static.nvd.nist.gov/feeds/xml/cve/nvdcve-2.0-" + str(i) + ".xml.gz")
cve_list = get_cve_list_from_file(compressed_content, i)
if len(cve_list) > 0:
self.mongoDbDriver.bulk_insert_cves(cve_list)
# Add CVE info collection with additional info like score
compressed_content_info = get_http_resource_content("https://nvd.nist.gov/download/nvdcve-"
+ str(i) + ".xml.zip")
cve_info_list = get_cve_description_from_file(compressed_content_info)
compressed_ext_content_info = \
get_http_resource_content("https://static.nvd.nist.gov/feeds/xml/cve/nvdcve-2.0-"
+ str(i) + ".xml.zip")
cve_ext_info_list = get_cve_cweid_from_file(compressed_ext_content_info, cve_info_list)
if len(cve_ext_info_list) > 0:
self.mongoDbDriver.bulk_insert_cves_info(cve_ext_info_list)
if InternalServer.is_debug_logging_enabled():
DagdaLogger.get_logger().debug('CVE collection updated')
# -- Exploit DB
# Adding or updating Exploit_db and Exploit_db info
if InternalServer.is_debug_logging_enabled():
DagdaLogger.get_logger().debug('Updating Exploit DB collection ...')
self.mongoDbDriver.delete_exploit_db_collection()
self.mongoDbDriver.delete_exploit_db_info_collection()
csv_content = get_http_resource_content(
'https://raw.githubusercontent.com/offensive-security/exploit-database/master/files_exploits.csv')
exploit_db_list, exploit_db_info_list = get_exploit_db_list_from_csv(csv_content.decode("utf-8"))
self.mongoDbDriver.bulk_insert_exploit_db_ids(exploit_db_list)
self.mongoDbDriver.bulk_insert_exploit_db_info(exploit_db_info_list)
if InternalServer.is_debug_logging_enabled():
DagdaLogger.get_logger().debug('Exploit DB collection updated')
# -- BID
# Adding BugTraqs from 20180328_sf_db.json.gz, where 103525 is the max bid in the gz file
if InternalServer.is_debug_logging_enabled():
DagdaLogger.get_logger().debug('Updating BugTraqs Id collection ...')
max_bid = self.mongoDbDriver.get_max_bid_inserted()
if max_bid < 103525:
# Clean
if max_bid != 0:
self.mongoDbDriver.delete_bid_collection()
self.mongoDbDriver.delete_bid_info_collection()
# Adding BIDs
compressed_file = io.BytesIO(get_http_resource_content(
"https://github.com/eliasgranderubio/bidDB_downloader/raw/master/bonus_track/20180328_sf_db.json.gz"))
bid_items_array, bid_detail_array = get_bug_traqs_lists_from_file(compressed_file)
# Insert BIDs
for bid_items_list in bid_items_array:
self.mongoDbDriver.bulk_insert_bids(bid_items_list)
bid_items_list.clear()
# Insert BID details
self.mongoDbDriver.bulk_insert_bid_info(bid_detail_array)
bid_detail_array.clear()
# Set the new max bid
max_bid = 103525
# Updating BugTraqs from http://www.securityfocus.com/
bid_items_array, bid_detail_array = get_bug_traqs_lists_from_online_mode(bid_downloader(first_bid=max_bid+1,
last_bid=104000))
# Insert BIDs
if len(bid_items_array) > 0:
for bid_items_list in bid_items_array:
self.mongoDbDriver.bulk_insert_bids(bid_items_list)
bid_items_list.clear()
# Insert BID details
if len(bid_detail_array) > 0:
self.mongoDbDriver.bulk_insert_bid_info(bid_detail_array)
bid_detail_array.clear()
if InternalServer.is_debug_logging_enabled():
DagdaLogger.get_logger().debug('BugTraqs Id collection updated')
# -- RHSA (Red Hat Security Advisory) and RHBA (Red Hat Bug Advisory)
# Adding or updating rhsa and rhba collections
if InternalServer.is_debug_logging_enabled():
DagdaLogger.get_logger().debug('Updating RHSA & RHBA collections ...')
self.mongoDbDriver.delete_rhba_collection()
self.mongoDbDriver.delete_rhba_info_collection()
self.mongoDbDriver.delete_rhsa_collection()
self.mongoDbDriver.delete_rhsa_info_collection()
bz2_file = get_http_resource_content('https://www.redhat.com/security/data/oval/rhsa.tar.bz2')
rhsa_list, rhba_list, rhsa_info_list, rhba_info_list = get_rhsa_and_rhba_lists_from_file(bz2_file)
self.mongoDbDriver.bulk_insert_rhsa(rhsa_list)
self.mongoDbDriver.bulk_insert_rhba(rhba_list)
self.mongoDbDriver.bulk_insert_rhsa_info(rhsa_info_list)
self.mongoDbDriver.bulk_insert_rhba_info(rhba_info_list)
if InternalServer.is_debug_logging_enabled():
DagdaLogger.get_logger().debug('RHSA & RHBA collections updated')
if InternalServer.is_debug_logging_enabled():
DagdaLogger.get_logger().debug('EXIT from the method for composing VulnDB')
def compose_vuln_db(self):
# -- CVE
# Adding or updating CVEs
first_year = self.mongoDbDriver.remove_only_cve_for_update()
for i in range(first_year, next_year):
compressed_content = get_http_resource_content(
"https://static.nvd.nist.gov/feeds/xml/cve/nvdcve-2.0-" +
str(i) + ".xml.gz")
cve_list = get_cve_list_from_file(compressed_content, i)
if len(cve_list) > 0:
self.mongoDbDriver.bulk_insert_cves(cve_list)
# Add CVE info collection with additional info like score
compressed_content_info = get_http_resource_content(
"https://nvd.nist.gov/download/nvdcve-" + str(i) + ".xml.zip")
cve_info_list = get_cve_description_from_file(
compressed_content_info)
compressed_ext_content_info = \
get_http_resource_content("https://static.nvd.nist.gov/feeds/xml/cve/nvdcve-2.0-"
+ str(i) + ".xml.zip")
cve_ext_info_list = get_cve_cweid_from_file(
compressed_ext_content_info, cve_info_list)
if len(cve_ext_info_list) > 0:
self.mongoDbDriver.bulk_insert_cves_info(cve_ext_info_list)
# -- Exploit DB
# Adding or updating Exploit_db and Exploit_db info
self.mongoDbDriver.delete_exploit_db_collection()
self.mongoDbDriver.delete_exploit_db_info_collection()
csv_content = get_http_resource_content(
'https://github.com/offensive-security/exploit-database/raw/master/files.csv'
)
exploit_db_list, exploit_db_info_list = get_exploit_db_list_from_csv(
csv_content.decode("utf-8"))
self.mongoDbDriver.bulk_insert_exploit_db_ids(exploit_db_list)
self.mongoDbDriver.bulk_insert_exploit_db_info(exploit_db_info_list)
# -- BID
# Adding BugTraqs from 20170507_sf_db.json.gz, where 98325 is the max bid in the gz file
max_bid = self.mongoDbDriver.get_max_bid_inserted()
if max_bid < 98325:
# Clean
if max_bid != 0:
self.mongoDbDriver.delete_bid_collection()
self.mongoDbDriver.delete_bid_info_collection()
# Adding BIDs
compressed_file = io.BytesIO(
get_http_resource_content(
"https://github.com/eliasgranderubio/bidDB_downloader/raw/master/bonus_track/20170507_sf_db.json.gz"
))
bid_items_array, bid_detail_array = get_bug_traqs_lists_from_file(
compressed_file)
# Insert BIDs
for bid_items_list in bid_items_array:
self.mongoDbDriver.bulk_insert_bids(bid_items_list)
bid_items_list.clear()
# Insert BID details
self.mongoDbDriver.bulk_insert_bid_info(bid_detail_array)
bid_detail_array.clear()
# Set the new max bid
max_bid = 98325
# Updating BugTraqs from http://www.securityfocus.com/
bid_items_array, bid_detail_array = get_bug_traqs_lists_from_online_mode(
bid_downloader(first_bid=max_bid + 1, last_bid=100000))
# Insert BIDs
for bid_items_list in bid_items_array:
self.mongoDbDriver.bulk_insert_bids(bid_items_list)
bid_items_list.clear()
# Insert BID details
self.mongoDbDriver.bulk_insert_bid_info(bid_detail_array)
bid_detail_array.clear()