def _threaded_cve_gathering(mongoDbDriver, i): if InternalServer.is_debug_logging_enabled(): DagdaLogger.get_logger().debug('... Including CVEs - ' + str(i)) compressed_content = get_http_resource_content( "https://nvd.nist.gov/feeds/json/cve/1.1/nvdcve-1.1-" + str(i) + ".json.gz") cve_list, cve_ext_info_list = get_cve_list_from_file(compressed_content, i) if len(cve_list) > 0: mongoDbDriver.bulk_insert_cves(cve_list) if len(cve_ext_info_list) > 0: mongoDbDriver.bulk_insert_cves_info(cve_ext_info_list)
def compose_vuln_db(self): # -- CVE # Adding or updating CVEs first_year = self.mongoDbDriver.remove_only_cve_for_update() for i in range(first_year, next_year): compressed_content = get_http_resource_content( "https://static.nvd.nist.gov/feeds/xml/cve/nvdcve-2.0-" + str(i) + ".xml.gz") cve_list = get_cve_list_from_file(compressed_content, i) if len(cve_list) > 0: self.mongoDbDriver.bulk_insert_cves(cve_list) # -- Exploit DB # Adding or updating Exploit_db self.mongoDbDriver.delete_exploit_db_collection() csv_content = get_http_resource_content( 'https://github.com/offensive-security/exploit-database/raw/master/files.csv' ) self.mongoDbDriver.bulk_insert_exploit_db_ids( get_exploit_db_list_from_csv(csv_content.decode("utf-8"))) # -- BID # Adding BugTraqs from 20161118_sf_db.json.gz, where 94417 is the max bid in the gz file max_bid = self.mongoDbDriver.get_max_bid_inserted() if max_bid < 94417: # Clean if max_bid != 0: self.mongoDbDriver.delete_bid_collection() # Adding BIDs compressed_file = io.BytesIO( get_http_resource_content( "https://github.com/eliasgranderubio/bidDB_downloader/raw/master/bonus_track/20161118_sf_db.json.gz" )) bid_items_array = get_bug_traqs_lists_from_file(compressed_file) for bid_items_list in bid_items_array: self.mongoDbDriver.bulk_insert_bids(bid_items_list) bid_items_list.clear() # Set the new max bid max_bid = 94417 # Updating BugTraqs from http://www.securityfocus.com/ bid_items_array = get_bug_traqs_lists_from_online_mode( bid_downloader(first_bid=max_bid + 1, last_bid=95500)) for bid_items_list in bid_items_array: self.mongoDbDriver.bulk_insert_bids(bid_items_list) bid_items_list.clear()
def _threaded_cve_gathering(mongoDbDriver, i): if InternalServer.is_debug_logging_enabled(): DagdaLogger.get_logger().debug('... Including CVEs - ' + str(i)) compressed_content = get_http_resource_content( "https://static.nvd.nist.gov/feeds/xml/cve/nvdcve-2.0-" + str(i) + ".xml.gz") cve_list = get_cve_list_from_file(compressed_content, i) if len(cve_list) > 0: mongoDbDriver.bulk_insert_cves(cve_list) # Add CVE info collection with additional info like score compressed_content_info = get_http_resource_content( "https://nvd.nist.gov/download/nvdcve-" + str(i) + ".xml.zip") cve_info_list = get_cve_description_from_file(compressed_content_info) compressed_ext_content_info = \ get_http_resource_content("https://static.nvd.nist.gov/feeds/xml/cve/nvdcve-2.0-" + str(i) + ".xml.zip") cve_ext_info_list = get_cve_cweid_from_file( compressed_ext_content_info, cve_info_list) if len(cve_ext_info_list) > 0: mongoDbDriver.bulk_insert_cves_info(cve_ext_info_list)
def compose_vuln_db(self): if InternalServer.is_debug_logging_enabled(): DagdaLogger.get_logger().debug('ENTRY to the method for composing VulnDB') # -- CVE # Adding or updating CVEs if InternalServer.is_debug_logging_enabled(): DagdaLogger.get_logger().debug('Updating CVE collection ...') first_year = self.mongoDbDriver.remove_only_cve_for_update() for i in range(first_year, next_year): if InternalServer.is_debug_logging_enabled(): DagdaLogger.get_logger().debug('... Including CVEs - ' + str(i)) compressed_content = get_http_resource_content( "https://static.nvd.nist.gov/feeds/xml/cve/nvdcve-2.0-" + str(i) + ".xml.gz") cve_list = get_cve_list_from_file(compressed_content, i) if len(cve_list) > 0: self.mongoDbDriver.bulk_insert_cves(cve_list) # Add CVE info collection with additional info like score compressed_content_info = get_http_resource_content("https://nvd.nist.gov/download/nvdcve-" + str(i) + ".xml.zip") cve_info_list = get_cve_description_from_file(compressed_content_info) compressed_ext_content_info = \ get_http_resource_content("https://static.nvd.nist.gov/feeds/xml/cve/nvdcve-2.0-" + str(i) + ".xml.zip") cve_ext_info_list = get_cve_cweid_from_file(compressed_ext_content_info, cve_info_list) if len(cve_ext_info_list) > 0: self.mongoDbDriver.bulk_insert_cves_info(cve_ext_info_list) if InternalServer.is_debug_logging_enabled(): DagdaLogger.get_logger().debug('CVE collection updated') # -- Exploit DB # Adding or updating Exploit_db and Exploit_db info if InternalServer.is_debug_logging_enabled(): DagdaLogger.get_logger().debug('Updating Exploit DB collection ...') self.mongoDbDriver.delete_exploit_db_collection() self.mongoDbDriver.delete_exploit_db_info_collection() csv_content = get_http_resource_content( 'https://raw.githubusercontent.com/offensive-security/exploit-database/master/files_exploits.csv') exploit_db_list, exploit_db_info_list = get_exploit_db_list_from_csv(csv_content.decode("utf-8")) self.mongoDbDriver.bulk_insert_exploit_db_ids(exploit_db_list) self.mongoDbDriver.bulk_insert_exploit_db_info(exploit_db_info_list) if InternalServer.is_debug_logging_enabled(): DagdaLogger.get_logger().debug('Exploit DB collection updated') # -- BID # Adding BugTraqs from 20180328_sf_db.json.gz, where 103525 is the max bid in the gz file if InternalServer.is_debug_logging_enabled(): DagdaLogger.get_logger().debug('Updating BugTraqs Id collection ...') max_bid = self.mongoDbDriver.get_max_bid_inserted() if max_bid < 103525: # Clean if max_bid != 0: self.mongoDbDriver.delete_bid_collection() self.mongoDbDriver.delete_bid_info_collection() # Adding BIDs compressed_file = io.BytesIO(get_http_resource_content( "https://github.com/eliasgranderubio/bidDB_downloader/raw/master/bonus_track/20180328_sf_db.json.gz")) bid_items_array, bid_detail_array = get_bug_traqs_lists_from_file(compressed_file) # Insert BIDs for bid_items_list in bid_items_array: self.mongoDbDriver.bulk_insert_bids(bid_items_list) bid_items_list.clear() # Insert BID details self.mongoDbDriver.bulk_insert_bid_info(bid_detail_array) bid_detail_array.clear() # Set the new max bid max_bid = 103525 # Updating BugTraqs from http://www.securityfocus.com/ bid_items_array, bid_detail_array = get_bug_traqs_lists_from_online_mode(bid_downloader(first_bid=max_bid+1, last_bid=104000)) # Insert BIDs if len(bid_items_array) > 0: for bid_items_list in bid_items_array: self.mongoDbDriver.bulk_insert_bids(bid_items_list) bid_items_list.clear() # Insert BID details if len(bid_detail_array) > 0: self.mongoDbDriver.bulk_insert_bid_info(bid_detail_array) bid_detail_array.clear() if InternalServer.is_debug_logging_enabled(): DagdaLogger.get_logger().debug('BugTraqs Id collection updated') # -- RHSA (Red Hat Security Advisory) and RHBA (Red Hat Bug Advisory) # Adding or updating rhsa and rhba collections if InternalServer.is_debug_logging_enabled(): DagdaLogger.get_logger().debug('Updating RHSA & RHBA collections ...') self.mongoDbDriver.delete_rhba_collection() self.mongoDbDriver.delete_rhba_info_collection() self.mongoDbDriver.delete_rhsa_collection() self.mongoDbDriver.delete_rhsa_info_collection() bz2_file = get_http_resource_content('https://www.redhat.com/security/data/oval/rhsa.tar.bz2') rhsa_list, rhba_list, rhsa_info_list, rhba_info_list = get_rhsa_and_rhba_lists_from_file(bz2_file) self.mongoDbDriver.bulk_insert_rhsa(rhsa_list) self.mongoDbDriver.bulk_insert_rhba(rhba_list) self.mongoDbDriver.bulk_insert_rhsa_info(rhsa_info_list) self.mongoDbDriver.bulk_insert_rhba_info(rhba_info_list) if InternalServer.is_debug_logging_enabled(): DagdaLogger.get_logger().debug('RHSA & RHBA collections updated') if InternalServer.is_debug_logging_enabled(): DagdaLogger.get_logger().debug('EXIT from the method for composing VulnDB')
def compose_vuln_db(self): # -- CVE # Adding or updating CVEs first_year = self.mongoDbDriver.remove_only_cve_for_update() for i in range(first_year, next_year): compressed_content = get_http_resource_content( "https://static.nvd.nist.gov/feeds/xml/cve/nvdcve-2.0-" + str(i) + ".xml.gz") cve_list = get_cve_list_from_file(compressed_content, i) if len(cve_list) > 0: self.mongoDbDriver.bulk_insert_cves(cve_list) # Add CVE info collection with additional info like score compressed_content_info = get_http_resource_content( "https://nvd.nist.gov/download/nvdcve-" + str(i) + ".xml.zip") cve_info_list = get_cve_description_from_file( compressed_content_info) compressed_ext_content_info = \ get_http_resource_content("https://static.nvd.nist.gov/feeds/xml/cve/nvdcve-2.0-" + str(i) + ".xml.zip") cve_ext_info_list = get_cve_cweid_from_file( compressed_ext_content_info, cve_info_list) if len(cve_ext_info_list) > 0: self.mongoDbDriver.bulk_insert_cves_info(cve_ext_info_list) # -- Exploit DB # Adding or updating Exploit_db and Exploit_db info self.mongoDbDriver.delete_exploit_db_collection() self.mongoDbDriver.delete_exploit_db_info_collection() csv_content = get_http_resource_content( 'https://github.com/offensive-security/exploit-database/raw/master/files.csv' ) exploit_db_list, exploit_db_info_list = get_exploit_db_list_from_csv( csv_content.decode("utf-8")) self.mongoDbDriver.bulk_insert_exploit_db_ids(exploit_db_list) self.mongoDbDriver.bulk_insert_exploit_db_info(exploit_db_info_list) # -- BID # Adding BugTraqs from 20170507_sf_db.json.gz, where 98325 is the max bid in the gz file max_bid = self.mongoDbDriver.get_max_bid_inserted() if max_bid < 98325: # Clean if max_bid != 0: self.mongoDbDriver.delete_bid_collection() self.mongoDbDriver.delete_bid_info_collection() # Adding BIDs compressed_file = io.BytesIO( get_http_resource_content( "https://github.com/eliasgranderubio/bidDB_downloader/raw/master/bonus_track/20170507_sf_db.json.gz" )) bid_items_array, bid_detail_array = get_bug_traqs_lists_from_file( compressed_file) # Insert BIDs for bid_items_list in bid_items_array: self.mongoDbDriver.bulk_insert_bids(bid_items_list) bid_items_list.clear() # Insert BID details self.mongoDbDriver.bulk_insert_bid_info(bid_detail_array) bid_detail_array.clear() # Set the new max bid max_bid = 98325 # Updating BugTraqs from http://www.securityfocus.com/ bid_items_array, bid_detail_array = get_bug_traqs_lists_from_online_mode( bid_downloader(first_bid=max_bid + 1, last_bid=100000)) # Insert BIDs for bid_items_list in bid_items_array: self.mongoDbDriver.bulk_insert_bids(bid_items_list) bid_items_list.clear() # Insert BID details self.mongoDbDriver.bulk_insert_bid_info(bid_detail_array) bid_detail_array.clear()