def parse_cve_and_udpatedb( download_latest_nvd=True, nvd_files=[ CVEDataDir.NVD_RECENT_FILE_UNCOMPRESSED[:-3], CVEDataDir.NVD_MODIFIED_FILE_UNCOMPRESSED[:-3], CVEDataDir.NVD_CURRENT_FILE[:-3], ], ): """This begins the actual parsing of the xml files and loads up the data into the database Kwargs: download_latest_nvd (bool): Whether or not to download the latest nvd data nvd_files (list): This is a list of the files you want to parse """ if download_latest_nvd: start_nvd_xml_download() parser = NvdParser() for nvd_file in nvd_files: cve_data_list = [] cve_data = {} for event, entry in etree.iterparse(nvd_file, events=["start", "end"]): if entry.tag == NVDFeeds.ENTRY and event == "start": cve_data = parser.get_entry_info(entry) if entry.tag == NVDFeeds.DESC and event == "start": cve_data[CveKey.CveDescriptions] = parser.get_descriptions(entry) if entry.tag == NVDFeeds.REFS and event == "start": cve_data[CveKey.CveRefs] = parser.get_refs(entry) # if entry.tag == NVDFeeds.VULN_SOFT and event == 'start': # cve_data[CveKey.CveVulnsSoft] = parser.get_vulns_soft(entry) cve_data[CveKey.CveCategories] = [] if entry.tag == NVDFeeds.ENTRY and event == "end": for key in cve_data.keys(): if ( key != CveKey.CveDescriptions and key != CveKey.CveRefs and key != CveKey.CveVulnsSoft and key != CveKey.CvePublishedDate and key != CveKey.CveCategories and key != CveKey.CvssVector and key != CveKey.CveModifiedDate ): cve_data[key] = unicode(cve_data[key]) cve_data_list.append(cve_data) # entry.clear() # while entry.getprevious() is not None: # del entry.getparent()[0] # del entry insert_cve_data(cve_data_list) del cve_data_list del cve_data gc.collect()
def parse_cve_and_udpatedb(download_latest_nvd=True, nvd_files=[ CVEDataDir.NVD_RECENT_FILE_UNCOMPRESSED[:-3], CVEDataDir.NVD_MODIFIED_FILE_UNCOMPRESSED[:-3], CVEDataDir.NVD_CURRENT_FILE[:-3] ]): """This begins the actual parsing of the xml files and loads up the data into the database Kwargs: download_latest_nvd (bool): Whether or not to download the latest nvd data nvd_files (list): This is a list of the files you want to parse """ if download_latest_nvd: start_nvd_xml_download() parser = NvdParser() for nvd_file in nvd_files: cve_data_list = [] cve_data = {} for event, entry in etree.iterparse(nvd_file, events=['start', 'end']): if entry.tag == NVDFeeds.ENTRY and event == 'start': cve_data = parser.get_entry_info(entry) if entry.tag == NVDFeeds.DESC and event == 'start': cve_data[CveKey.CveDescriptions] = \ parser.get_descriptions(entry) if entry.tag == NVDFeeds.REFS and event == 'start': cve_data[CveKey.CveRefs] = parser.get_refs(entry) #if entry.tag == NVDFeeds.VULN_SOFT and event == 'start': # cve_data[CveKey.CveVulnsSoft] = parser.get_vulns_soft(entry) cve_data[CveKey.CveCategories] = [] if entry.tag == NVDFeeds.ENTRY and event == 'end': for key in cve_data.keys(): if (key != CveKey.CveDescriptions and key != CveKey.CveRefs and key != CveKey.CveVulnsSoft and key != CveKey.CvePublishedDate and key != CveKey.CveCategories and key != CveKey.CvssVector and key != CveKey.CveModifiedDate): cve_data[key] = unicode(cve_data[key]) cve_data_list.append(cve_data) #entry.clear() #while entry.getprevious() is not None: # del entry.getparent()[0] #del entry insert_cve_data(cve_data_list) del cve_data_list del cve_data gc.collect()
def load_up_all_xml_into_db(): nvd_files = [] if not os.path.exists(CVEDataDir.XML_DIR): os.makedirs(CVEDataDir.XML_DIR) xml_exists = os.listdir(CVEDataDir.XML_DIR) logger.info('starting cve/nvd update process') if not xml_exists: logger.info('downloading nvd/cve xml data files') start_nvd_xml_download() for directory, _, files in os.walk(CVEDataDir.XML_DIR): for xml_file in files: nvd_file = os.path.join(directory, xml_file) nvd_files.append(nvd_file) parse_cve_and_udpatedb(False, nvd_files) update_cve_categories() logger.info('finished cve/nvd update process') gc.collect()