def load_up_all_xml_into_db(): if not os.path.exists(XML_DIR): os.makedirs(XML_DIR) xml_exists = os.listdir(XML_DIR) if not xml_exists: logger.info('downloading nvd/cve xml data files') start_nvd_xml_download() for directory, subdirectories, files in os.walk(XML_DIR): for xml_file in files: nvd_file = os.path.join(directory, xml_file) parse_cve_and_udpatedb(False, nvd_file) update_cve_categories()
def parse_cve_and_udpatedb(download_latest_nvd=True, nvd_file=NVD_MODIFIED_FILE): if download_latest_nvd: start_nvd_xml_download() parser = NvdParser() cve_data_list = [] cve_data = {} for event, entry in etree.iterparse(nvd_file, events=['start', 'end']): if entry.tag == NVD_FEEDS_ENTRY and event == 'start': cve_data = parser.get_entry_info(entry) if entry.tag == NVD_FEEDS_DESC and event == 'start': cve_data[CveKey.CveDescriptions] = parser.get_descriptions(entry) if entry.tag == NVD_FEEDS_REFS and event == 'start': cve_data[CveKey.CveRefs] = parser.get_refs(entry) if entry.tag == NVD_FEEDS_VULN_SOFT and event == 'start': cve_data[CveKey.CveVulnsSoft] = parser.get_vulns_soft(entry) cve_data[CveKey.CveCategories] = [] if entry.tag == NVD_FEEDS_ENTRY and event == 'end': for key in cve_data.keys(): if (key != CveKey.CveDescriptions and key != CveKey.CveRefs and key != CveKey.CveVulnsSoft and key != CveKey.CvePublishedDate and key != CveKey.CveCategories and key != CveKey.CveModifiedDate): cve_data[key] = unicode(cve_data[key]) cve_data_list.append(cve_data) entry.clear() while entry.getprevious() is not None: del entry.getparent()[0] del entry insert_into_cve_collection(cve_data_list) del cve_data_list del cve_data del parser gc.collect()