def load_up_all_xml_into_db():
if not os.path.exists(XML_DIR):
os.makedirs(XML_DIR)
xml_exists = os.listdir(XML_DIR)
if not xml_exists:
logger.info('downloading nvd/cve xml data files')
start_nvd_xml_download()
for directory, subdirectories, files in os.walk(XML_DIR):
for xml_file in files:
nvd_file = os.path.join(directory, xml_file)
parse_cve_and_udpatedb(False, nvd_file)
update_cve_categories()
def load_up_all_xml_into_db():
if not os.path.exists(XML_DIR):
os.makedirs(XML_DIR)
xml_exists = os.listdir(XML_DIR)
if not xml_exists:
logger.info('downloading nvd/cve xml data files')
start_nvd_xml_download()
for directory, subdirectories, files in os.walk(XML_DIR):
for xml_file in files:
nvd_file = os.path.join(directory, xml_file)
parse_cve_and_udpatedb(False, nvd_file)
update_cve_categories()
def parse_cve_and_udpatedb(download_latest_nvd=True,
nvd_file=NVD_MODIFIED_FILE):
if download_latest_nvd:
start_nvd_xml_download()
parser = NvdParser()
cve_data_list = []
cve_data = {}
for event, entry in etree.iterparse(nvd_file, events=['start', 'end']):
if entry.tag == NVD_FEEDS_ENTRY and event == 'start':
cve_data = parser.get_entry_info(entry)
if entry.tag == NVD_FEEDS_DESC and event == 'start':
cve_data[CveKey.CveDescriptions] = parser.get_descriptions(entry)
if entry.tag == NVD_FEEDS_REFS and event == 'start':
cve_data[CveKey.CveRefs] = parser.get_refs(entry)
if entry.tag == NVD_FEEDS_VULN_SOFT and event == 'start':
cve_data[CveKey.CveVulnsSoft] = parser.get_vulns_soft(entry)
cve_data[CveKey.CveCategories] = []
if entry.tag == NVD_FEEDS_ENTRY and event == 'end':
for key in cve_data.keys():
if (key != CveKey.CveDescriptions and key != CveKey.CveRefs
and key != CveKey.CveVulnsSoft
and key != CveKey.CvePublishedDate
and key != CveKey.CveCategories
and key != CveKey.CveModifiedDate):
cve_data[key] = unicode(cve_data[key])
cve_data_list.append(cve_data)
entry.clear()
while entry.getprevious() is not None:
del entry.getparent()[0]
del entry
insert_into_cve_collection(cve_data_list)
del cve_data_list
del cve_data
del parser
gc.collect()
def parse_cve_and_udpatedb(download_latest_nvd=True, nvd_file=NVD_MODIFIED_FILE):
if download_latest_nvd:
start_nvd_xml_download()
parser = NvdParser()
cve_data_list = []
cve_data = {}
for event, entry in etree.iterparse(nvd_file, events=['start', 'end']):
if entry.tag == NVD_FEEDS_ENTRY and event == 'start':
cve_data = parser.get_entry_info(entry)
if entry.tag == NVD_FEEDS_DESC and event == 'start':
cve_data[CveKey.CveDescriptions] = parser.get_descriptions(entry)
if entry.tag == NVD_FEEDS_REFS and event == 'start':
cve_data[CveKey.CveRefs] = parser.get_refs(entry)
if entry.tag == NVD_FEEDS_VULN_SOFT and event == 'start':
cve_data[CveKey.CveVulnsSoft] = parser.get_vulns_soft(entry)
cve_data[CveKey.CveCategories] = []
if entry.tag == NVD_FEEDS_ENTRY and event == 'end':
for key in cve_data.keys():
if (key != CveKey.CveDescriptions and
key != CveKey.CveRefs and
key != CveKey.CveVulnsSoft and
key != CveKey.CvePublishedDate and
key != CveKey.CveCategories and
key != CveKey.CveModifiedDate):
cve_data[key] = unicode(cve_data[key])
cve_data_list.append(cve_data)
entry.clear()
while entry.getprevious() is not None:
del entry.getparent()[0]
del entry
insert_into_cve_collection(cve_data_list)
del cve_data_list
del cve_data
del parser
gc.collect()
