def parse_cve_and_udpatedb(
download_latest_nvd=True,
nvd_files=[
CVEDataDir.NVD_RECENT_FILE_UNCOMPRESSED[:-3],
CVEDataDir.NVD_MODIFIED_FILE_UNCOMPRESSED[:-3],
CVEDataDir.NVD_CURRENT_FILE[:-3],
],
):
"""This begins the actual parsing of the xml files and loads up the data
into the database
Kwargs:
download_latest_nvd (bool): Whether or not to download
the latest nvd data
nvd_files (list): This is a list of the files you want to parse
"""
if download_latest_nvd:
start_nvd_xml_download()
parser = NvdParser()
for nvd_file in nvd_files:
cve_data_list = []
cve_data = {}
for event, entry in etree.iterparse(nvd_file, events=["start", "end"]):
if entry.tag == NVDFeeds.ENTRY and event == "start":
cve_data = parser.get_entry_info(entry)
if entry.tag == NVDFeeds.DESC and event == "start":
cve_data[CveKey.CveDescriptions] = parser.get_descriptions(entry)
if entry.tag == NVDFeeds.REFS and event == "start":
cve_data[CveKey.CveRefs] = parser.get_refs(entry)
# if entry.tag == NVDFeeds.VULN_SOFT and event == 'start':
# cve_data[CveKey.CveVulnsSoft] = parser.get_vulns_soft(entry)
cve_data[CveKey.CveCategories] = []
if entry.tag == NVDFeeds.ENTRY and event == "end":
for key in cve_data.keys():
if (
key != CveKey.CveDescriptions
and key != CveKey.CveRefs
and key != CveKey.CveVulnsSoft
and key != CveKey.CvePublishedDate
and key != CveKey.CveCategories
and key != CveKey.CvssVector
and key != CveKey.CveModifiedDate
):
cve_data[key] = unicode(cve_data[key])
cve_data_list.append(cve_data)
# entry.clear()
# while entry.getprevious() is not None:
# del entry.getparent()[0]
# del entry
insert_cve_data(cve_data_list)
del cve_data_list
del cve_data
gc.collect()
def parse_cve_and_udpatedb(download_latest_nvd=True,
nvd_files=[
CVEDataDir.NVD_RECENT_FILE_UNCOMPRESSED[:-3],
CVEDataDir.NVD_MODIFIED_FILE_UNCOMPRESSED[:-3],
CVEDataDir.NVD_CURRENT_FILE[:-3]
]):
"""This begins the actual parsing of the xml files and loads up the data
into the database
Kwargs:
download_latest_nvd (bool): Whether or not to download
the latest nvd data
nvd_files (list): This is a list of the files you want to parse
"""
if download_latest_nvd:
start_nvd_xml_download()
parser = NvdParser()
for nvd_file in nvd_files:
cve_data_list = []
cve_data = {}
for event, entry in etree.iterparse(nvd_file, events=['start', 'end']):
if entry.tag == NVDFeeds.ENTRY and event == 'start':
cve_data = parser.get_entry_info(entry)
if entry.tag == NVDFeeds.DESC and event == 'start':
cve_data[CveKey.CveDescriptions] = \
parser.get_descriptions(entry)
if entry.tag == NVDFeeds.REFS and event == 'start':
cve_data[CveKey.CveRefs] = parser.get_refs(entry)
#if entry.tag == NVDFeeds.VULN_SOFT and event == 'start':
# cve_data[CveKey.CveVulnsSoft] = parser.get_vulns_soft(entry)
cve_data[CveKey.CveCategories] = []
if entry.tag == NVDFeeds.ENTRY and event == 'end':
for key in cve_data.keys():
if (key != CveKey.CveDescriptions and key != CveKey.CveRefs
and key != CveKey.CveVulnsSoft
and key != CveKey.CvePublishedDate
and key != CveKey.CveCategories
and key != CveKey.CvssVector
and key != CveKey.CveModifiedDate):
cve_data[key] = unicode(cve_data[key])
cve_data_list.append(cve_data)
#entry.clear()
#while entry.getprevious() is not None:
# del entry.getparent()[0]
#del entry
insert_cve_data(cve_data_list)
del cve_data_list
del cve_data
gc.collect()
def load_up_all_xml_into_db():
nvd_files = []
if not os.path.exists(CVEDataDir.XML_DIR):
os.makedirs(CVEDataDir.XML_DIR)
xml_exists = os.listdir(CVEDataDir.XML_DIR)
logger.info('starting cve/nvd update process')
if not xml_exists:
logger.info('downloading nvd/cve xml data files')
start_nvd_xml_download()
for directory, _, files in os.walk(CVEDataDir.XML_DIR):
for xml_file in files:
nvd_file = os.path.join(directory, xml_file)
nvd_files.append(nvd_file)
parse_cve_and_udpatedb(False, nvd_files)
update_cve_categories()
logger.info('finished cve/nvd update process')
gc.collect()
def load_up_all_xml_into_db():
nvd_files = []
if not os.path.exists(CVEDataDir.XML_DIR):
os.makedirs(CVEDataDir.XML_DIR)
xml_exists = os.listdir(CVEDataDir.XML_DIR)
logger.info('starting cve/nvd update process')
if not xml_exists:
logger.info('downloading nvd/cve xml data files')
start_nvd_xml_download()
for directory, _, files in os.walk(CVEDataDir.XML_DIR):
for xml_file in files:
nvd_file = os.path.join(directory, xml_file)
nvd_files.append(nvd_file)
parse_cve_and_udpatedb(False, nvd_files)
update_cve_categories()
logger.info('finished cve/nvd update process')
gc.collect()
