def import_data(confirm):
"""
Perform initial imports (cve, cwe, cpe).
"""
if Cve.query.first():
info("Import already done.")
return
if not confirm:
msg = ("This command will import initial data in your database. "
"Do you want to continue ?".format(CVE_FIRST_YEAR,
CURRENT_YEAR))
if not click.confirm(msg):
info("Bye.")
return
# Import the CWE list
cwe.run()
# Import the CVE, then use the returned list of vendors
# to merge them with the official CPE dictionnary list.
vendors = cve.run()
cpe.run(vendors)
# Populate the metas table
with timed_operation("Populating metas table..."):
meta = Meta(name="nvd_last_sha256", value="default")
db.session.add(meta)
db.session.commit()
def run(mappings):
"""
Import the Vendors and Products list.
"""
header("Importing CPE list...")
# Download the XML file
with timed_operation("Downloading {}...".format(NVD_CPE_URL)):
resp = requests.get(NVD_CPE_URL).content
# Parse the XML elements
with timed_operation("Parsing XML elements..."):
raw = gzip.GzipFile(fileobj=BytesIO(resp)).read()
obj = untangle.parse(raw.decode("utf-8"))
items = obj.cpe_list.cpe_item
del obj
# Create the objects
with timed_operation("Creating list of mappings..."):
for item in items:
obj = CPE(item.cpe_23_cpe23_item["name"])
vendor = obj.get_vendor()[0]
product = obj.get_product()[0]
if vendor not in mappings["vendors"].keys():
mappings["vendors"][vendor] = dict(id=get_uuid(), name=vendor)
if get_slug(vendor, product) not in mappings["products"].keys():
mappings["products"][get_slug(vendor, product)] = dict(
id=get_uuid(),
name=product,
vendor_id=mappings["vendors"][vendor]["id"],
)
del items
# Insert the objects in database
with timed_operation("Inserting Vendors and Products..."):
db.session.bulk_insert_mappings(Vendor, mappings["vendors"].values())
db.session.bulk_insert_mappings(Product, mappings["products"].values())
db.session.commit()
info(
"{} vendors and {} products imported.".format(
len(mappings["vendors"]), len(mappings["products"])
)
)
del mappings
def run():
"""
Import the CWE list.
"""
header("Importing CWE list...")
# Download the file
with timed_operation("Downloading {}...".format(MITRE_CWE_URL)):
resp = requests.get(MITRE_CWE_URL).content
# Parse weaknesses
with timed_operation("Parsing cwes..."):
z = ZipFile(BytesIO(resp))
raw = z.open(z.namelist()[0]).read()
obj = untangle.parse(raw.decode("utf-8"))
weaknesses = obj.Weakness_Catalog.Weaknesses.Weakness
categories = obj.Weakness_Catalog.Categories.Category
# Create the objects
cwes = {}
with timed_operation("Creating mappings..."):
for c in weaknesses + categories:
cwes[c["ID"]] = dict(
id=get_uuid(),
cwe_id=f"CWE-{c['ID']}",
name=c["Name"],
description=c.Description.cdata
if hasattr(c, "Description") else c.Summary.cdata,
)
# Insert the objects in database
with timed_operation("Inserting CWE..."):
db.session.bulk_insert_mappings(Cwe, cwes.values())
db.session.commit()
info("{} CWE imported.".format(len(cwes)))
del cwes
def run():
"""
Import the CVE list.
"""
mappings = {"vendors": {}, "products": {}}
from opencve.commands.imports import CURRENT_YEAR, CVE_FIRST_YEAR
# Create the initial task
task = Task()
db.session.add(task)
db.session.commit()
task_id = task.id
for year in range(CVE_FIRST_YEAR, CURRENT_YEAR + 1):
header("Importing CVE for {}".format(year))
mappings.update({"cves": [], "changes": []})
# Download the file
url = NVD_CVE_URL.format(year=year)
with timed_operation("Downloading {}...".format(url)):
resp = requests.get(url).content
# Parse the XML elements
with timed_operation("Parsing JSON elements..."):
raw = gzip.GzipFile(fileobj=BytesIO(resp)).read()
del resp
items = json.loads(raw.decode("utf-8"))["CVE_Items"]
del raw
with timed_operation("Creating model objects..."):
for item in items:
cve_db_id = get_uuid()
summary = item["cve"]["description"]["description_data"][0]["value"]
cvss2 = (
item["impact"]["baseMetricV2"]["cvssV2"]["baseScore"]
if "baseMetricV2" in item["impact"]
else None
)
cvss3 = (
item["impact"]["baseMetricV3"]["cvssV3"]["baseScore"]
if "baseMetricV3" in item["impact"]
else None
)
# Construct CWE and CPE lists
cwes = get_cwes(
item["cve"]["problemtype"]["problemtype_data"][0]["description"]
)
cpes = convert_cpes(item["configurations"])
vendors = flatten_vendors(cpes)
# Create the CVEs mappings
mappings["cves"].append(
dict(
id=cve_db_id,
cve_id=item["cve"]["CVE_data_meta"]["ID"],
summary=summary,
json=item,
vendors=vendors,
cwes=cwes,
cvss2=cvss2,
cvss3=cvss3,
created_at=arrow.get(item["publishedDate"]).datetime,
updated_at=arrow.get(item["lastModifiedDate"]).datetime,
)
)
# Create the vendors and their products
for vendor, products in cpes.items():
# Create the vendor
if vendor not in mappings["vendors"].keys():
mappings["vendors"][vendor] = dict(id=get_uuid(), name=vendor)
for product in products:
if get_slug(vendor, product) not in mappings["products"].keys():
mappings["products"][get_slug(vendor, product)] = dict(
id=get_uuid(),
name=product,
vendor_id=mappings["vendors"][vendor]["id"],
)
# Insert the objects in database
with timed_operation("Inserting CVE..."):
db.session.bulk_insert_mappings(Cve, mappings["cves"])
db.session.commit()
# Create the changes based on CVEs data
for cve in mappings["cves"]:
mappings["changes"].append(
dict(
id=get_uuid(),
created_at=cve["created_at"],
updated_at=cve["updated_at"],
json=cve["json"],
cve_id=cve["id"],
task_id=task_id,
)
)
db.session.bulk_insert_mappings(Change, mappings["changes"])
db.session.commit()
info("{} CVE imported.".format(len(mappings["cves"])))
# Free the memory after each processed year
del mappings["cves"]
del mappings["changes"]
return mappings