def execute(self):
old = flatten_vendors(convert_cpes(
self.cve_obj.json["configurations"]))
new = flatten_vendors(convert_cpes(self.cve_json["configurations"]))
payload = list(set(new) - set(old))
if payload:
event = CveUtil.create_event(self.cve_obj, self.cve_json,
"first_time", payload)
return event
return None
def test_flatten_multiple_vendors():
assert flatten_vendors({"foo": ["bar", "baz"], "bar": ["baz"]}) == [
"foo",
f"foo{PRODUCT_SEPARATOR}bar",
f"foo{PRODUCT_SEPARATOR}baz",
"bar",
f"bar{PRODUCT_SEPARATOR}baz",
]
def execute(self):
old = nested_lookup("cpe23Uri", self.cve_obj.json["configurations"])
new = nested_lookup("cpe23Uri", self.cve_json["configurations"])
payload = {
"added": list(set(new) - set(old)),
"removed": list(set(old) - set(new)),
}
# The CPEs list has been modified
if payload["added"] or payload["removed"]:
# Change the CVE's vendors attribute
self.cve_obj.vendors = flatten_vendors(
convert_cpes(self.cve_json["configurations"])
)
db.session.commit()
# Create the vendors and products objects if they don't exist
vendors_products = convert_cpes(payload["added"])
for vendor, products in vendors_products.items():
v_obj = Vendor.query.filter_by(name=vendor).first()
# Create the vendor and associate it to the CVE
if not v_obj:
v_obj = Vendor(name=vendor)
db.session.add(v_obj)
db.session.commit()
# Do the same for its products
for product in products:
p_obj = Product.query.filter_by(name=product, vendor=v_obj).first()
if not p_obj:
p_obj = Product(name=product, vendor=v_obj)
db.session.add(p_obj)
db.session.commit()
# Create the event
event = CveUtil.create_event(self.cve_obj, self.cve_json, "cpes", payload)
return event
return None
def run():
"""
Import the CVE list.
"""
mappings = {"vendors": {}, "products": {}}
from opencve.commands.imports import CURRENT_YEAR, CVE_FIRST_YEAR
# Create the initial task
task = Task()
db.session.add(task)
db.session.commit()
task_id = task.id
for year in range(CVE_FIRST_YEAR, CURRENT_YEAR + 1):
header("Importing CVE for {}".format(year))
mappings.update({"cves": [], "changes": []})
# Download the file
url = NVD_CVE_URL.format(year=year)
with timed_operation("Downloading {}...".format(url)):
resp = requests.get(url).content
# Parse the XML elements
with timed_operation("Parsing JSON elements..."):
raw = gzip.GzipFile(fileobj=BytesIO(resp)).read()
del resp
items = json.loads(raw.decode("utf-8"))["CVE_Items"]
del raw
with timed_operation("Creating model objects..."):
for item in items:
cve_db_id = get_uuid()
summary = item["cve"]["description"]["description_data"][0]["value"]
cvss2 = (
item["impact"]["baseMetricV2"]["cvssV2"]["baseScore"]
if "baseMetricV2" in item["impact"]
else None
)
cvss3 = (
item["impact"]["baseMetricV3"]["cvssV3"]["baseScore"]
if "baseMetricV3" in item["impact"]
else None
)
# Construct CWE and CPE lists
cwes = get_cwes(
item["cve"]["problemtype"]["problemtype_data"][0]["description"]
)
cpes = convert_cpes(item["configurations"])
vendors = flatten_vendors(cpes)
# Create the CVEs mappings
mappings["cves"].append(
dict(
id=cve_db_id,
cve_id=item["cve"]["CVE_data_meta"]["ID"],
summary=summary,
json=item,
vendors=vendors,
cwes=cwes,
cvss2=cvss2,
cvss3=cvss3,
created_at=arrow.get(item["publishedDate"]).datetime,
updated_at=arrow.get(item["lastModifiedDate"]).datetime,
)
)
# Create the vendors and their products
for vendor, products in cpes.items():
# Create the vendor
if vendor not in mappings["vendors"].keys():
mappings["vendors"][vendor] = dict(id=get_uuid(), name=vendor)
for product in products:
if get_slug(vendor, product) not in mappings["products"].keys():
mappings["products"][get_slug(vendor, product)] = dict(
id=get_uuid(),
name=product,
vendor_id=mappings["vendors"][vendor]["id"],
)
# Insert the objects in database
with timed_operation("Inserting CVE..."):
db.session.bulk_insert_mappings(Cve, mappings["cves"])
db.session.commit()
# Create the changes based on CVEs data
for cve in mappings["cves"]:
mappings["changes"].append(
dict(
id=get_uuid(),
created_at=cve["created_at"],
updated_at=cve["updated_at"],
json=cve["json"],
cve_id=cve["id"],
task_id=task_id,
)
)
db.session.bulk_insert_mappings(Change, mappings["changes"])
db.session.commit()
info("{} CVE imported.".format(len(mappings["cves"])))
# Free the memory after each processed year
del mappings["cves"]
del mappings["changes"]
return mappings
def test_flatten_simple_vendor():
assert flatten_vendors({"foo":
["bar"]}) == ["foo", f"foo{PRODUCT_SEPARATOR}bar"]
def test_flatten_empty_vendors():
assert flatten_vendors({}) == []
def create_cve(cls, cve_json):
cvss2 = (cve_json["impact"]["baseMetricV2"]["cvssV2"]["baseScore"]
if "baseMetricV2" in cve_json["impact"] else None)
cvss3 = (cve_json["impact"]["baseMetricV3"]["cvssV3"]["baseScore"]
if "baseMetricV3" in cve_json["impact"] else None)
# Construct CWE and CPE lists
cwes = get_cwes(cve_json["cve"]["problemtype"]["problemtype_data"][0]
["description"])
cpes = convert_cpes(cve_json["configurations"])
vendors = flatten_vendors(cpes)
# Create the CVE
cve = Cve(
cve_id=cve_json["cve"]["CVE_data_meta"]["ID"],
summary=cve_json["cve"]["description"]["description_data"][0]
["value"],
json=cve_json,
vendors=vendors,
cwes=cwes,
cvss2=cvss2,
cvss3=cvss3,
created_at=arrow.get(cve_json["publishedDate"]).datetime,
updated_at=arrow.get(cve_json["lastModifiedDate"]).datetime,
)
db.session.add(cve)
db.session.commit()
# Add the CWE that not exists yet in database
for cwe in cwes:
cwe_obj = Cwe.query.filter_by(cwe_id=cwe).first()
if not cwe_obj:
info(
f"{cwe} detected in {cve.cve_id} but not existing in database, adding it..."
)
cwe_obj = Cwe(cwe_id=cwe)
db.session.add(cwe_obj)
db.session.commit()
# Add the CPEs
vendors_products = convert_cpes(
nested_lookup("cpe23Uri", cve_json["configurations"]))
for vendor, products in vendors_products.items():
v_obj = Vendor.query.filter_by(name=vendor).first()
# Create the vendor
if not v_obj:
v_obj = Vendor(name=vendor)
db.session.add(v_obj)
db.session.commit()
# Create the products
for product in products:
p_obj = Product.query.filter_by(name=product,
vendor=v_obj).first()
if not p_obj:
p_obj = Product(name=product, vendor=v_obj)
db.session.add(p_obj)
db.session.commit()
return cve