def execute(self):
old = flatten_vendors(convert_cpes(
self.cve_obj.json["configurations"]))
new = flatten_vendors(convert_cpes(self.cve_json["configurations"]))
payload = list(set(new) - set(old))
if payload:
event = CveUtil.create_event(self.cve_obj, self.cve_json,
"first_time", payload)
return event
return None
def test_convert_conf_multiple_vendors(open_file):
conf = open_file("configurations/multiple_vendors.json")
vendors = convert_cpes(conf)
vendors["foo"] = sorted(vendors["foo"])
assert len(vendors) == 2
assert vendors["foo"] == ["bar", "baz"]
assert vendors["bar"] == ["baz"]
def cve(cve_id):
cve = CveController.get({"cve_id": cve_id})
vendors = convert_cpes(cve.json["configurations"])
cwes = get_cwes_details(
cve.json["cve"]["problemtype"]["problemtype_data"][0]["description"])
# Get the user tags
user_tags = []
if current_user.is_authenticated:
user_tags = UserTagController.list_items({"user_id": current_user.id})
# We have to pass an encoded list of tags for the modal box
cve_tags_encoded = json.dumps([t.name for t in cve.tags])
events = Event.query.filter_by(cve_id=cve.id).order_by(
Event.created_at.desc())
events_by_time = [(time, list(evs)) for time, evs in (
itertools.groupby(events, operator.attrgetter("created_at")))]
return render_template(
"cve.html",
cve=cve,
cve_dumped=json.dumps(cve.json),
vendors=vendors,
cwes=cwes,
user_tags=user_tags,
cve_tags_encoded=cve_tags_encoded,
events_by_time=events_by_time,
)
def execute(self):
old = nested_lookup("cpe23Uri", self.cve_obj.json["configurations"])
new = nested_lookup("cpe23Uri", self.cve_json["configurations"])
payload = {
"added": list(set(new) - set(old)),
"removed": list(set(old) - set(new)),
}
# The CPEs list has been modified
if payload["added"] or payload["removed"]:
# Change the CVE's vendors attribute
self.cve_obj.vendors = flatten_vendors(
convert_cpes(self.cve_json["configurations"])
)
db.session.commit()
# Create the vendors and products objects if they don't exist
vendors_products = convert_cpes(payload["added"])
for vendor, products in vendors_products.items():
v_obj = Vendor.query.filter_by(name=vendor).first()
# Create the vendor and associate it to the CVE
if not v_obj:
v_obj = Vendor(name=vendor)
db.session.add(v_obj)
db.session.commit()
# Do the same for its products
for product in products:
p_obj = Product.query.filter_by(name=product, vendor=v_obj).first()
if not p_obj:
p_obj = Product(name=product, vendor=v_obj)
db.session.add(p_obj)
db.session.commit()
# Create the event
event = CveUtil.create_event(self.cve_obj, self.cve_json, "cpes", payload)
return event
return None
def cve(cve_id):
cve = CveController.get({"cve_id": cve_id})
vendors = convert_cpes(cve.json["configurations"])
cwes = get_cwes_details(
cve.json["cve"]["problemtype"]["problemtype_data"][0]["description"])
return render_template("cve.html",
cve=cve,
cve_dumped=json.dumps(cve.json),
vendors=vendors,
cwes=cwes)
def cve(cve_id):
q = Cve.query
# Search the CVE
cve = q.filter_by(cve_id=cve_id).first()
if not cve:
return redirect(url_for("main.cves"))
# Nested dict of vendors and their products
vendors = convert_cpes(cve.json["configurations"])
return render_template(
"cve.html", cve=cve, cve_dumped=json.dumps(cve.json), vendors=vendors
)
def cve(cve_id):
q = Cve.query
# Search the CVE
cve = q.filter_by(cve_id=cve_id).first()
if not cve:
return redirect(url_for("main.cves"))
# Nested dict of vendors and their products
vendors = convert_cpes(cve.json["configurations"])
cwes = get_cwes_details(
cve.json["cve"]["problemtype"]["problemtype_data"][0]["description"])
return render_template("cve.html",
cve=cve,
cve_dumped=json.dumps(cve.json),
vendors=vendors,
cwes=cwes)
def test_check_cpes(create_cve, handle_events, open_file):
cve = create_cve("CVE-2018-18074")
assert sorted(cve.vendors) == sorted([
"python-requests",
f"python-requests{PRODUCT_SEPARATOR}requests",
"canonical",
f"canonical{PRODUCT_SEPARATOR}ubuntu_linux",
])
# The CVE creation already created vendors and products
vendors = Vendor.query.all()
assert len(vendors) == 2
assert sorted([v.name for v in vendors
]) == sorted(["python-requests", "canonical"])
products = Product.query.all()
assert len(products) == 2
assert sorted([p.name
for p in products]) == sorted(["requests", "ubuntu_linux"])
# 1 CPE added and 1 CPE removed (including 1 new vendor and product)
handle_events("modified_cves/CVE-2018-18074_cpes.json")
cve = Cve.query.filter_by(cve_id="CVE-2018-18074").first()
assert sorted(cve.vendors) == sorted([
"canonical",
f"canonical{PRODUCT_SEPARATOR}ubuntu_linux",
"opencveio",
f"opencveio{PRODUCT_SEPARATOR}opencve",
])
# 1 new vendor
vendors = Vendor.query.all()
assert len(vendors) == 3
assert sorted([v.name for v in vendors
]) == sorted(["python-requests", "canonical", "opencveio"])
# 1 new product
products = Product.query.all()
assert len(products) == 3
assert sorted([p.name for p in products
]) == sorted(["requests", "ubuntu_linux", "opencve"])
# Task has been created
tasks = Task.query.all()
assert len(tasks) == 1
task = tasks[0]
# Change has been created
changes = Change.query.all()
assert len(changes) == 1
change = changes[0]
assert change.task.id == task.id
assert convert_cpes(change.json["configurations"]) == {
"canonical": ["ubuntu_linux"],
"opencveio": ["opencve"],
}
# Event has been created
event = Event.query.filter_by(type="cpes").first()
assert event.type == "cpes"
assert event.details == {
"added": ["cpe:2.3:a:opencveio:opencve:*:*:*:*:*:*:*:*"],
"removed": ["cpe:2.3:a:python-requests:requests:*:*:*:*:*:*:*:*"],
}
assert event.review == False
assert event.cve.cve_id == "CVE-2018-18074"
assert event.change.id == change.id
def run():
"""
Import the CVE list.
"""
mappings = {"vendors": {}, "products": {}}
from opencve.commands.imports import CURRENT_YEAR, CVE_FIRST_YEAR
# Create the initial task
task = Task()
db.session.add(task)
db.session.commit()
task_id = task.id
for year in range(CVE_FIRST_YEAR, CURRENT_YEAR + 1):
header("Importing CVE for {}".format(year))
mappings.update({"cves": [], "changes": []})
# Download the file
url = NVD_CVE_URL.format(year=year)
with timed_operation("Downloading {}...".format(url)):
resp = requests.get(url).content
# Parse the XML elements
with timed_operation("Parsing JSON elements..."):
raw = gzip.GzipFile(fileobj=BytesIO(resp)).read()
del resp
items = json.loads(raw.decode("utf-8"))["CVE_Items"]
del raw
with timed_operation("Creating model objects..."):
for item in items:
cve_db_id = get_uuid()
summary = item["cve"]["description"]["description_data"][0]["value"]
cvss2 = (
item["impact"]["baseMetricV2"]["cvssV2"]["baseScore"]
if "baseMetricV2" in item["impact"]
else None
)
cvss3 = (
item["impact"]["baseMetricV3"]["cvssV3"]["baseScore"]
if "baseMetricV3" in item["impact"]
else None
)
# Construct CWE and CPE lists
cwes = get_cwes(
item["cve"]["problemtype"]["problemtype_data"][0]["description"]
)
cpes = convert_cpes(item["configurations"])
vendors = flatten_vendors(cpes)
# Create the CVEs mappings
mappings["cves"].append(
dict(
id=cve_db_id,
cve_id=item["cve"]["CVE_data_meta"]["ID"],
summary=summary,
json=item,
vendors=vendors,
cwes=cwes,
cvss2=cvss2,
cvss3=cvss3,
created_at=arrow.get(item["publishedDate"]).datetime,
updated_at=arrow.get(item["lastModifiedDate"]).datetime,
)
)
# Create the vendors and their products
for vendor, products in cpes.items():
# Create the vendor
if vendor not in mappings["vendors"].keys():
mappings["vendors"][vendor] = dict(id=get_uuid(), name=vendor)
for product in products:
if get_slug(vendor, product) not in mappings["products"].keys():
mappings["products"][get_slug(vendor, product)] = dict(
id=get_uuid(),
name=product,
vendor_id=mappings["vendors"][vendor]["id"],
)
# Insert the objects in database
with timed_operation("Inserting CVE..."):
db.session.bulk_insert_mappings(Cve, mappings["cves"])
db.session.commit()
# Create the changes based on CVEs data
for cve in mappings["cves"]:
mappings["changes"].append(
dict(
id=get_uuid(),
created_at=cve["created_at"],
updated_at=cve["updated_at"],
json=cve["json"],
cve_id=cve["id"],
task_id=task_id,
)
)
db.session.bulk_insert_mappings(Change, mappings["changes"])
db.session.commit()
info("{} CVE imported.".format(len(mappings["cves"])))
# Free the memory after each processed year
del mappings["cves"]
del mappings["changes"]
return mappings
def test_convert_conf_nested(open_file):
conf = open_file("configurations/nested.json")
vendors = convert_cpes(conf)
vendors["foo"] = sorted(vendors["foo"])
assert vendors == {"foo": ["bar", "baz"], "bar": ["baz"]}
def test_convert_conf_multiple_products(open_file):
conf = open_file("configurations/multiple_products.json")
vendors = convert_cpes(conf)
vendors["foo"] = sorted(vendors["foo"])
assert vendors == {"foo": ["bar", "baz"]}
def test_convert_simple_conf(open_file):
conf = open_file("configurations/simple.json")
assert convert_cpes(conf) == {"foo": ["bar"]}
def test_convert_empty_conf():
assert convert_cpes({}) == {}
def format(self, json):
return convert_cpes(json["configurations"])
def create_cve(cls, cve_json):
cvss2 = (cve_json["impact"]["baseMetricV2"]["cvssV2"]["baseScore"]
if "baseMetricV2" in cve_json["impact"] else None)
cvss3 = (cve_json["impact"]["baseMetricV3"]["cvssV3"]["baseScore"]
if "baseMetricV3" in cve_json["impact"] else None)
# Construct CWE and CPE lists
cwes = get_cwes(cve_json["cve"]["problemtype"]["problemtype_data"][0]
["description"])
cpes = convert_cpes(cve_json["configurations"])
vendors = flatten_vendors(cpes)
# Create the CVE
cve = Cve(
cve_id=cve_json["cve"]["CVE_data_meta"]["ID"],
summary=cve_json["cve"]["description"]["description_data"][0]
["value"],
json=cve_json,
vendors=vendors,
cwes=cwes,
cvss2=cvss2,
cvss3=cvss3,
created_at=arrow.get(cve_json["publishedDate"]).datetime,
updated_at=arrow.get(cve_json["lastModifiedDate"]).datetime,
)
db.session.add(cve)
db.session.commit()
# Add the CWE that not exists yet in database
for cwe in cwes:
cwe_obj = Cwe.query.filter_by(cwe_id=cwe).first()
if not cwe_obj:
info(
f"{cwe} detected in {cve.cve_id} but not existing in database, adding it..."
)
cwe_obj = Cwe(cwe_id=cwe)
db.session.add(cwe_obj)
db.session.commit()
# Add the CPEs
vendors_products = convert_cpes(
nested_lookup("cpe23Uri", cve_json["configurations"]))
for vendor, products in vendors_products.items():
v_obj = Vendor.query.filter_by(name=vendor).first()
# Create the vendor
if not v_obj:
v_obj = Vendor(name=vendor)
db.session.add(v_obj)
db.session.commit()
# Create the products
for product in products:
p_obj = Product.query.filter_by(name=product,
vendor=v_obj).first()
if not p_obj:
p_obj = Product(name=product, vendor=v_obj)
db.session.add(p_obj)
db.session.commit()
return cve
