def execute(self): old = flatten_vendors(convert_cpes( self.cve_obj.json["configurations"])) new = flatten_vendors(convert_cpes(self.cve_json["configurations"])) payload = list(set(new) - set(old)) if payload: event = CveUtil.create_event(self.cve_obj, self.cve_json, "first_time", payload) return event return None
def test_convert_conf_multiple_vendors(open_file): conf = open_file("configurations/multiple_vendors.json") vendors = convert_cpes(conf) vendors["foo"] = sorted(vendors["foo"]) assert len(vendors) == 2 assert vendors["foo"] == ["bar", "baz"] assert vendors["bar"] == ["baz"]
def cve(cve_id): cve = CveController.get({"cve_id": cve_id}) vendors = convert_cpes(cve.json["configurations"]) cwes = get_cwes_details( cve.json["cve"]["problemtype"]["problemtype_data"][0]["description"]) # Get the user tags user_tags = [] if current_user.is_authenticated: user_tags = UserTagController.list_items({"user_id": current_user.id}) # We have to pass an encoded list of tags for the modal box cve_tags_encoded = json.dumps([t.name for t in cve.tags]) events = Event.query.filter_by(cve_id=cve.id).order_by( Event.created_at.desc()) events_by_time = [(time, list(evs)) for time, evs in ( itertools.groupby(events, operator.attrgetter("created_at")))] return render_template( "cve.html", cve=cve, cve_dumped=json.dumps(cve.json), vendors=vendors, cwes=cwes, user_tags=user_tags, cve_tags_encoded=cve_tags_encoded, events_by_time=events_by_time, )
def execute(self): old = nested_lookup("cpe23Uri", self.cve_obj.json["configurations"]) new = nested_lookup("cpe23Uri", self.cve_json["configurations"]) payload = { "added": list(set(new) - set(old)), "removed": list(set(old) - set(new)), } # The CPEs list has been modified if payload["added"] or payload["removed"]: # Change the CVE's vendors attribute self.cve_obj.vendors = flatten_vendors( convert_cpes(self.cve_json["configurations"]) ) db.session.commit() # Create the vendors and products objects if they don't exist vendors_products = convert_cpes(payload["added"]) for vendor, products in vendors_products.items(): v_obj = Vendor.query.filter_by(name=vendor).first() # Create the vendor and associate it to the CVE if not v_obj: v_obj = Vendor(name=vendor) db.session.add(v_obj) db.session.commit() # Do the same for its products for product in products: p_obj = Product.query.filter_by(name=product, vendor=v_obj).first() if not p_obj: p_obj = Product(name=product, vendor=v_obj) db.session.add(p_obj) db.session.commit() # Create the event event = CveUtil.create_event(self.cve_obj, self.cve_json, "cpes", payload) return event return None
def cve(cve_id): cve = CveController.get({"cve_id": cve_id}) vendors = convert_cpes(cve.json["configurations"]) cwes = get_cwes_details( cve.json["cve"]["problemtype"]["problemtype_data"][0]["description"]) return render_template("cve.html", cve=cve, cve_dumped=json.dumps(cve.json), vendors=vendors, cwes=cwes)
def cve(cve_id): q = Cve.query # Search the CVE cve = q.filter_by(cve_id=cve_id).first() if not cve: return redirect(url_for("main.cves")) # Nested dict of vendors and their products vendors = convert_cpes(cve.json["configurations"]) return render_template( "cve.html", cve=cve, cve_dumped=json.dumps(cve.json), vendors=vendors )
def cve(cve_id): q = Cve.query # Search the CVE cve = q.filter_by(cve_id=cve_id).first() if not cve: return redirect(url_for("main.cves")) # Nested dict of vendors and their products vendors = convert_cpes(cve.json["configurations"]) cwes = get_cwes_details( cve.json["cve"]["problemtype"]["problemtype_data"][0]["description"]) return render_template("cve.html", cve=cve, cve_dumped=json.dumps(cve.json), vendors=vendors, cwes=cwes)
def test_check_cpes(create_cve, handle_events, open_file): cve = create_cve("CVE-2018-18074") assert sorted(cve.vendors) == sorted([ "python-requests", f"python-requests{PRODUCT_SEPARATOR}requests", "canonical", f"canonical{PRODUCT_SEPARATOR}ubuntu_linux", ]) # The CVE creation already created vendors and products vendors = Vendor.query.all() assert len(vendors) == 2 assert sorted([v.name for v in vendors ]) == sorted(["python-requests", "canonical"]) products = Product.query.all() assert len(products) == 2 assert sorted([p.name for p in products]) == sorted(["requests", "ubuntu_linux"]) # 1 CPE added and 1 CPE removed (including 1 new vendor and product) handle_events("modified_cves/CVE-2018-18074_cpes.json") cve = Cve.query.filter_by(cve_id="CVE-2018-18074").first() assert sorted(cve.vendors) == sorted([ "canonical", f"canonical{PRODUCT_SEPARATOR}ubuntu_linux", "opencveio", f"opencveio{PRODUCT_SEPARATOR}opencve", ]) # 1 new vendor vendors = Vendor.query.all() assert len(vendors) == 3 assert sorted([v.name for v in vendors ]) == sorted(["python-requests", "canonical", "opencveio"]) # 1 new product products = Product.query.all() assert len(products) == 3 assert sorted([p.name for p in products ]) == sorted(["requests", "ubuntu_linux", "opencve"]) # Task has been created tasks = Task.query.all() assert len(tasks) == 1 task = tasks[0] # Change has been created changes = Change.query.all() assert len(changes) == 1 change = changes[0] assert change.task.id == task.id assert convert_cpes(change.json["configurations"]) == { "canonical": ["ubuntu_linux"], "opencveio": ["opencve"], } # Event has been created event = Event.query.filter_by(type="cpes").first() assert event.type == "cpes" assert event.details == { "added": ["cpe:2.3:a:opencveio:opencve:*:*:*:*:*:*:*:*"], "removed": ["cpe:2.3:a:python-requests:requests:*:*:*:*:*:*:*:*"], } assert event.review == False assert event.cve.cve_id == "CVE-2018-18074" assert event.change.id == change.id
def run(): """ Import the CVE list. """ mappings = {"vendors": {}, "products": {}} from opencve.commands.imports import CURRENT_YEAR, CVE_FIRST_YEAR # Create the initial task task = Task() db.session.add(task) db.session.commit() task_id = task.id for year in range(CVE_FIRST_YEAR, CURRENT_YEAR + 1): header("Importing CVE for {}".format(year)) mappings.update({"cves": [], "changes": []}) # Download the file url = NVD_CVE_URL.format(year=year) with timed_operation("Downloading {}...".format(url)): resp = requests.get(url).content # Parse the XML elements with timed_operation("Parsing JSON elements..."): raw = gzip.GzipFile(fileobj=BytesIO(resp)).read() del resp items = json.loads(raw.decode("utf-8"))["CVE_Items"] del raw with timed_operation("Creating model objects..."): for item in items: cve_db_id = get_uuid() summary = item["cve"]["description"]["description_data"][0]["value"] cvss2 = ( item["impact"]["baseMetricV2"]["cvssV2"]["baseScore"] if "baseMetricV2" in item["impact"] else None ) cvss3 = ( item["impact"]["baseMetricV3"]["cvssV3"]["baseScore"] if "baseMetricV3" in item["impact"] else None ) # Construct CWE and CPE lists cwes = get_cwes( item["cve"]["problemtype"]["problemtype_data"][0]["description"] ) cpes = convert_cpes(item["configurations"]) vendors = flatten_vendors(cpes) # Create the CVEs mappings mappings["cves"].append( dict( id=cve_db_id, cve_id=item["cve"]["CVE_data_meta"]["ID"], summary=summary, json=item, vendors=vendors, cwes=cwes, cvss2=cvss2, cvss3=cvss3, created_at=arrow.get(item["publishedDate"]).datetime, updated_at=arrow.get(item["lastModifiedDate"]).datetime, ) ) # Create the vendors and their products for vendor, products in cpes.items(): # Create the vendor if vendor not in mappings["vendors"].keys(): mappings["vendors"][vendor] = dict(id=get_uuid(), name=vendor) for product in products: if get_slug(vendor, product) not in mappings["products"].keys(): mappings["products"][get_slug(vendor, product)] = dict( id=get_uuid(), name=product, vendor_id=mappings["vendors"][vendor]["id"], ) # Insert the objects in database with timed_operation("Inserting CVE..."): db.session.bulk_insert_mappings(Cve, mappings["cves"]) db.session.commit() # Create the changes based on CVEs data for cve in mappings["cves"]: mappings["changes"].append( dict( id=get_uuid(), created_at=cve["created_at"], updated_at=cve["updated_at"], json=cve["json"], cve_id=cve["id"], task_id=task_id, ) ) db.session.bulk_insert_mappings(Change, mappings["changes"]) db.session.commit() info("{} CVE imported.".format(len(mappings["cves"]))) # Free the memory after each processed year del mappings["cves"] del mappings["changes"] return mappings
def test_convert_conf_nested(open_file): conf = open_file("configurations/nested.json") vendors = convert_cpes(conf) vendors["foo"] = sorted(vendors["foo"]) assert vendors == {"foo": ["bar", "baz"], "bar": ["baz"]}
def test_convert_conf_multiple_products(open_file): conf = open_file("configurations/multiple_products.json") vendors = convert_cpes(conf) vendors["foo"] = sorted(vendors["foo"]) assert vendors == {"foo": ["bar", "baz"]}
def test_convert_simple_conf(open_file): conf = open_file("configurations/simple.json") assert convert_cpes(conf) == {"foo": ["bar"]}
def test_convert_empty_conf(): assert convert_cpes({}) == {}
def format(self, json): return convert_cpes(json["configurations"])
def create_cve(cls, cve_json): cvss2 = (cve_json["impact"]["baseMetricV2"]["cvssV2"]["baseScore"] if "baseMetricV2" in cve_json["impact"] else None) cvss3 = (cve_json["impact"]["baseMetricV3"]["cvssV3"]["baseScore"] if "baseMetricV3" in cve_json["impact"] else None) # Construct CWE and CPE lists cwes = get_cwes(cve_json["cve"]["problemtype"]["problemtype_data"][0] ["description"]) cpes = convert_cpes(cve_json["configurations"]) vendors = flatten_vendors(cpes) # Create the CVE cve = Cve( cve_id=cve_json["cve"]["CVE_data_meta"]["ID"], summary=cve_json["cve"]["description"]["description_data"][0] ["value"], json=cve_json, vendors=vendors, cwes=cwes, cvss2=cvss2, cvss3=cvss3, created_at=arrow.get(cve_json["publishedDate"]).datetime, updated_at=arrow.get(cve_json["lastModifiedDate"]).datetime, ) db.session.add(cve) db.session.commit() # Add the CWE that not exists yet in database for cwe in cwes: cwe_obj = Cwe.query.filter_by(cwe_id=cwe).first() if not cwe_obj: info( f"{cwe} detected in {cve.cve_id} but not existing in database, adding it..." ) cwe_obj = Cwe(cwe_id=cwe) db.session.add(cwe_obj) db.session.commit() # Add the CPEs vendors_products = convert_cpes( nested_lookup("cpe23Uri", cve_json["configurations"])) for vendor, products in vendors_products.items(): v_obj = Vendor.query.filter_by(name=vendor).first() # Create the vendor if not v_obj: v_obj = Vendor(name=vendor) db.session.add(v_obj) db.session.commit() # Create the products for product in products: p_obj = Product.query.filter_by(name=product, vendor=v_obj).first() if not p_obj: p_obj = Product(name=product, vendor=v_obj) db.session.add(p_obj) db.session.commit() return cve