def execute(self): old = flatten_vendors(convert_cpes( self.cve_obj.json["configurations"])) new = flatten_vendors(convert_cpes(self.cve_json["configurations"])) payload = list(set(new) - set(old)) if payload: event = CveUtil.create_event(self.cve_obj, self.cve_json, "first_time", payload) return event return None
def test_flatten_multiple_vendors(): assert flatten_vendors({"foo": ["bar", "baz"], "bar": ["baz"]}) == [ "foo", f"foo{PRODUCT_SEPARATOR}bar", f"foo{PRODUCT_SEPARATOR}baz", "bar", f"bar{PRODUCT_SEPARATOR}baz", ]
def execute(self): old = nested_lookup("cpe23Uri", self.cve_obj.json["configurations"]) new = nested_lookup("cpe23Uri", self.cve_json["configurations"]) payload = { "added": list(set(new) - set(old)), "removed": list(set(old) - set(new)), } # The CPEs list has been modified if payload["added"] or payload["removed"]: # Change the CVE's vendors attribute self.cve_obj.vendors = flatten_vendors( convert_cpes(self.cve_json["configurations"]) ) db.session.commit() # Create the vendors and products objects if they don't exist vendors_products = convert_cpes(payload["added"]) for vendor, products in vendors_products.items(): v_obj = Vendor.query.filter_by(name=vendor).first() # Create the vendor and associate it to the CVE if not v_obj: v_obj = Vendor(name=vendor) db.session.add(v_obj) db.session.commit() # Do the same for its products for product in products: p_obj = Product.query.filter_by(name=product, vendor=v_obj).first() if not p_obj: p_obj = Product(name=product, vendor=v_obj) db.session.add(p_obj) db.session.commit() # Create the event event = CveUtil.create_event(self.cve_obj, self.cve_json, "cpes", payload) return event return None
def run(): """ Import the CVE list. """ mappings = {"vendors": {}, "products": {}} from opencve.commands.imports import CURRENT_YEAR, CVE_FIRST_YEAR # Create the initial task task = Task() db.session.add(task) db.session.commit() task_id = task.id for year in range(CVE_FIRST_YEAR, CURRENT_YEAR + 1): header("Importing CVE for {}".format(year)) mappings.update({"cves": [], "changes": []}) # Download the file url = NVD_CVE_URL.format(year=year) with timed_operation("Downloading {}...".format(url)): resp = requests.get(url).content # Parse the XML elements with timed_operation("Parsing JSON elements..."): raw = gzip.GzipFile(fileobj=BytesIO(resp)).read() del resp items = json.loads(raw.decode("utf-8"))["CVE_Items"] del raw with timed_operation("Creating model objects..."): for item in items: cve_db_id = get_uuid() summary = item["cve"]["description"]["description_data"][0]["value"] cvss2 = ( item["impact"]["baseMetricV2"]["cvssV2"]["baseScore"] if "baseMetricV2" in item["impact"] else None ) cvss3 = ( item["impact"]["baseMetricV3"]["cvssV3"]["baseScore"] if "baseMetricV3" in item["impact"] else None ) # Construct CWE and CPE lists cwes = get_cwes( item["cve"]["problemtype"]["problemtype_data"][0]["description"] ) cpes = convert_cpes(item["configurations"]) vendors = flatten_vendors(cpes) # Create the CVEs mappings mappings["cves"].append( dict( id=cve_db_id, cve_id=item["cve"]["CVE_data_meta"]["ID"], summary=summary, json=item, vendors=vendors, cwes=cwes, cvss2=cvss2, cvss3=cvss3, created_at=arrow.get(item["publishedDate"]).datetime, updated_at=arrow.get(item["lastModifiedDate"]).datetime, ) ) # Create the vendors and their products for vendor, products in cpes.items(): # Create the vendor if vendor not in mappings["vendors"].keys(): mappings["vendors"][vendor] = dict(id=get_uuid(), name=vendor) for product in products: if get_slug(vendor, product) not in mappings["products"].keys(): mappings["products"][get_slug(vendor, product)] = dict( id=get_uuid(), name=product, vendor_id=mappings["vendors"][vendor]["id"], ) # Insert the objects in database with timed_operation("Inserting CVE..."): db.session.bulk_insert_mappings(Cve, mappings["cves"]) db.session.commit() # Create the changes based on CVEs data for cve in mappings["cves"]: mappings["changes"].append( dict( id=get_uuid(), created_at=cve["created_at"], updated_at=cve["updated_at"], json=cve["json"], cve_id=cve["id"], task_id=task_id, ) ) db.session.bulk_insert_mappings(Change, mappings["changes"]) db.session.commit() info("{} CVE imported.".format(len(mappings["cves"]))) # Free the memory after each processed year del mappings["cves"] del mappings["changes"] return mappings
def test_flatten_simple_vendor(): assert flatten_vendors({"foo": ["bar"]}) == ["foo", f"foo{PRODUCT_SEPARATOR}bar"]
def test_flatten_empty_vendors(): assert flatten_vendors({}) == []
def create_cve(cls, cve_json): cvss2 = (cve_json["impact"]["baseMetricV2"]["cvssV2"]["baseScore"] if "baseMetricV2" in cve_json["impact"] else None) cvss3 = (cve_json["impact"]["baseMetricV3"]["cvssV3"]["baseScore"] if "baseMetricV3" in cve_json["impact"] else None) # Construct CWE and CPE lists cwes = get_cwes(cve_json["cve"]["problemtype"]["problemtype_data"][0] ["description"]) cpes = convert_cpes(cve_json["configurations"]) vendors = flatten_vendors(cpes) # Create the CVE cve = Cve( cve_id=cve_json["cve"]["CVE_data_meta"]["ID"], summary=cve_json["cve"]["description"]["description_data"][0] ["value"], json=cve_json, vendors=vendors, cwes=cwes, cvss2=cvss2, cvss3=cvss3, created_at=arrow.get(cve_json["publishedDate"]).datetime, updated_at=arrow.get(cve_json["lastModifiedDate"]).datetime, ) db.session.add(cve) db.session.commit() # Add the CWE that not exists yet in database for cwe in cwes: cwe_obj = Cwe.query.filter_by(cwe_id=cwe).first() if not cwe_obj: info( f"{cwe} detected in {cve.cve_id} but not existing in database, adding it..." ) cwe_obj = Cwe(cwe_id=cwe) db.session.add(cwe_obj) db.session.commit() # Add the CPEs vendors_products = convert_cpes( nested_lookup("cpe23Uri", cve_json["configurations"])) for vendor, products in vendors_products.items(): v_obj = Vendor.query.filter_by(name=vendor).first() # Create the vendor if not v_obj: v_obj = Vendor(name=vendor) db.session.add(v_obj) db.session.commit() # Create the products for product in products: p_obj = Product.query.filter_by(name=product, vendor=v_obj).first() if not p_obj: p_obj = Product(name=product, vendor=v_obj) db.session.add(p_obj) db.session.commit() return cve