def create_user(username, email, password, admin):
"""Create a user or admin."""
if User.query.filter_by(username=username).first():
raise click.BadParameter(f"{username} already exists.",
param_hint="username")
if User.query.filter_by(email=email).first():
raise click.BadParameter(f"{email} already exists.",
param_hint="email")
user = User(
username=username,
email=email,
active=True,
admin=admin,
email_confirmed_at=datetime.datetime.utcnow(),
password=app.user_manager.hash_password(password),
)
db.session.add(user)
try:
db.session.commit()
except IntegrityError as e:
error(e)
else:
info("User {} created.".format(username))
def import_data(confirm):
"""
Perform initial imports (cve, cwe, cpe).
"""
if Cve.query.first():
info("Import already done.")
return
if not confirm:
msg = ("This command will import initial data in your database. "
"Do you want to continue ?".format(CVE_FIRST_YEAR,
CURRENT_YEAR))
if not click.confirm(msg):
info("Bye.")
return
# Import the CWE list
cwe.run()
# Import the CVE, then use the returned list of vendors
# to merge them with the official CPE dictionnary list.
vendors = cve.run()
cpe.run(vendors)
# Populate the metas table
with timed_operation("Populating metas table..."):
meta = Meta(name="nvd_last_sha256", value="default")
db.session.add(meta)
db.session.commit()
def init():
"""Initialize the configuration file."""
path, created = create_config()
if created:
info(f"Configuration created in {path}")
else:
error(f"Configuration already exists ({path})")
def create_change(cls, cve_obj, cve_json, task, events):
change = Change(
created_at=arrow.get(cve_json["lastModifiedDate"]).datetime,
updated_at=arrow.get(cve_json["lastModifiedDate"]).datetime,
cve=cve_obj,
task=task,
events=events,
json=cve_json,
)
db.session.add(change)
db.session.commit()
info("Change created (ID: {})".format(change.id))
return change
def create_event(cls, cve_obj, cve_json, type, payload={}):
event = Event(
created_at=arrow.get(cve_json["lastModifiedDate"]).datetime,
updated_at=arrow.get(cve_json["lastModifiedDate"]).datetime,
cve=cve_obj,
type=type,
details=payload,
review=False,
)
db.session.add(event)
db.session.commit()
info("Event {} created (ID: {})".format(type, event.id))
return event
def run(mappings):
"""
Import the Vendors and Products list.
"""
header("Importing CPE list...")
# Download the XML file
with timed_operation("Downloading {}...".format(NVD_CPE_URL)):
resp = requests.get(NVD_CPE_URL).content
# Parse the XML elements
with timed_operation("Parsing XML elements..."):
raw = gzip.GzipFile(fileobj=BytesIO(resp)).read()
obj = untangle.parse(raw.decode("utf-8"))
items = obj.cpe_list.cpe_item
del obj
# Create the objects
with timed_operation("Creating list of mappings..."):
for item in items:
obj = CPE(item.cpe_23_cpe23_item["name"])
vendor = obj.get_vendor()[0]
product = obj.get_product()[0]
if vendor not in mappings["vendors"].keys():
mappings["vendors"][vendor] = dict(id=get_uuid(), name=vendor)
if get_slug(vendor, product) not in mappings["products"].keys():
mappings["products"][get_slug(vendor, product)] = dict(
id=get_uuid(),
name=product,
vendor_id=mappings["vendors"][vendor]["id"],
)
del items
# Insert the objects in database
with timed_operation("Inserting Vendors and Products..."):
db.session.bulk_insert_mappings(Vendor, mappings["vendors"].values())
db.session.bulk_insert_mappings(Product, mappings["products"].values())
db.session.commit()
info(
"{} vendors and {} products imported.".format(
len(mappings["vendors"]), len(mappings["products"])
)
)
del mappings
def create_user(username, email, password, admin):
"""Create a user or admin."""
user = User(
username=username,
email=email,
active=True,
admin=admin,
email_confirmed_at=datetime.datetime.utcnow(),
password=app.user_manager.hash_password(password),
)
db.session.add(user)
try:
db.session.commit()
except IntegrityError as e:
error(e)
else:
info("User {} created.".format(username))
def execute(self):
old = self.cve_obj.cwes
new = [
c["value"] for c in self.cve_json["cve"]["problemtype"]
["problemtype_data"][0]["description"]
]
payload = {
"added": list(set(new) - set(old)),
"removed": list(set(old) - set(new)),
}
# It's possible that a CVE links a CWE not yet defined in database.
# In this case we'll save it in the `cwes` table and a periodic task
# will populate later its name and description using the MITRE file.
for cwe_id in payload["added"]:
cwe = Cwe.query.filter_by(cwe_id=cwe_id).first()
if not cwe:
info(
f"{cwe_id} detected in {self.cve_obj.cve_id} but not existing in database, adding it..."
)
cwe = Cwe(cwe_id=cwe_id)
db.session.add(cwe)
db.session.commit()
# If the list of CWE changed
if payload["added"] or payload["removed"]:
# Save the new list
self.cve_obj.cwes = new
db.session.commit()
# Create the event
event = CveUtil.create_event(self.cve_obj, self.cve_json, "cwes",
payload)
return event
return None
def run():
"""
Import the CWE list.
"""
header("Importing CWE list...")
# Download the file
with timed_operation("Downloading {}...".format(MITRE_CWE_URL)):
resp = requests.get(MITRE_CWE_URL).content
# Parse weaknesses
with timed_operation("Parsing cwes..."):
z = ZipFile(BytesIO(resp))
raw = z.open(z.namelist()[0]).read()
obj = untangle.parse(raw.decode("utf-8"))
weaknesses = obj.Weakness_Catalog.Weaknesses.Weakness
categories = obj.Weakness_Catalog.Categories.Category
# Create the objects
cwes = {}
with timed_operation("Creating mappings..."):
for c in weaknesses + categories:
cwes[c["ID"]] = dict(
id=get_uuid(),
cwe_id=f"CWE-{c['ID']}",
name=c["Name"],
description=c.Description.cdata
if hasattr(c, "Description") else c.Summary.cdata,
)
# Insert the objects in database
with timed_operation("Inserting CWE..."):
db.session.bulk_insert_mappings(Cwe, cwes.values())
db.session.commit()
info("{} CWE imported.".format(len(cwes)))
del cwes
def run():
"""
Import the CVE list.
"""
mappings = {"vendors": {}, "products": {}}
from opencve.commands.imports import CURRENT_YEAR, CVE_FIRST_YEAR
# Create the initial task
task = Task()
db.session.add(task)
db.session.commit()
task_id = task.id
for year in range(CVE_FIRST_YEAR, CURRENT_YEAR + 1):
header("Importing CVE for {}".format(year))
mappings.update({"cves": [], "changes": []})
# Download the file
url = NVD_CVE_URL.format(year=year)
with timed_operation("Downloading {}...".format(url)):
resp = requests.get(url).content
# Parse the XML elements
with timed_operation("Parsing JSON elements..."):
raw = gzip.GzipFile(fileobj=BytesIO(resp)).read()
del resp
items = json.loads(raw.decode("utf-8"))["CVE_Items"]
del raw
with timed_operation("Creating model objects..."):
for item in items:
cve_db_id = get_uuid()
summary = item["cve"]["description"]["description_data"][0]["value"]
cvss2 = (
item["impact"]["baseMetricV2"]["cvssV2"]["baseScore"]
if "baseMetricV2" in item["impact"]
else None
)
cvss3 = (
item["impact"]["baseMetricV3"]["cvssV3"]["baseScore"]
if "baseMetricV3" in item["impact"]
else None
)
# Construct CWE and CPE lists
cwes = get_cwes(
item["cve"]["problemtype"]["problemtype_data"][0]["description"]
)
cpes = convert_cpes(item["configurations"])
vendors = flatten_vendors(cpes)
# Create the CVEs mappings
mappings["cves"].append(
dict(
id=cve_db_id,
cve_id=item["cve"]["CVE_data_meta"]["ID"],
summary=summary,
json=item,
vendors=vendors,
cwes=cwes,
cvss2=cvss2,
cvss3=cvss3,
created_at=arrow.get(item["publishedDate"]).datetime,
updated_at=arrow.get(item["lastModifiedDate"]).datetime,
)
)
# Create the vendors and their products
for vendor, products in cpes.items():
# Create the vendor
if vendor not in mappings["vendors"].keys():
mappings["vendors"][vendor] = dict(id=get_uuid(), name=vendor)
for product in products:
if get_slug(vendor, product) not in mappings["products"].keys():
mappings["products"][get_slug(vendor, product)] = dict(
id=get_uuid(),
name=product,
vendor_id=mappings["vendors"][vendor]["id"],
)
# Insert the objects in database
with timed_operation("Inserting CVE..."):
db.session.bulk_insert_mappings(Cve, mappings["cves"])
db.session.commit()
# Create the changes based on CVEs data
for cve in mappings["cves"]:
mappings["changes"].append(
dict(
id=get_uuid(),
created_at=cve["created_at"],
updated_at=cve["updated_at"],
json=cve["json"],
cve_id=cve["id"],
task_id=task_id,
)
)
db.session.bulk_insert_mappings(Change, mappings["changes"])
db.session.commit()
info("{} CVE imported.".format(len(mappings["cves"])))
# Free the memory after each processed year
del mappings["cves"]
del mappings["changes"]
return mappings
def create_cve(cls, cve_json):
cvss2 = (cve_json["impact"]["baseMetricV2"]["cvssV2"]["baseScore"]
if "baseMetricV2" in cve_json["impact"] else None)
cvss3 = (cve_json["impact"]["baseMetricV3"]["cvssV3"]["baseScore"]
if "baseMetricV3" in cve_json["impact"] else None)
# Construct CWE and CPE lists
cwes = get_cwes(cve_json["cve"]["problemtype"]["problemtype_data"][0]
["description"])
cpes = convert_cpes(cve_json["configurations"])
vendors = flatten_vendors(cpes)
# Create the CVE
cve = Cve(
cve_id=cve_json["cve"]["CVE_data_meta"]["ID"],
summary=cve_json["cve"]["description"]["description_data"][0]
["value"],
json=cve_json,
vendors=vendors,
cwes=cwes,
cvss2=cvss2,
cvss3=cvss3,
created_at=arrow.get(cve_json["publishedDate"]).datetime,
updated_at=arrow.get(cve_json["lastModifiedDate"]).datetime,
)
db.session.add(cve)
db.session.commit()
# Add the CWE that not exists yet in database
for cwe in cwes:
cwe_obj = Cwe.query.filter_by(cwe_id=cwe).first()
if not cwe_obj:
info(
f"{cwe} detected in {cve.cve_id} but not existing in database, adding it..."
)
cwe_obj = Cwe(cwe_id=cwe)
db.session.add(cwe_obj)
db.session.commit()
# Add the CPEs
vendors_products = convert_cpes(
nested_lookup("cpe23Uri", cve_json["configurations"]))
for vendor, products in vendors_products.items():
v_obj = Vendor.query.filter_by(name=vendor).first()
# Create the vendor
if not v_obj:
v_obj = Vendor(name=vendor)
db.session.add(v_obj)
db.session.commit()
# Create the products
for product in products:
p_obj = Product.query.filter_by(name=product,
vendor=v_obj).first()
if not p_obj:
p_obj = Product(name=product, vendor=v_obj)
db.session.add(p_obj)
db.session.commit()
return cve