def _vmaas_repos_modified_since(modified_since: str) -> list:
"""Get list of modified repose since `modified since`"""
repos_json = {"repository_list": [".*"], "page": 1, "page_size": DEFAULT_PAGE_SIZE,
"modified_since": modified_since}
repos_pages = paging(VMAAS_REPOS_ENDPOINT, repos_json)[1]
repos = [repo_name_key for repo_name_key in repos_pages]
LOGGER.info("%d repos found updated since %s", len(repos), modified_since)
return repos
def test_cves_paging(self, caplog): # pylint: disable=unused-argument
"""Test downloading all cves pages"""
cves_json = {"cve_list": [".*"], "page": DEFAULT_PAGE, "page_size": DEFAULT_PAGE_SIZE, "rh_only": True}
with caplog.at_level(logging.INFO):
success, cves_pages = paging(VMAAS_CVES_ENDPOINT, cves_json)
assert success
assert cves_pages['page'] == 10
caplog.clear()
def test_repos_paging(self, caplog): # pylint: disable=unused-argument
"""Test downloading all repos pages"""
repos_json = {"repository_list": [".*"], "page": DEFAULT_PAGE, "page_size": DEFAULT_PAGE_SIZE}
with caplog.at_level(logging.INFO):
success, repos_pages = paging(VMAAS_REPOS_ENDPOINT, repos_json)
assert success
assert repos_pages['page'] == 2
caplog.clear()
def _vmaas_repos_modified_since(modified_since: str) -> list:
"""Get list of modified repose since `modified since`"""
repos_json = {"repository_list": [".*"], "page": 1, "page_size": DEFAULT_PAGE_SIZE,
"modified_since": modified_since}
success, repos_pages = paging(VMAAS_REPOS_ENDPOINT, repos_json)
if not success:
return []
repos = list(repos_pages["repository_list"])
LOGGER.info("%d repos found updated since %s", len(repos), modified_since)
return repos
def sync_cve_md():
"""Sync all CVE metadata from VMaaS"""
LOGGER.info('Syncing CVE metadata')
with DatabasePoolConnection() as conn:
with conn.cursor() as cur:
impact_id_map = {}
cur.execute("select name, id from cve_impact")
for impact_name, impact_id in cur.fetchall():
impact_id_map[impact_name] = impact_id
cur.execute('select id, cve from cve_metadata')
cves_in_db = {}
for cve_tuple in cur.fetchall():
cves_in_db[cve_tuple[1]] = cve_tuple[0]
cve_json = {'cve_list': [".*"], 'page': 1, 'page_size': DEFAULT_PAGE_SIZE, 'rh_only': True, 'errata_associated': True}
success, cve_pages = paging(VMAAS_CVES_ENDPOINT, cve_json)
if not success:
return success
cves = cve_pages['cve_list']
LOGGER.info("Importing CVE metadata")
to_insert, to_update, to_delete = process_cve_list(cves, cves_in_db, impact_id_map)
insert_cves(cur, to_insert)
update_cves(cur, to_update)
if to_delete:
associated_cves = set()
LOGGER.info("Deleting %s unnecessary CVE metadata", len(to_delete))
cur.execute("""select distinct cve_id from system_vulnerabilities""")
for row in cur.fetchall():
associated_cves.add(row[0])
cur.execute("""select distinct cve_id from cve_rule_mapping""")
for row in cur.fetchall():
associated_cves.add(row[0])
safety_delete = []
unable_to_delete = []
for cve_to_delete in to_delete:
cve_id = cves_in_db[cve_to_delete[0]]
if cve_id in associated_cves:
unable_to_delete.append(cve_to_delete[0])
else:
safety_delete.append(cve_id)
if safety_delete:
execute_values(cur, """delete from cve_account_data
where cve_id in (%s)""",
list(zip(safety_delete)), page_size=len(safety_delete))
execute_values(cur, """delete from cve_metadata where id in (%s)""",
list(zip(safety_delete)), page_size=len(safety_delete))
LOGGER.info('Finished deleting unnecessary CVE metadata')
if unable_to_delete:
LOGGER.warning(
'Unable to delete %s cves (still referenced from system_vulnerabilities table or have rules): %s',
len(unable_to_delete), str(unable_to_delete))
LOGGER.debug('Attempting to update information about %s', str(unable_to_delete))
cve_json = {'cve_list': unable_to_delete, 'page': 1, 'page_size': DEFAULT_PAGE_SIZE, 'rh_only': True}
success, cve_pages = paging(VMAAS_CVES_ENDPOINT, cve_json)
if not success:
return success
cves = cve_pages['cve_list']
_, to_update, _ = process_cve_list(cves, cves_in_db, impact_id_map)
update_cves(cur, to_update)
conn.commit()
LOGGER.info('Finished syncing CVE metadata')
return success
def sync_cve_md(webhook_queue): # pylint: disable=too-many-branches, too-many-statements
"""Sync all CVE metadata from VMaaS"""
LOGGER.info('Syncing CVE metadata')
with DatabasePoolConnection() as conn:
with conn.cursor() as cur:
impact_id_map = {}
cur.execute("select name, id from cve_impact")
for impact_name, impact_id in cur.fetchall():
impact_id_map[impact_name] = impact_id
cur.execute('select cve from cve_metadata')
cves_in_db = []
for cve_tuple in cur.fetchall():
cves_in_db.append(cve_tuple[0])
cvss_buckets = {'0to3': 0, '3to7': 0, '7to10': 0}
cve_json = {'cve_list': [".*"], 'page': 1, 'page_size': DEFAULT_PAGE_SIZE, 'rh_only': True}
success, cve_pages = paging(VMAAS_CVES_ENDPOINT, cve_json)
if not success:
return success
cves = cve_pages['cve_list']
LOGGER.info("Importing CVE metadata")
to_insert = []
to_update = []
to_delete = []
for cve in cves:
description = cves[cve]['description']
impact_id = impact_id_map[cves[cve]['impact']]
public_date = cves[cve]['public_date'] or None
modified_date = cves[cve]['modified_date'] or None
cvss3_score = float(cves[cve]['cvss3_score']) if cves[cve].get('cvss3_score') else None
cvss3_metrics = cves[cve].get('cvss3_metrics')
cvss2_score = float(cves[cve]['cvss2_score']) if cves[cve].get('cvss2_score') else None
cvss2_metrics = cves[cve].get('cvss2_metrics')
redhat_url = cves[cve].get('redhat_url', None)
secondary_url = cves[cve].get('secondary_url', None)
row = (cve, description, impact_id, public_date, modified_date, cvss3_score, cvss3_metrics,
cvss2_score, cvss2_metrics, redhat_url, secondary_url)
if cve not in cves_in_db:
to_insert.append(row)
cvss_score = cvss3_score if cvss3_score is not None else cvss2_score
if cvss_score is None:
pass
elif cvss_score < 3:
cvss_buckets['0to3'] += 1
elif 3 <= cvss_score < 7:
cvss_buckets['3to7'] += 1
elif 7 <= cvss_score <= 10:
cvss_buckets['7to10'] += 1
else:
to_update.append(row)
to_delete = [(cve,) for cve in cves_in_db if cve not in cves]
if to_insert:
execute_values(cur, """insert into cve_metadata
(cve, description, impact_id, public_date, modified_date,
cvss3_score, cvss3_metrics, cvss2_score, cvss2_metrics, redhat_url,
secondary_url)
values %s""", to_insert, page_size=len(to_insert))
if to_update:
execute_values(cur, """update cve_metadata set description = data.description,
impact_id = data.impact_id,
public_date = cast(data.public_date as timestamp with time zone),
modified_date = cast(data.modified_date as timestamp with time zone),
cvss3_score = cast(data.cvss3_score as numeric),
cvss3_metrics = data.cvss3_metrics,
cvss2_score = cast(data.cvss2_score as numeric),
cvss2_metrics = data.cvss2_metrics,
redhat_url = data.redhat_url,
secondary_url = data.secondary_url
from (values %s) as data
(cve, description, impact_id, public_date, modified_date,
cvss3_score, cvss3_metrics, cvss2_score, cvss2_metrics, redhat_url,
secondary_url)
where cve_metadata.cve = data.cve""",
to_update, page_size=len(to_update))
if to_delete:
LOGGER.info("Deleting %s unnecessary CVE metadata", len(to_delete))
execute_values(cur, """select cm.cve from cve_metadata cm
where not exists(select sv.cve_id from system_vulnerabilities sv
where cm.id = sv.cve_id)
and cm.cve in (%s)""",
to_delete, page_size=len(to_delete))
safety_delete = [cve_name_tuple[0] for cve_name_tuple in cur.fetchall()]
unable_to_delete = [cve for cve in zip(*to_delete).__next__() if cve not in safety_delete]
if safety_delete:
execute_values(cur, """delete from cve_account_data
where cve_id in (select id from cve_metadata
where cve in (%s))""",
[i for i in zip(safety_delete)], page_size=len(safety_delete))
execute_values(cur, """delete from cve_metadata where cve in (%s)""",
[i for i in zip(safety_delete)], page_size=len(safety_delete))
LOGGER.info('Finished deleting unnecessary CVE metadata')
if unable_to_delete:
LOGGER.warning(
'Unable to delete %s cves (still referenced from system_vulnerabilities table): %s',
len(unable_to_delete), str(unable_to_delete))
LOGGER.info("Finished importing CVE metadata (page: %s, page_size: %s, pages: %s)",
cve_pages['page'], cve_pages['page_size'], cve_pages['pages'])
msgs = []
time = dt.datetime.utcnow().isoformat()
for level in cvss_buckets:
if cvss_buckets[level] > 0:
msg = {
'application': 'vulnerability',
'event_type': 'new-cve',
'level': level,
'message': 'Discovered %s new CVEs with cvss score within %s radius' % (cvss_buckets[level],
level),
'timestamp': time,
}
msgs.append(msg)
if msgs:
cur.execute('select name from rh_account')
for rh_account in cur.fetchall():
for msg in msgs:
msg.update({'account_id': rh_account[0]})
webhook_queue.send(msg)
conn.commit()
LOGGER.info('Finished syncing CVE metadata')
return success
def sync_cve_md(): # pylint: disable=too-many-branches, too-many-statements
"""Sync all CVE metadata from VMaaS"""
LOGGER.info('Syncing CVE metadata')
with DatabasePoolConnection() as conn:
with conn.cursor() as cur:
impact_id_map = {}
cur.execute("select name, id from cve_impact")
for impact_name, impact_id in cur.fetchall():
impact_id_map[impact_name] = impact_id
cur.execute('select cve from cve_metadata')
cves_in_db = []
for cve_tuple in cur.fetchall():
cves_in_db.append(cve_tuple[0])
cve_json = {
'cve_list': [".*"],
'page': 1,
'page_size': DEFAULT_PAGE_SIZE,
'rh_only': True
}
success, cve_pages = paging(VMAAS_CVES_ENDPOINT, cve_json)
if not success:
return success
cves = cve_pages['cve_list']
LOGGER.info("Importing CVE metadata")
to_insert = []
to_update = []
to_delete = []
for cve in cves:
description = cves[cve]['description']
impact_id = impact_id_map[cves[cve]['impact']]
public_date = cves[cve]['public_date'] or None
modified_date = cves[cve]['modified_date'] or None
cvss3_score = float(cves[cve]['cvss3_score']) if cves[cve].get(
'cvss3_score') else None
cvss3_metrics = cves[cve].get('cvss3_metrics')
cvss2_score = float(cves[cve]['cvss2_score']) if cves[cve].get(
'cvss2_score') else None
cvss2_metrics = cves[cve].get('cvss2_metrics')
redhat_url = cves[cve].get('redhat_url', None)
secondary_url = cves[cve].get('secondary_url', None)
row = (cve, description, impact_id, public_date, modified_date,
cvss3_score, cvss3_metrics, cvss2_score, cvss2_metrics,
redhat_url, secondary_url)
if cve not in cves_in_db:
to_insert.append(row)
else:
to_update.append(row)
to_delete = [(cve, ) for cve in cves_in_db if cve not in cves]
if to_insert:
execute_values(cur,
"""insert into cve_metadata
(cve, description, impact_id, public_date, modified_date,
cvss3_score, cvss3_metrics, cvss2_score, cvss2_metrics, redhat_url,
secondary_url)
values %s""",
to_insert,
page_size=len(to_insert))
if to_update:
execute_values(
cur,
"""update cve_metadata set description = data.description,
impact_id = data.impact_id,
public_date = cast(data.public_date as timestamp with time zone),
modified_date = cast(data.modified_date as timestamp with time zone),
cvss3_score = cast(data.cvss3_score as numeric),
cvss3_metrics = data.cvss3_metrics,
cvss2_score = cast(data.cvss2_score as numeric),
cvss2_metrics = data.cvss2_metrics,
redhat_url = data.redhat_url,
secondary_url = data.secondary_url
from (values %s) as data
(cve, description, impact_id, public_date, modified_date,
cvss3_score, cvss3_metrics, cvss2_score, cvss2_metrics, redhat_url,
secondary_url)
where cve_metadata.cve = data.cve""",
to_update,
page_size=len(to_update))
if to_delete:
LOGGER.info("Deleting %s unnecessary CVE metadata",
len(to_delete))
execute_values(cur,
"""select cm.cve from cve_metadata cm
where not exists(select sv.cve_id from system_vulnerabilities sv
where cm.id = sv.cve_id)
and cm.cve in (%s)""",
to_delete,
page_size=len(to_delete))
safety_delete = [
cve_name_tuple[0] for cve_name_tuple in cur.fetchall()
]
unable_to_delete = [
cve for cve in zip(*to_delete).__next__()
if cve not in safety_delete
]
if safety_delete:
execute_values(cur,
"""delete from cve_account_data
where cve_id in (select id from cve_metadata
where cve in (%s))""",
[i for i in zip(safety_delete)],
page_size=len(safety_delete))
execute_values(
cur,
"""delete from cve_metadata where cve in (%s)""",
[i for i in zip(safety_delete)],
page_size=len(safety_delete))
LOGGER.info('Finished deleting unnecessary CVE metadata')
if unable_to_delete:
LOGGER.warning(
'Unable to delete %s cves (still referenced from system_vulnerabilities table): %s',
len(unable_to_delete), str(unable_to_delete))
LOGGER.info(
"Finished importing CVE metadata (page: %s, page_size: %s, pages: %s)",
cve_pages['page'], cve_pages['page_size'], cve_pages['pages'])
conn.commit()
LOGGER.info('Finished syncing CVE metadata')
return success
