def __init__(self, filename, lastmodified):
self.lastmodified = lastmodified
self.cves = {}
root = None
updated = None
for event, elem in eT.iterparse(filename, events=("start", "end")):
if elem.tag == "cvemap" and event == "start":
root = elem
updated = parse_datetime(elem.get('updated'))
elif elem.tag == "Vulnerability" and event == "end":
name = elem.get('name')
self.cves[name] = {
'impact': text_strip(elem.find('ThreatSeverity')),
'published_date': parse_datetime(text_strip(elem.find('PublicDate'))),
'modified_date': updated,
'cvss2_score': text_strip(elem.find('CVSS/CVSSBaseScore')),
'cvss2_metrics': text_strip(elem.find('CVSS/CVSSScoringVector')),
'cvss3_score': text_strip(elem.find('CVSS3/CVSS3BaseScore')),
'cvss3_metrics': text_strip(elem.find('CVSS3/CVSS3ScoringVector')),
'cwe_list': self._cwe_list(text_strip(elem.find('CWE'))),
'description': self._cve_description(elem.findall('Details[@{%s}lang="en:us"]' % NS)),
'iava': text_strip(elem.find('IAVA')),
'redhat_url': "https://access.redhat.com/security/cve/" + str.lower(name),
'secondary_url': text_strip(elem.find('References'))
}
# Clear the XML tree continuously
root.clear()
def _read_head(self, failed):
"""Reads downloaded meta files and checks for updates."""
header_path = self._tmp_head()
if not failed:
header = CvemapHead(header_path)
# already synced before?
db_lastmodified = parse_datetime(self.cvemap_store.lastmodified())
self.lastmodified = parse_datetime(header.get_lastmodified())
# if header doesn't have last-modified
if self.lastmodified is None:
self.lastmodified = now()
# synced for the first time or has newer revision
if (db_lastmodified is None
or self.lastmodified > db_lastmodified):
self.updated = True
else:
self.logger.info("Cve map has not been updated (since %s).",
str(db_lastmodified))
else:
FAILED_CVEMAP.inc()
self.logger.warning("Download failed: %s (HTTP CODE %d)", URL, failed[header_path])
def check_repo(self, dbdump):
"""Check repo data in dump."""
repo_detail = {}
repolabel2ids = {}
pkgid2repoids = {}
for row in dbdump.execute("select * from repo_detail"):
repo_id = row[0]
repo = (row[1], row[2], row[3], row[4], row[5], row[6], row[7],
parse_datetime(row[8]), bool(row[9]))
repo_detail[repo_id] = repo
repolabel2ids.setdefault(repo[0], []).append(repo_id)
for row in dbdump.execute("select pkg_id, repo_id from pkg_repo"):
pkgid2repoids.setdefault(row[0], []).append(row[1])
assert 801 in repo_detail
assert repo_detail[801][0] == "content set 1"
assert repo_detail[801][1] == "content-set-name-1"
assert repo_detail[801][2] == "https://www.repourl.com/repo1"
assert repo_detail[801][3] == "noarch"
assert repo_detail[801][4] == "1"
assert repo_detail[801][5] == "product1"
assert repo_detail[801][6] == 501
assert repo_detail[801][7] == parse_datetime(
"2019-08-01T01:00:00-05:00")
assert "content set 1" in repolabel2ids
assert repolabel2ids["content set 1"] == [801]
assert 301 in pkgid2repoids
assert 302 in pkgid2repoids
assert 303 in pkgid2repoids
assert 304 in pkgid2repoids
assert 305 in pkgid2repoids
assert 306 in pkgid2repoids
repo_list = pkgid2repoids[306]
assert 801 in repo_list
assert 802 in repo_list
assert 307 in pkgid2repoids
def __init__(self, filename):
self.lastmodified = None
self.cpes = {}
root = None
for event, elem in eT.iterparse(filename, events=("start", "end")):
if elem.tag == "{%s}cpe-list" % NS["cpe"] and event == "start":
root = elem
elif elem.tag == "{%s}generator" % NS["cpe"] and event == "end":
self.lastmodified = parse_datetime(text_strip(elem.find('cpe:timestamp', NS)))
elif elem.tag == "{%s}cpe-item" % NS["cpe"] and event == "end":
name = elem.get('name')
self.cpes[name] = text_strip(elem.find('cpe:title', NS))
# Clear the XML tree continuously
root.clear()
def store(self): # pylint: disable=too-many-branches,too-many-statements
"""Sync all OVAL feeds. Process files in batches due to disk space and memory usage."""
self.logger.info("Checking OVAL feed.")
failed = self._download_feed()
if failed:
for path in failed:
FAILED_IMPORT_OVAL.inc()
self.logger.warning("OVAL feed failed to download, %s (HTTP CODE %d).", path, failed[path])
self.clean()
return
db_oval_files = self.oval_store.oval_file_map.copy()
batches = BatchList()
up_to_date = 0
# Filter out all not updated OVAL definition files
with open(self.feed_path, 'r', encoding='utf8') as feed_file:
feed = json.load(feed_file)
feed_oval_files = {entry["id"]: entry for entry in feed["feed"]["entry"]}
for entry in feed_oval_files.values():
if self._skip_oval_definition_file(entry['id'], feed_oval_files):
continue
db_timestamp = db_oval_files.get(entry['id'])
feed_timestamp = parse_datetime(entry["updated"])
if not db_timestamp or feed_timestamp > db_timestamp[1]:
local_path = os.path.join(self.tmp_directory, entry["content"]["src"].replace(OVAL_FEED_BASE_URL, ""))
oval_definitions_file = OvalDefinitions(entry["id"], feed_timestamp,
entry["content"]["src"], local_path)
batches.add_item(oval_definitions_file)
else:
up_to_date += 1
db_oval_files.pop(entry["id"], None)
feed_updated = parse_datetime(feed["feed"]["updated"])
self.logger.info("%d OVAL definition files are up to date.", up_to_date)
total_oval_files = batches.get_total_items()
completed_oval_files = 0
self.logger.info("%d OVAL definition files need to be synced.", total_oval_files)
self.logger.info("%d OVAL definition files need to be deleted.", len(db_oval_files))
try:
for batch in batches:
self.logger.info("Syncing a batch of %d OVAL definition files", len(batch))
failed = self._download_definitions(batch)
if failed:
self.logger.warning("%d OVAL definition files failed to download.", len(failed))
batch = [oval_file for oval_file in batch if oval_file.local_path not in failed]
self._unpack_definitions(batch)
for oval_definitions_file in batch:
completed_oval_files += 1
try:
oval_definitions_file.load_metadata()
self.logger.info("Syncing OVAL definition file: %s [%s/%s]", oval_definitions_file.oval_id,
completed_oval_files, total_oval_files)
self.oval_store.store(oval_definitions_file)
finally:
oval_definitions_file.unload_metadata()
self.delete_oval_file(list(db_oval_files))
# Timestamp of main feed file
self.oval_store.save_lastmodified(feed_updated)
finally:
self.clean()
def _load_sqlite(self, data):
for (id, arch) in self._sqlite_execute(data,
'select id, arch from arch'):
self.id2arch[int(id)] = arch
self.arch2id[arch] = int(id)
for (src, dst) in self._sqlite_execute(
data, 'select from_arch_id, to_arch_id from arch_compat'):
self.arch_compat.setdefault(src, set()).add(int(dst))
for (id,
string) in self._sqlite_execute(data,
'select id, string from string'):
self.strings[int(id)] = string
for (id, label) in self._sqlite_execute(
data, 'select id, label from content_set'):
self.content_set_id2label[id] = label
self.label2content_set_id[label] = id
for (id, name) in self._sqlite_execute(
data, 'select id, packagename from packagename'):
self.packagename2id[str(name)] = int(id)
self.id2packagename[int(id)] = str(name)
for (cs_id, name_id) in self._sqlite_execute(
data,
"select content_set_id, pkg_name_id from content_set_pkg_name"
):
self.content_set_id2pkg_name_ids.setdefault(
cs_id, array.array('q')).append(name_id)
for (cs_id, src_name_id) in self._sqlite_execute(
data,
"select content_set_id, src_pkg_name_id from content_set_src_pkg_name"
):
self.src_pkg_name_id2cs_ids.setdefault(
src_name_id, array.array('q')).append(cs_id)
for (cpe_id,
label) in self._sqlite_execute(data, "select id, label from cpe"):
self.cpe_id2label[cpe_id] = label
self.label2cpe_id[label] = cpe_id
for (cpe_id, cs_id) in self._sqlite_execute(
data, "select cpe_id, content_set_id from cpe_content_set"):
self.content_set_id2cpe_ids.setdefault(
cs_id, array.array('q')).append(cpe_id)
for (cpe_id, repo_id) in self._sqlite_execute(
data, "select cpe_id, repo_id from cpe_repo"):
self.repo_id2cpe_ids.setdefault(repo_id,
array.array('q')).append(cpe_id)
for (name_id, pkg_id) in self._sqlite_execute(
data,
'select name_id, package_id from updates order by name_id, package_order'
):
self.updates.setdefault(int(name_id), []).append(int(pkg_id))
for (name_id, evr_id, order) in self._sqlite_execute(
data,
'select name_id, evr_id, package_order from updates_index order by name_id, package_order'
):
name_id = int(name_id)
evr_id = int(evr_id)
order = int(order)
self.updates_index.setdefault(name_id,
{}).setdefault(evr_id,
[]).append(order)
for (id, epoch, ver, rel) in self._sqlite_execute(
data, 'select id, epoch, version, release from evr'):
evr = (str(epoch), str(ver), str(rel))
self.id2evr[int(id)] = evr
self.evr2id[evr] = int(id)
for (id, name_id, evr_id, arch_id, sum_id, descr_id, src_pkg_id,
modified_str) in self._sqlite_execute(
data, 'select * from package_detail'):
detail = array.array('q')
modified_dt = datetime.datetime.strptime(modified_str,
'%Y-%m-%d %H:%M:%S.%f')
modified_int = int(modified_dt.timestamp())
detail.fromlist([
name_id, evr_id, arch_id, sum_id, descr_id, src_pkg_id or 0,
modified_int
])
self.package_details[id] = detail
self.nevra2pkgid[(name_id, evr_id, arch_id)] = id
if src_pkg_id:
self.src_pkg_id2pkg_ids.setdefault(src_pkg_id,
array.array('q')).append(id)
for row in self._sqlite_execute(data, "select * from repo_detail"):
id = row[0]
repo = (row[1], row[2], row[3], row[4], row[5], row[6], row[7],
parse_datetime(row[8]), bool(row[9]))
self.repo_detail[id] = repo
self.repolabel2ids.setdefault(repo[0], array.array('q')).append(id)
for row in self._sqlite_execute(
data, "select pkg_id, repo_id from pkg_repo"):
self.pkgid2repoids.setdefault(row[0],
array.array('q')).append(row[1])
errataid2cves = {}
cve2eid = {}
for row in self._sqlite_execute(
data, "select errata_id, cve from errata_cve"):
errataid2cves.setdefault(row[0], []).append(row[1])
cve2eid.setdefault(row[1], array.array('q')).append(row[0])
errataid2pkgid = {}
for row in self._sqlite_execute(
data, "select pkg_id, errata_id from pkg_errata "):
self.pkgid2errataids.setdefault(row[0],
array.array('q')).append(row[1])
errataid2pkgid.setdefault(row[1], array.array('q')).append(row[0])
errataid2bzs = {}
for row in self._sqlite_execute(
data, "select errata_id, bugzilla from errata_bugzilla"):
errataid2bzs.setdefault(row[0], []).append(row[1])
errataid2refs = {}
for row in self._sqlite_execute(
data, "select errata_id, ref from errata_refs"):
errataid2refs.setdefault(row[0], []).append(row[1])
errataidmodulestream2pkgid = {}
for row in self._sqlite_execute(
data,
"select pkg_id, errata_id, module_stream_id from errata_modulepkg"
):
self.pkgerrata2module.setdefault((row[0], row[1]),
set()).add(row[2])
errataidmodulestream2pkgid.setdefault(
(row[1], row[2]), array.array('q')).append(row[0])
errataid2modules = {}
for row in self._sqlite_execute(
data,
"""select errata_id, module_name, module_stream_id, module_stream,
module_version, module_context from errata_module"""
):
if row[0] not in errataid2modules:
errataid2modules[row[0]] = {}
if (row[1], row[3], row[4],
row[5]) not in errataid2modules[row[0]]:
errataid2modules[row[0]][(row[1], row[3], row[4], row[5])] = {
"module_name":
row[1],
"module_stream":
row[3],
"module_version":
row[4],
"module_context":
row[5],
"package_list":
errataidmodulestream2pkgid.get((row[0], row[2]),
array.array('q')),
"source_package_list": [] # populated in API
}
else:
# Add packages from module with same name but different architecture, etc.
errataid2modules[row[0]][(row[1], row[3], row[4],
row[5])]["package_list"].extend(
errataidmodulestream2pkgid.get(
(row[0], row[2]),
array.array('q')))
for row in self._sqlite_execute(data, "select * from errata_detail"):
id = row[0]
name = row[1]
errata = (
row[2],
row[3],
row[4],
row[5],
row[6],
row[7],
parse_datetime(row[8]),
parse_datetime(row[9]),
errataid2cves.get(id, []),
errataid2pkgid.get(id, array.array('q')),
errataid2bzs.get(id, []),
errataid2refs.get(id, []),
list(errataid2modules.get(id, {}).values()),
row[10],
bool(row[11]),
bool(row[12]),
id,
)
self.errata_detail[name] = errata
self.errataid2name[id] = name
for row in self._sqlite_execute(
data, "select errata_id, repo_id from errata_repo"):
self.errataid2repoids.setdefault(row[0],
array.array('q')).append(row[1])
for row in self._sqlite_execute(
data, "select module, stream, stream_id from module_stream"):
self.modulename2id.setdefault((row[0], row[1]), set()).add(row[2])
for row in self._sqlite_execute(
data,
"select stream_id, require_id from module_stream_require"):
self.modulerequire.setdefault(row[0], set()).add(row[1])
cveid2cwe = {}
for row in self._sqlite_execute(data,
"select cve_id, cwe from cve_cwe"):
cveid2cwe.setdefault(row[0], []).append(row[1])
cveid2pid = {}
for row in self._sqlite_execute(data,
"select cve_id, pkg_id from cve_pkg"):
cveid2pid.setdefault(row[0], array.array('q')).append(row[1])
for row in self._sqlite_execute(data, "select * from cve_detail"):
id = row[0]
name = row[1]
item = (row[2], row[3], row[4], row[5], row[6],
parse_datetime(row[7]), parse_datetime(row[8]), row[9],
row[10], cveid2cwe.get(id, []),
cveid2pid.get(id, array.array('q')),
cve2eid.get(name,
array.array('q')), row[11], row[12], row[13])
self.cve_detail[name] = item
for row in self._sqlite_execute(data,
"select * from oval_definition_cpe"):
self.cpe_id2ovaldefinition_ids.setdefault(
row[0], array.array('q')).append(row[1])
for row in self._sqlite_execute(
data, "select * from packagename_oval_definition"):
self.packagename_id2definition_ids.setdefault(
row[0], array.array('q')).append(row[1])
for row in self._sqlite_execute(
data, "select * from oval_definition_detail"):
self.ovaldefinition_detail[row[0]] = (row[1], row[2])
for row in self._sqlite_execute(data,
"select * from oval_definition_cve"):
self.ovaldefinition_id2cves.setdefault(row[0], []).append(row[1])
for row in self._sqlite_execute(data,
"select * from oval_criteria_type"):
self.ovalcriteria_id2type[row[0]] = row[1]
for row in self._sqlite_execute(
data, "select * from oval_criteria_dependency"):
if row[2] is None and row[3] is None:
self.ovalcriteria_id2depcriteria_ids.setdefault(
row[0], array.array('q')).append(row[1])
elif row[1] is None and row[3] is None:
self.ovalcriteria_id2deptest_ids.setdefault(
row[0], array.array('q')).append(row[2])
else:
self.ovalcriteria_id2depmoduletest_ids.setdefault(
row[0], array.array('q')).append(row[3])
for row in self._sqlite_execute(data,
"select * from oval_test_detail"):
self.ovaltest_detail[row[0]] = (row[1], row[2])
for row in self._sqlite_execute(data, "select * from oval_test_state"):
self.ovaltest_id2states.setdefault(row[0], []).append(
(row[1], row[2], row[3]))
for row in self._sqlite_execute(data, "select * from oval_state_arch"):
self.ovalstate_id2arches.setdefault(row[0], set()).add(row[1])
for row in self._sqlite_execute(
data, "select * from oval_module_test_detail"):
self.ovalmoduletest_detail[row[0]] = row[1]
names = [
"exported", "last_change", "repository_changes", "cve_changes",
"errata_changes"
]
for row in self._sqlite_execute(
data, "select %s from dbchange" % ','.join(names)):
for (i, v) in enumerate(row):
self.dbchange[names[i]] = v
def test_parse_string(self):
"""Test parsing datetime from string."""
assert isinstance(
date_utils.parse_datetime("2018-10-24 15:27:40.058353"), datetime)
def test_parse_none(self):
"""Test parsing date = None."""
assert date_utils.parse_datetime(None) is None