def __init__(self):
self.logger = get_logger(__name__)
self.conn = DatabaseHandler.get_connection()
self.module_store = ModulesStore()
self.package_store = PackageStore()
self.update_store = UpdateStore()
self.content_set_to_db_id = self._prepare_content_set_map()
def __init__(self):
super().__init__()
self.cpe_store = CpeStore()
self.package_store = PackageStore()
self.evr_operation_map = self._prepare_table_map(
cols=["name"], table="oval_operation_evr")
self.cve_map = self._prepare_table_map(cols=["name"], table="cve")
self.errata_map = self._prepare_table_map(cols=["name"],
table="errata")
self.oval_file_map = self._prepare_table_map(cols=["oval_id"],
to_cols=["id", "updated"],
table="oval_file")
self.oval_check_map = self._prepare_table_map(
cols=["name"], table="oval_check_rpminfo")
self.oval_check_existence_map = self._prepare_table_map(
cols=["name"], table="oval_check_existence_rpminfo")
self.oval_object_map = {}
self.oval_state_map = {}
self.oval_test_map = {}
self.oval_module_test_map = {}
self.oval_definition_map = {}
self.oval_definition_type_map = self._prepare_table_map(
cols=["name"], table="oval_definition_type")
self.oval_criteria_operator_map = self._prepare_table_map(
cols=["name"], table="oval_criteria_operator")
class RepositoryStore:
"""
Class providing interface for listing repositories stored in DB and storing repositories one by one.
"""
def __init__(self):
self.logger = get_logger(__name__)
self.conn = DatabaseHandler.get_connection()
self.module_store = ModulesStore()
self.package_store = PackageStore()
self.update_store = UpdateStore()
self.content_set_to_db_id = self._prepare_content_set_map()
def _prepare_content_set_map(self):
cur = self.conn.cursor()
cur.execute("""select id, label from content_set""")
content_sets = {}
for cs_id, cs_label in cur.fetchall():
content_sets[cs_label] = cs_id
cur.close()
return content_sets
def list_repositories(self):
"""List repositories stored in DB. Dictionary with repository label as key is returned."""
cur = self.conn.cursor()
cur.execute(
"""select cs.label, a.name, r.releasever, r.id, r.url, r.revision, cs.id,
c.name, c.ca_cert, c.cert, c.key
from repo r
left join arch a on r.basearch_id = a.id
left join certificate c on r.certificate_id = c.id
left join content_set cs on r.content_set_id = cs.id""")
repos = {}
for row in cur.fetchall():
# (content_set_label, repo_arch, repo_releasever) -> repo_id, repo_url, repo_revision...
repos[(row[0], row[1], row[2])] = {
"id": row[3],
"url": row[4],
"revision": row[5],
"content_set_id": row[6],
"cert_name": row[7],
"ca_cert": row[8],
"cert": row[9],
"key": row[10]
}
cur.close()
return repos
def _import_basearch(self, basearch):
cur = self.conn.cursor()
try:
cur.execute("select id from arch where name = %s", (basearch, ))
arch_id = cur.fetchone()
if not arch_id:
cur.execute("insert into arch (name) values(%s) returning id",
(basearch, ))
arch_id = cur.fetchone()
self.conn.commit()
except Exception:
self.logger.exception("Failed to import basearch.")
self.conn.rollback()
raise
finally:
cur.close()
return arch_id[0]
def _import_certificate(self, cert_name, ca_cert, cert, key):
if not key:
key = None
cur = self.conn.cursor()
try:
cur.execute("select id from certificate where name = %s",
(cert_name, ))
cert_id = cur.fetchone()
if not cert_id:
cur.execute(
"""insert into certificate (name, ca_cert, cert, key)
values (%s, %s, %s, %s) returning id""", (
cert_name,
ca_cert,
cert,
key,
))
cert_id = cur.fetchone()
else:
cur.execute(
"update certificate set ca_cert = %s, cert = %s, key = %s where name = %s",
(
ca_cert,
cert,
key,
cert_name,
))
self.conn.commit()
except Exception:
self.logger.exception("Failed to import certificate.")
self.conn.rollback()
raise
finally:
cur.close()
return cert_id[0]
def cleanup_unused_data(self):
"""
Deletes packages and errata not associated with any repo etc.
"""
cur = self.conn.cursor()
try:
cur.execute("""select p.id from package p where not exists (
select 1 from pkg_repo pr where pr.pkg_id = p.id
) and p.source_package_id is not null
""")
packages_to_delete = cur.fetchall()
cur.execute("""select e.id from errata e where not exists (
select 1 from errata_repo er where er.errata_id = e.id
)
""")
updates_to_delete = cur.fetchall()
if packages_to_delete:
cur.execute(
"""delete from pkg_errata pe where pe.pkg_id in %s""",
(tuple(packages_to_delete), ))
cur.execute(
"""delete from module_rpm_artifact mra where mra.pkg_id in %s""",
(tuple(packages_to_delete), ))
cur.execute("""delete from package p where p.id in %s""",
(tuple(packages_to_delete), ))
if updates_to_delete:
cur.execute(
"""delete from pkg_errata pe where pe.errata_id in %s""",
(tuple(updates_to_delete), ))
cur.execute(
"""delete from errata_cve ec where ec.errata_id in %s""",
(tuple(updates_to_delete), ))
cur.execute(
"""delete from errata_refs er where er.errata_id in %s""",
(tuple(updates_to_delete), ))
cur.execute(
"""delete from oval_definition_errata ode where ode.errata_id in %s""",
(tuple(updates_to_delete), ))
cur.execute("""delete from errata e where e.id in %s""",
(tuple(updates_to_delete), ))
self.conn.commit()
except Exception: #pylint: disable=broad-except
self.logger.exception("Failed to clean up unused data.")
self.conn.rollback()
finally:
cur.close()
def delete_content_set(self, content_set_label):
"""
Deletes repositories and their content from DB.
"""
content_set_id = self.content_set_to_db_id[content_set_label]
cur = self.conn.cursor()
try:
cur.execute("""select id from repo where content_set_id = %s""",
(content_set_id, ))
repo_ids = cur.fetchall()
for repo_id in repo_ids:
cur.execute("select id from module where repo_id = %s",
(repo_id, ))
module_ids = cur.fetchall()
if module_ids:
cur.execute(
"select id from module_stream where module_id in %s",
(tuple(module_ids), ))
module_stream_ids = cur.fetchall()
if module_stream_ids:
cur.execute(
"""delete from module_profile_pkg
where profile_id in (select id from module_profile
where stream_id in %s)""",
(tuple(module_stream_ids), ))
cur.execute(
"delete from module_rpm_artifact where stream_id in %s",
(tuple(module_stream_ids), ))
cur.execute(
"delete from pkg_errata where module_stream_id in %s",
(tuple(module_stream_ids), ))
cur.execute(
"delete from module_profile where stream_id in %s",
(tuple(module_stream_ids), ))
cur.execute(
"delete from module_stream_require where module_stream_id in %s",
(tuple(module_stream_ids), ))
cur.execute(
"delete from module_stream where module_id in %s",
(tuple(module_ids), ))
cur.execute("delete from module where repo_id = %s",
(repo_id, ))
cur.execute("delete from pkg_repo where repo_id = %s",
(repo_id, ))
cur.execute("delete from errata_repo where repo_id = %s",
(repo_id, ))
cur.execute("delete from repo where id = %s", (repo_id, ))
cur.execute(
"delete from cpe_content_set where content_set_id = %s",
(content_set_id, ))
cur.execute("delete from content_set where id = %s",
(content_set_id, ))
self.conn.commit()
except Exception: # pylint: disable=broad-except
self.logger.exception("Failed to delete content set.")
self.conn.rollback()
finally:
cur.close()
def import_repository(self, repo):
"""
Imports or updates repository record in DB.
"""
if repo.ca_cert:
# will raise exception if db error occurs
cert_id = self._import_certificate(repo.cert_name, repo.ca_cert,
repo.cert, repo.key)
else:
cert_id = None
if repo.basearch:
# will raise exception if db error occurs
basearch_id = self._import_basearch(repo.basearch)
else:
basearch_id = None
cur = self.conn.cursor()
try:
content_set_id = self.content_set_to_db_id[repo.content_set]
cur.execute(
"""select id, revision from repo where content_set_id = %s
and ((%s is null and basearch_id is null) or basearch_id = %s)
and ((%s is null and releasever is null) or releasever = %s)
""", (content_set_id, basearch_id, basearch_id,
repo.releasever, repo.releasever))
db_repo = cur.fetchone()
if not db_repo:
cur.execute(
"""insert into repo (url, content_set_id, basearch_id, releasever,
revision, eol, certificate_id)
values (%s, %s, %s, %s, %s, false, %s) returning id, revision""",
(
repo.repo_url,
content_set_id,
basearch_id,
repo.releasever,
repo.get_revision(),
cert_id,
))
db_repo = cur.fetchone()
else:
revision = repo.get_revision()
# if revision in repo object is None, re-use current revision from DB (don't update)
# this method is called from 2 different places with 2 different states of repo object
if not revision:
revision = db_repo[1]
cur.execute(
"""update repo set revision = %s, url = %s, certificate_id = %s where id = %s""",
(
revision,
repo.repo_url,
cert_id,
db_repo[0],
))
self.conn.commit()
return db_repo[0]
except Exception:
self.logger.exception("Failed to import or update repository.")
self.conn.rollback()
raise
finally:
cur.close()
def store(self, repository):
"""
Store single repository content into DB.
First, basic repository info is processed, then all packages, then all updates.
Some steps may be skipped if given data doesn't exist or are already synced.
"""
try:
repo_id = self.import_repository(repository)
self.package_store.store(repo_id, repository.list_packages())
self.module_store.store(repo_id, repository.list_modules())
self.update_store.store(repo_id, repository.list_updates())
except Exception: #pylint: disable=broad-except
# exception already logged.
pass
class OvalStore(ObjectStore): # pylint: disable=too-many-instance-attributes
"""
Class providing interface for fetching/importing OVAL data from/into the DB.
"""
OVAL_FEED_UPDATED_KEY = 'redhatovalfeed:updated'
# Not in DB table like other operations because we don't need this information further
SUPPORTED_ARCH_OPERATIONS = ["equals", "pattern match"]
def __init__(self):
super().__init__()
self.cpe_store = CpeStore()
self.package_store = PackageStore()
self.evr_operation_map = self._prepare_table_map(
cols=["name"], table="oval_operation_evr")
self.cve_map = self._prepare_table_map(cols=["name"], table="cve")
self.errata_map = self._prepare_table_map(cols=["name"],
table="errata")
self.oval_file_map = self._prepare_table_map(cols=["oval_id"],
to_cols=["id", "updated"],
table="oval_file")
self.oval_check_map = self._prepare_table_map(
cols=["name"], table="oval_check_rpminfo")
self.oval_check_existence_map = self._prepare_table_map(
cols=["name"], table="oval_check_existence_rpminfo")
self.oval_object_map = {}
self.oval_state_map = {}
self.oval_test_map = {}
self.oval_module_test_map = {}
self.oval_definition_map = {}
self.oval_definition_type_map = self._prepare_table_map(
cols=["name"], table="oval_definition_type")
self.oval_criteria_operator_map = self._prepare_table_map(
cols=["name"], table="oval_criteria_operator")
def save_lastmodified(self, lastmodified):
"""Store OVAL file timestamp."""
lastmodified = format_datetime(lastmodified)
cur = self.conn.cursor()
# Update timestamp
cur.execute("update metadata set value = %s where key = %s", (
lastmodified,
self.OVAL_FEED_UPDATED_KEY,
))
if cur.rowcount < 1:
cur.execute("insert into metadata (key, value) values (%s, %s)",
(self.OVAL_FEED_UPDATED_KEY, lastmodified))
cur.close()
self.conn.commit()
def delete_oval_file(self, oval_id):
"""
Deletes oval file from DB.
"""
db_id = self.oval_file_map[oval_id][0]
cur = self.conn.cursor()
try:
cur.execute(
"""delete from oval_criteria_dependency
where dep_test_id in (select id from oval_rpminfo_test where file_id = %s)""",
(db_id, ))
cur.execute(
"""delete from oval_criteria_dependency
where dep_module_test_id in (select id from oval_module_test where file_id = %s)""",
(db_id, ))
cur.execute(
"""delete from oval_rpminfo_test_state
where rpminfo_test_id in (select id from oval_rpminfo_test where file_id = %s)""",
(db_id, ))
cur.execute(
"""delete from oval_definition_test
where rpminfo_test_id in (select id from oval_rpminfo_test where file_id = %s)""",
(db_id, ))
cur.execute(
"""delete from oval_definition_cve
where definition_id in (select id from oval_definition where file_id = %s)""",
(db_id, ))
cur.execute(
"""delete from oval_definition_errata
where definition_id in (select id from oval_definition where file_id = %s)""",
(db_id, ))
cur.execute(
"""delete from oval_definition_cpe
where definition_id in (select id from oval_definition where file_id = %s)""",
(db_id, ))
cur.execute(
"""delete from oval_rpminfo_state_arch
where rpminfo_state_id in (select id from oval_rpminfo_state where file_id = %s)""",
(db_id, ))
cur.execute("delete from oval_definition where file_id = %s",
(db_id, ))
cur.execute("delete from oval_rpminfo_test where file_id = %s",
(db_id, ))
cur.execute("delete from oval_module_test where file_id = %s",
(db_id, ))
cur.execute("delete from oval_rpminfo_state where file_id = %s",
(db_id, ))
cur.execute("delete from oval_rpminfo_object where file_id = %s",
(db_id, ))
cur.execute("delete from oval_file where id = %s", (db_id, ))
self.conn.commit()
except Exception: # pylint: disable=broad-except
OVAL_FAILED_DELETE.inc()
self.logger.exception("Failed to delete oval file.")
self.conn.rollback()
finally:
cur.close()
def _save_oval_file_updated(self, oval_id, updated):
cur = self.conn.cursor()
# Update timestamp
cur.execute(
"update oval_file set updated = %s where oval_id = %s returning id",
(
updated,
oval_id,
))
if cur.rowcount < 1:
cur.execute(
"insert into oval_file (oval_id, updated) values (%s, %s) returning id",
(
oval_id,
updated,
))
db_id_row = cur.fetchone()
cur.close()
self.conn.commit()
return db_id_row[0]
def _populate_data(self, entity, file_id, data, item_check_func,
table_name, cols, refresh_maps_func,
items_to_delete_func, update_tmpl_str):
"""Generic method to populate table with OVAL entity (objects, states, tests, etc.)."""
to_insert = []
to_update = []
for item in data:
to_insert_row, to_update_row = item_check_func(file_id, item)
if to_insert_row:
to_insert.append(to_insert_row)
if to_update_row:
to_update.append(to_update_row)
to_delete = items_to_delete_func(file_id,
{item["id"]
for item in data})
self.logger.debug("OVAL %s to insert: %d", entity, len(to_insert))
self.logger.debug("OVAL %s to update: %d", entity, len(to_update))
self.logger.debug("OVAL %s to delete: %d", entity, len(to_delete))
try:
cur = self.conn.cursor()
if to_insert:
execute_values(
cur,
f"""insert into {table_name} ({', '.join(cols)}) values %s
returning id, {', '.join(cols)}""",
to_insert,
page_size=len(to_insert))
refresh_maps_func(cur)
if to_update:
execute_values(
cur,
f"""update {table_name} set {', '.join([f'{col} = v.{col}' for col in cols])}
from (values %s) as v(id, {', '.join(cols)})
where {table_name}.id = v.id
returning {table_name}.id, {', '.join([f'{table_name}.{col}' for col in cols])}
""",
to_update,
page_size=len(to_update),
template=update_tmpl_str)
refresh_maps_func(cur)
if to_delete:
cur.execute(
f"""delete from {table_name} where file_id = %s and oval_id in %s
returning file_id, oval_id""",
(file_id, tuple(to_delete)))
refresh_maps_func(cur, delete=True)
self.conn.commit()
except Exception: # pylint: disable=broad-except
OVAL_FAILED_IMPORT.inc()
OVAL_FAILED_UPDATE.inc()
OVAL_FAILED_DELETE.inc()
self.logger.exception(
"Failure while inserting, updating or deleting %s data: ",
entity)
self.conn.rollback()
finally:
cur.close()
def _populate_associations(self,
entity_one,
entity_many,
ent_one_id_col,
ent_many_id_col,
table_name,
ent_one_id,
ent_many_data,
import_check_map,
file_id=None,
missing_ok=False):
"""Associate/disassociate many_entities (objects, states, archs) with entity_one (state, test)."""
try:
cur = self.conn.cursor()
associated_with_entity_one = set()
cur.execute(
f"select {ent_many_id_col} from {table_name} where {ent_one_id_col} = %s",
(ent_one_id, ))
for row in cur.fetchall():
associated_with_entity_one.add(row[0])
self.logger.debug("OVAL %s associated with %s %s: %d", entity_many,
entity_one, ent_one_id,
len(associated_with_entity_one))
to_associate = []
for item in ent_many_data:
checked_key = (file_id, item["id"]) if file_id else item["id"]
item_id = import_check_map.get(checked_key)
if not item_id:
if not missing_ok:
self.logger.warning("Item (%s) not found: %s",
entity_many, checked_key)
continue
if isinstance(item_id, tuple):
item_id = item_id[0]
if item_id in associated_with_entity_one:
associated_with_entity_one.remove(item_id)
else:
to_associate.append(item_id)
self.logger.debug("New OVAL %s to associate with %s %s: %d",
entity_many, entity_one, ent_one_id,
len(to_associate))
self.logger.debug("OVAL %s to disassociate with %s %s: %d",
entity_many, entity_one, ent_one_id,
len(associated_with_entity_one))
if to_associate:
execute_values(
cur,
f"insert into {table_name} ({ent_one_id_col}, {ent_many_id_col}) values %s",
[(ent_one_id, item_id) for item_id in to_associate],
page_size=len(to_associate))
if associated_with_entity_one:
cur.execute(
f"delete from {table_name} where {ent_one_id_col} = %s and {ent_many_id_col} in %s",
(
ent_one_id,
tuple(associated_with_entity_one),
))
self.conn.commit()
except Exception: # pylint: disable=broad-except
OVAL_FAILED_IMPORT.inc()
OVAL_FAILED_DELETE.inc()
self.logger.exception(
"Failure while (dis)associating OVAL %s with %s %s: ",
entity_many, entity_one, ent_one_id)
self.conn.rollback()
finally:
cur.close()
def _object_import_check(self, file_id, item):
"""Check if object is in DB to insert/update."""
name_id = self.package_store.package_name_map.get(item["name"])
if not name_id:
self.logger.warning("Package name not found: %s", item["name"])
return None, None
to_insert_row = to_update_row = None
current_version = self.oval_object_map.get((file_id, item["id"]))
if current_version:
# Data in DB are different
if current_version[1:] != (name_id, item["version"]):
to_update_row = (current_version[0], file_id, item["id"],
name_id, item["version"])
else:
to_insert_row = (file_id, item["id"], name_id, item["version"])
return to_insert_row, to_update_row
def _object_refresh_maps(self, cur, delete=False):
"""Add imported data to caches."""
if not delete:
for obj_id, file_id, oval_id, name_id, version in cur.fetchall():
self.oval_object_map[(file_id, oval_id)] = (obj_id, name_id,
version)
else:
for file_id, oval_id in cur.fetchall():
del self.oval_object_map[(file_id, oval_id)]
def _objects_to_delete(self, file_id, latest_data):
"""Get list of oval_ids which are in DB but not in latest file."""
return [
oval_id for (f_id, oval_id) in self.oval_object_map
if f_id == file_id and oval_id not in latest_data
]
def _populate_objects(self, oval_file_id, objects):
self.oval_object_map = self._prepare_table_map(
cols=["file_id", "oval_id"],
to_cols=["id", "package_name_id", "version"],
table="oval_rpminfo_object",
where=f"file_id = {oval_file_id}")
# Populate missing package names
self.package_store.populate_dep_table(
"package_name", {obj["name"]
for obj in objects},
self.package_store.package_name_map)
# Insert/update data
self._populate_data(
"objects", oval_file_id, objects, self._object_import_check,
"oval_rpminfo_object",
["file_id", "oval_id", "package_name_id", "version"],
self._object_refresh_maps, self._objects_to_delete,
"(%s::int, %s::int, %s, %s::int, %s::int)")
def _state_import_check(self, file_id, item):
"""Check if state is in DB to insert/update."""
evr_id = evr_operation_id = None
if item['evr'] is not None:
epoch, version, release = item['evr']
evr_id = self.package_store.evr_map.get((epoch, version, release))
if not evr_id:
self.logger.warning("EVR not found: %s, %s, %s", epoch,
version, release)
return None, None
if item["evr_operation"] is not None:
evr_operation_id = self.evr_operation_map.get(
item["evr_operation"])
if not evr_operation_id:
self.logger.warning("Unsupported EVR operation: %s",
item["evr_operation"])
return None, None
to_insert_row = to_update_row = None
current_version = self.oval_state_map.get((file_id, item["id"]))
if current_version:
# Data in DB are different
if current_version[1:] != (evr_id, evr_operation_id,
item["version"]):
to_update_row = (current_version[0], file_id, item["id"],
evr_id, evr_operation_id, item["version"])
else:
to_insert_row = (file_id, item["id"], evr_id, evr_operation_id,
item["version"])
return to_insert_row, to_update_row
def _state_refresh_maps(self, cur, delete=False):
"""Add imported data to caches."""
if not delete:
for state_id, file_id, oval_id, evr_id, evr_operation_id, version in cur.fetchall(
):
self.oval_state_map[(file_id,
oval_id)] = (state_id, evr_id,
evr_operation_id, version)
else:
for file_id, oval_id in cur.fetchall():
del self.oval_state_map[(file_id, oval_id)]
def _states_to_delete(self, file_id, latest_data):
"""Get list of oval_ids which are in DB but not in latest file."""
return [
oval_id for (f_id, oval_id) in self.oval_state_map
if f_id == file_id and oval_id not in latest_data
]
def _populate_states(self, oval_file_id, states):
self.oval_state_map = self._prepare_table_map(
cols=["file_id", "oval_id"],
to_cols=["id", "evr_id", "evr_operation_id", "version"],
table="oval_rpminfo_state",
where=f"file_id = {oval_file_id}")
# Parse EVR first
for state in states:
if state['evr'] is not None:
# FIXME: as an input to common.rpm.parse_rpm_name, we don't have function to parse evr only
fake_nevra = f"pn-{state['evr']}.noarch"
_, epoch, version, release, _ = parse_rpm_name(fake_nevra)
state['evr'] = (epoch, version, release)
# Populate missing EVRs
self.package_store.populate_evrs(
{state['evr']
for state in states if state['evr'] is not None})
# Insert/update data
self._populate_data(
"states", oval_file_id, states, self._state_import_check,
"oval_rpminfo_state",
["file_id", "oval_id", "evr_id", "evr_operation_id", "version"],
self._state_refresh_maps, self._states_to_delete,
"(%s::int, %s::int, %s, %s::int, %s::int, %s::int)")
for state in states:
if state["arch_operation"] is not None and state[
"arch_operation"] not in self.SUPPORTED_ARCH_OPERATIONS:
self.logger.warning("Unsupported arch operation: %s",
state["arch_operation"])
continue
if (oval_file_id, state["id"]
) in self.oval_state_map: # Make sure state is imported
# Simplified logic, can contain any regex but RH oval files contains only logical OR
archs = []
if state["arch"] is not None:
archs.extend([{
"id": arch
} for arch in state["arch"].split("|")])
self._populate_associations(
"state", "archs", "rpminfo_state_id", "arch_id",
"oval_rpminfo_state_arch",
self.oval_state_map[(oval_file_id, state["id"])][0], archs,
self.package_store.arch_map)
def _test_import_check(self, file_id, item):
"""Check if test is in DB to insert/update."""
rpminfo_object_id = self.oval_object_map.get((file_id, item["object"]))
if not rpminfo_object_id:
self.logger.warning("OVAL object not found: %s", item["object"])
return None, None
rpminfo_object_id = rpminfo_object_id[0]
check_id = self.oval_check_map.get(item["check"])
if not check_id:
self.logger.warning("OVAL check not found: %s", item["check"])
return None, None
check_existence_id = self.oval_check_existence_map.get(
item["check_existence"])
if not check_existence_id:
self.logger.warning("OVAL check_existence not found: %s",
item["check_existence"])
return None, None
to_insert_row = to_update_row = None
current_version = self.oval_test_map.get((file_id, item["id"]))
if current_version:
# Data in DB are different
if current_version[1:] != (rpminfo_object_id, check_id,
check_existence_id, item["version"]):
to_update_row = (current_version[0], file_id, item["id"],
rpminfo_object_id, check_id,
check_existence_id, item["version"])
else:
to_insert_row = (file_id, item["id"], rpminfo_object_id, check_id,
check_existence_id, item["version"])
return to_insert_row, to_update_row
def _test_refresh_maps(self, cur, delete=False):
"""Add imported data to caches."""
if not delete:
for test_id, file_id, oval_id, rpminfo_object_id, check_id, check_existence_id, version in cur.fetchall(
):
self.oval_test_map[(file_id,
oval_id)] = (test_id, rpminfo_object_id,
check_id, check_existence_id,
version)
else:
for file_id, oval_id in cur.fetchall():
del self.oval_test_map[(file_id, oval_id)]
def _tests_to_delete(self, file_id, latest_data):
"""Get list of oval_ids which are in DB but not in latest file."""
return [
oval_id for (f_id, oval_id) in self.oval_test_map
if f_id == file_id and oval_id not in latest_data
]
def _populate_tests(self, oval_file_id, tests):
self.oval_test_map = self._prepare_table_map(
cols=["file_id", "oval_id"],
to_cols=[
"id", "rpminfo_object_id", "check_id", "check_existence_id",
"version"
],
table="oval_rpminfo_test",
where=f"file_id = {oval_file_id}")
# Insert/update data
self._populate_data(
"tests", oval_file_id, tests, self._test_import_check,
"oval_rpminfo_test", [
"file_id", "oval_id", "rpminfo_object_id", "check_id",
"check_existence_id", "version"
], self._test_refresh_maps, self._tests_to_delete,
"(%s::int, %s::int, %s, %s::int, %s::int, %s::int, %s::int)")
for test in tests:
if (oval_file_id, test["id"]
) in self.oval_test_map: # Make sure test is imported
states = [{"id": state} for state in test["states"]]
self._populate_associations(
"test",
"states",
"rpminfo_test_id",
"rpminfo_state_id",
"oval_rpminfo_test_state",
self.oval_test_map[(oval_file_id, test["id"])][0],
states,
self.oval_state_map,
file_id=oval_file_id)
def _module_test_import_check(self, file_id, item):
"""Check if module test is in DB to insert/update."""
to_insert_row = to_update_row = None
current_version = self.oval_module_test_map.get((file_id, item["id"]))
if current_version:
# Data in DB are different
if current_version[1:] != (item["module_stream"], item["version"]):
to_update_row = (current_version[0], file_id, item["id"],
item["module_stream"], item["version"])
else:
to_insert_row = (file_id, item["id"], item["module_stream"],
item["version"])
return to_insert_row, to_update_row
def _module_test_refresh_maps(self, cur, delete=False):
"""Add imported data to caches."""
if not delete:
for test_id, file_id, oval_id, module_stream, version in cur.fetchall(
):
self.oval_module_test_map[(file_id,
oval_id)] = (test_id, module_stream,
version)
else:
for file_id, oval_id in cur.fetchall():
del self.oval_module_test_map[(file_id, oval_id)]
def _module_tests_to_delete(self, file_id, latest_data):
"""Get list of oval_ids which are in DB but not in latest file."""
return [
oval_id for (f_id, oval_id) in self.oval_module_test_map
if f_id == file_id and oval_id not in latest_data
]
def _populate_module_tests(self, oval_file_id, module_tests):
self.oval_module_test_map = self._prepare_table_map(
cols=["file_id", "oval_id"],
to_cols=["id", "module_stream", "version"],
table="oval_module_test",
where=f"file_id = {oval_file_id}")
# Insert/update data
self._populate_data("module-tests", oval_file_id, module_tests,
self._module_test_import_check, "oval_module_test",
["file_id", "oval_id", "module_stream", "version"],
self._module_test_refresh_maps,
self._module_tests_to_delete,
"(%s::int, %s::int, %s, %s, %s::int)")
def _populate_definition_criteria(self, cur, file_id, criteria,
current_criteria_id):
# pylint: disable=too-many-branches, too-many-statements
operator_id = self.oval_criteria_operator_map.get(criteria["operator"])
if not operator_id:
self.logger.warning("OVAL criteria operator not found: %s",
criteria["operator"])
return None
current_dep_criteria = []
current_dep_tests = set()
current_dep_module_tests = set()
# Update type in current criteria row and fetch current dependencies
# If current_criteria_id is None in args, just insert it
if current_criteria_id:
cur.execute(
"update oval_criteria set operator_id = %s where id = %s",
(operator_id, current_criteria_id))
cur.execute(
"""select dep_criteria_id, dep_test_id, dep_module_test_id
from oval_criteria_dependency
where parent_criteria_id = %s""",
(current_criteria_id, ))
for dep_criteria_id, dep_test_id, dep_module_test_id in cur.fetchall(
):
if dep_criteria_id:
current_dep_criteria.append(dep_criteria_id)
if dep_test_id:
current_dep_tests.add(dep_test_id)
if dep_module_test_id:
current_dep_module_tests.add(dep_module_test_id)
criteria_id = current_criteria_id
else:
cur.execute(
"insert into oval_criteria (operator_id) values (%s) returning id",
(operator_id, ))
criteria_id = cur.fetchone()[0]
# Find out which test dependencies to insert and which to remove
dependencies_to_import = []
for test in criteria["criterions"]:
test_id = self.oval_test_map.get((file_id, test))
module_test_id = self.oval_module_test_map.get((file_id, test))
if test_id: # Unsuported test type may not be imported (rpmverifyfile etc.)
test_id = test_id[0]
if test_id not in current_dep_tests:
dependencies_to_import.append(
(criteria_id, None, test_id, None))
else:
current_dep_tests.remove(test_id)
if module_test_id:
module_test_id = module_test_id[0]
if module_test_id not in current_dep_module_tests:
dependencies_to_import.append(
(criteria_id, None, None, module_test_id))
else:
current_dep_module_tests.remove(module_test_id)
# Re-use dependent criteria ids when inserting/updating dependant criteria recursively
current_dep_criteria = sorted(current_dep_criteria)
for child_criteria in criteria["criteria"]:
if current_dep_criteria:
child_criteria_id = current_dep_criteria.pop(0)
child_criteria_id = self._populate_definition_criteria(
cur, file_id, child_criteria,
child_criteria_id) # Recursion
else:
child_criteria_id = None
child_criteria_id = self._populate_definition_criteria(
cur, file_id, child_criteria,
child_criteria_id) # Recursion
dependencies_to_import.append(
(criteria_id, child_criteria_id, None, None))
# Import dependencies
if dependencies_to_import:
execute_values(cur,
"""insert into oval_criteria_dependency
(parent_criteria_id, dep_criteria_id, dep_test_id, dep_module_test_id)
values %s""",
dependencies_to_import,
page_size=len(dependencies_to_import))
# Remove no longer needed dependencies
if current_dep_tests:
cur.execute(
"""delete from oval_criteria_dependency
where parent_criteria_id = %s and dep_test_id in %s""",
(criteria_id, tuple(current_dep_tests)))
if current_dep_module_tests:
cur.execute(
"""delete from oval_criteria_dependency
where parent_criteria_id = %s and dep_module_test_id in %s""",
(criteria_id, tuple(current_dep_module_tests)))
if current_dep_criteria:
cur.execute(
"""delete from oval_criteria_dependency
where parent_criteria_id = %s and dep_criteria_id in %s""",
(criteria_id, tuple(current_dep_criteria)))
return criteria_id
def _definition_import_check(self, file_id, item):
"""Check if definition is in DB to insert/update."""
definition_type = self.oval_definition_type_map.get(item["type"])
if not definition_type:
self.logger.warning("OVAL definition type not found: %s",
item["type"])
return None, None
to_insert_row = to_update_row = None
current_version = self.oval_definition_map.get((file_id, item["id"]))
if current_version:
criteria_id = current_version[2]
else:
criteria_id = None
try:
cur = self.conn.cursor()
criteria_id = self._populate_definition_criteria(
cur, file_id, item["criteria"], criteria_id)
self.conn.commit()
except Exception: # pylint: disable=broad-except
OVAL_FAILED_IMPORT.inc()
OVAL_FAILED_UPDATE.inc()
OVAL_FAILED_DELETE.inc()
self.logger.exception("Failure while inserting criteria: ")
self.conn.rollback()
if current_version:
# Data in DB are different
if current_version[1:] != (definition_type, criteria_id,
item["version"]):
to_update_row = (current_version[0], file_id, item["id"],
definition_type, criteria_id, item["version"])
else:
to_insert_row = (file_id, item["id"], definition_type, criteria_id,
item["version"])
return to_insert_row, to_update_row
def _definition_refresh_maps(self, cur, delete=False):
"""Add imported data to caches."""
if not delete:
for definition_id, file_id, oval_id, definition_type_id, criteria_id, version in cur.fetchall(
):
self.oval_definition_map[(file_id,
oval_id)] = (definition_id,
definition_type_id,
criteria_id, version)
else:
for file_id, oval_id in cur.fetchall():
del self.oval_definition_map[(file_id, oval_id)]
def _definitions_to_delete(self, file_id, latest_data):
"""Get list of oval_ids which are in DB but not in latest file."""
return [
oval_id for (f_id, oval_id) in self.oval_definition_map
if f_id == file_id and oval_id not in latest_data
]
def _populate_definitions(self, oval_file_id, definitions):
self.oval_definition_map = self._prepare_table_map(
cols=["file_id", "oval_id"],
to_cols=["id", "definition_type_id", "criteria_id", "version"],
table="oval_definition",
where=f"file_id = {oval_file_id}")
# Insert/update data
self._populate_data(
"definitions", oval_file_id, definitions,
self._definition_import_check, "oval_definition", [
"file_id", "oval_id", "definition_type_id", "criteria_id",
"version"
], self._definition_refresh_maps, self._definitions_to_delete,
"(%s::int, %s::int, %s, %s::int, %s::int, %s::int)")
for definition in definitions:
if (
oval_file_id, definition["id"]
) in self.oval_definition_map: # Make sure definition is imported
cves = [{"id": cve} for cve in definition["cves"]]
advisories = [{
"id": advisory
} for advisory in definition["advisories"]]
cpes = [{"id": cpe} for cpe in definition["cpes"]]
tests = [{
"id": test
} for test in list(set(definition["tests"]))] # Make unique
# Store missing CPEs (they are often substrings of CPEs already in DB)
self.cpe_store.populate_cpes({
cpe: None
for cpe in definition["cpes"]
if cpe not in self.cpe_store.cpe_label_to_id
})
if not cves:
self.logger.warning(
"OVAL definition has empty CVE list: %s",
definition["id"])
if not advisories and definition["type"] != "vulnerability":
self.logger.warning(
"OVAL definition has empty errata list: %s",
definition["id"])
if not tests:
self.logger.warning(
"OVAL definition has empty test list: %s",
definition["id"])
definition_id = self.oval_definition_map[(oval_file_id,
definition["id"])][0]
self._populate_associations("definition", "cves",
"definition_id", "cve_id",
"oval_definition_cve",
definition_id, cves, self.cve_map)
self._populate_associations("definition", "advisories",
"definition_id", "errata_id",
"oval_definition_errata",
definition_id, advisories,
self.errata_map)
self._populate_associations("definition", "cpes",
"definition_id", "cpe_id",
"oval_definition_cpe",
definition_id, cpes,
self.cpe_store.cpe_label_to_id)
self._populate_associations(
"definition",
"tests",
"definition_id",
"rpminfo_test_id",
"oval_definition_test",
definition_id,
tests,
self.oval_test_map,
file_id=oval_file_id,
missing_ok=True
) # Don't log unsupported test types (rpmverifyfile etc.)
def store(self, oval_file):
"""Store single OVAL definitions file into DB."""
oval_file_id = self._save_oval_file_updated(oval_file.oval_id,
oval_file.updated)
self._populate_objects(oval_file_id, oval_file.objects)
self._populate_states(oval_file_id, oval_file.states)
self._populate_tests(oval_file_id, oval_file.tests)
self._populate_module_tests(oval_file_id, oval_file.module_tests)
self._populate_definitions(oval_file_id, oval_file.definitions)