def __init__(self):
super().__init__()
self.cpe_store = CpeStore()
self.package_store = PackageStore()
self.evr_operation_map = self._prepare_table_map(
cols=["name"], table="oval_operation_evr")
self.cve_map = self._prepare_table_map(cols=["name"], table="cve")
self.errata_map = self._prepare_table_map(cols=["name"],
table="errata")
self.oval_check_map = self._prepare_table_map(
cols=["name"], table="oval_check_rpminfo")
self.oval_check_existence_map = self._prepare_table_map(
cols=["name"], table="oval_check_existence_rpminfo")
self.oval_object_map = self._prepare_table_map(
cols=["oval_id"], table="oval_rpminfo_object")
self.oval_object_version_map = self._prepare_table_map(
cols=["oval_id"], table="oval_rpminfo_object", to_col="version")
self.oval_state_map = self._prepare_table_map(
cols=["oval_id"], table="oval_rpminfo_state")
self.oval_state_version_map = self._prepare_table_map(
cols=["oval_id"], table="oval_rpminfo_state", to_col="version")
self.oval_test_map = self._prepare_table_map(cols=["oval_id"],
table="oval_rpminfo_test")
self.oval_test_version_map = self._prepare_table_map(
cols=["oval_id"], table="oval_rpminfo_test", to_col="version")
self.oval_definition_map = self._prepare_table_map(
cols=["oval_id"], table="oval_definition")
self.oval_definition_version_map = self._prepare_table_map(
cols=["oval_id"], table="oval_definition", to_col="version")
self.oval_definition_type_map = self._prepare_table_map(
cols=["name"], table="oval_definition_type")
self.oval_criteria_operator_map = self._prepare_table_map(
cols=["name"], table="oval_criteria_operator")
def __init__(self):
self.logger = get_logger(__name__)
self.conn = DatabaseHandler.get_connection()
self.module_store = ModulesStore()
self.package_store = PackageStore()
self.update_store = UpdateStore()
self.content_set_to_db_id = self._prepare_content_set_map()
class RepositoryStore:
"""
Class providing interface for listing repositories stored in DB and storing repositories one by one.
"""
def __init__(self):
self.logger = get_logger(__name__)
self.conn = DatabaseHandler.get_connection()
self.module_store = ModulesStore()
self.package_store = PackageStore()
self.update_store = UpdateStore()
self.content_set_to_db_id = self._prepare_content_set_map()
def _prepare_content_set_map(self):
cur = self.conn.cursor()
cur.execute("""select id, label from content_set""")
content_sets = {}
for cs_id, cs_label in cur.fetchall():
content_sets[cs_label] = cs_id
cur.close()
return content_sets
def list_repositories(self):
"""List repositories stored in DB. Dictionary with repository label as key is returned."""
cur = self.conn.cursor()
cur.execute("""select cs.label, a.name, r.releasever, r.id, r.url, r.revision, cs.id,
c.name, c.ca_cert, c.cert, c.key
from repo r
left join arch a on r.basearch_id = a.id
left join certificate c on r.certificate_id = c.id
left join content_set cs on r.content_set_id = cs.id""")
repos = {}
for row in cur.fetchall():
# (content_set_label, repo_arch, repo_releasever) -> repo_id, repo_url, repo_revision...
repos[(row[0], row[1], row[2])] = {"id": row[3], "url": row[4], "revision": row[5],
"content_set_id": row[6], "cert_name": row[7], "ca_cert": row[8],
"cert": row[9], "key": row[10]}
cur.close()
return repos
def _import_basearch(self, basearch):
cur = self.conn.cursor()
try:
cur.execute("select id from arch where name = %s", (basearch,))
arch_id = cur.fetchone()
if not arch_id:
cur.execute("insert into arch (name) values(%s) returning id", (basearch,))
arch_id = cur.fetchone()
self.conn.commit()
except Exception:
self.logger.exception("Failed to import basearch.")
self.conn.rollback()
raise
finally:
cur.close()
return arch_id[0]
def _import_certificate(self, cert_name, ca_cert, cert, key):
if not key:
key = None
cur = self.conn.cursor()
try:
cur.execute("select id from certificate where name = %s", (cert_name,))
cert_id = cur.fetchone()
if not cert_id:
cur.execute("""insert into certificate (name, ca_cert, cert, key)
values (%s, %s, %s, %s) returning id""", (cert_name, ca_cert, cert, key,))
cert_id = cur.fetchone()
else:
cur.execute("update certificate set ca_cert = %s, cert = %s, key = %s where name = %s",
(ca_cert, cert, key, cert_name,))
self.conn.commit()
except Exception:
self.logger.exception("Failed to import certificate.")
self.conn.rollback()
raise
finally:
cur.close()
return cert_id[0]
def cleanup_unused_data(self):
"""
Deletes packages and errata not associated with any repo etc.
"""
cur = self.conn.cursor()
try:
cur.execute("""select p.id from package p where not exists (
select 1 from pkg_repo pr where pr.pkg_id = p.id
) and p.source_package_id is not null
""")
packages_to_delete = cur.fetchall()
cur.execute("""select e.id from errata e where not exists (
select 1 from errata_repo er where er.errata_id = e.id
)
""")
updates_to_delete = cur.fetchall()
if packages_to_delete:
cur.execute("""delete from pkg_errata pe where pe.pkg_id in %s""", (tuple(packages_to_delete),))
cur.execute("""delete from module_rpm_artifact mra where mra.pkg_id in %s""",
(tuple(packages_to_delete),))
cur.execute("""delete from package p where p.id in %s""", (tuple(packages_to_delete),))
if updates_to_delete:
cur.execute("""delete from pkg_errata pe where pe.errata_id in %s""", (tuple(updates_to_delete),))
cur.execute("""delete from errata_cve ec where ec.errata_id in %s""", (tuple(updates_to_delete),))
cur.execute("""delete from errata_refs er where er.errata_id in %s""", (tuple(updates_to_delete),))
cur.execute("""delete from errata e where e.id in %s""", (tuple(updates_to_delete),))
self.conn.commit()
except Exception: # pylint: disable=broad-except
self.logger.exception("Failed to clean up unused data.")
self.conn.rollback()
finally:
cur.close()
def delete_content_set(self, content_set_label):
"""
Deletes repositories and their content from DB.
"""
content_set_id = self.content_set_to_db_id[content_set_label]
cur = self.conn.cursor()
try:
cur.execute("""select id from repo where content_set_id = %s""", (content_set_id,))
repo_ids = cur.fetchall()
for repo_id in repo_ids:
cur.execute("select id from module where repo_id = %s", (repo_id,))
module_ids = cur.fetchall()
if module_ids:
cur.execute("select id from module_stream where module_id in %s", (tuple(module_ids),))
module_stream_ids = cur.fetchall()
if module_stream_ids:
cur.execute("""delete from module_profile_pkg
where profile_id in (select id from module_profile
where stream_id in %s)""", (tuple(module_stream_ids),))
cur.execute("delete from module_rpm_artifact where stream_id in %s",
(tuple(module_stream_ids),))
cur.execute("delete from pkg_errata where module_stream_id in %s", (tuple(module_stream_ids),))
cur.execute("delete from module_profile where stream_id in %s", (tuple(module_stream_ids),))
cur.execute("delete from module_stream where module_id in %s", (tuple(module_ids),))
cur.execute("delete from module where repo_id = %s", (repo_id,))
cur.execute("delete from pkg_repo where repo_id = %s", (repo_id,))
cur.execute("delete from errata_repo where repo_id = %s", (repo_id,))
cur.execute("delete from repo where id = %s", (repo_id,))
cur.execute("delete from content_set where id = %s", (content_set_id,))
self.conn.commit()
except Exception: # pylint: disable=broad-except
self.logger.exception("Failed to delete content set.")
self.conn.rollback()
finally:
cur.close()
def import_repository(self, repo):
"""
Imports or updates repository record in DB.
"""
if repo.ca_cert:
# will raise exception if db error occurs
cert_id = self._import_certificate(repo.cert_name, repo.ca_cert, repo.cert, repo.key)
else:
cert_id = None
if repo.basearch:
# will raise exception if db error occurs
basearch_id = self._import_basearch(repo.basearch)
else:
basearch_id = None
cur = self.conn.cursor()
try:
content_set_id = self.content_set_to_db_id[repo.content_set]
cur.execute("""select id, revision from repo where content_set_id = %s
and ((%s is null and basearch_id is null) or basearch_id = %s)
and ((%s is null and releasever is null) or releasever = %s)
""",
(content_set_id, basearch_id, basearch_id, repo.releasever, repo.releasever))
db_repo = cur.fetchone()
if not db_repo:
cur.execute("""insert into repo (url, content_set_id, basearch_id, releasever,
revision, eol, certificate_id)
values (%s, %s, %s, %s, %s, false, %s) returning id, revision""",
(repo.repo_url, content_set_id, basearch_id, repo.releasever,
repo.get_revision(), cert_id,))
db_repo = cur.fetchone()
else:
revision = repo.get_revision()
# if revision in repo object is None, re-use current revision from DB (don't update)
# this method is called from 2 different places with 2 different states of repo object
if not revision:
revision = db_repo[1]
cur.execute("""update repo set revision = %s, url = %s, certificate_id = %s where id = %s""",
(revision, repo.repo_url, cert_id, db_repo[0],))
self.conn.commit()
return db_repo[0]
except Exception:
self.logger.exception("Failed to import or update repository.")
self.conn.rollback()
raise
finally:
cur.close()
def store(self, repository):
"""
Store single repository content into DB.
First, basic repository info is processed, then all packages, then all updates.
Some steps may be skipped if given data doesn't exist or are already synced.
"""
try:
repo_id = self.import_repository(repository)
self.package_store.store(repo_id, repository.list_packages())
self.module_store.store(repo_id, repository.list_modules())
self.update_store.store(repo_id, repository.list_updates())
except Exception: # pylint: disable=broad-except
# exception already logged.
pass
class RepositoryStore:
"""
Class providing interface for listing repositories stored in DB and storing repositories one by one.
"""
def __init__(self):
self.logger = get_logger(__name__)
self.conn = DatabaseHandler.get_connection()
self.module_store = ModulesStore()
self.package_store = PackageStore()
self.update_store = UpdateStore()
self.content_set_to_db_id = self._prepare_content_set_map()
def _prepare_content_set_map(self):
cur = self.conn.cursor()
cur.execute("""select id, label from content_set""")
content_sets = {}
for cs_id, cs_label in cur.fetchall():
content_sets[cs_label] = cs_id
cur.close()
return content_sets
def list_repositories(self):
"""List repositories stored in DB. Dictionary with repository label as key is returned."""
cur = self.conn.cursor()
cur.execute("""select cs.label, a.name, r.releasever, r.id, r.url, r.revision, cs.id,
c.name, c.ca_cert, c.cert, c.key
from repo r
left join arch a on r.basearch_id = a.id
left join certificate c on r.certificate_id = c.id
left join content_set cs on r.content_set_id = cs.id""")
repos = {}
for row in cur.fetchall():
# (content_set_label, repo_arch, repo_releasever) -> repo_id, repo_url, repo_revision...
repos[(row[0], row[1], row[2])] = {"id": row[3], "url": row[4], "revision": row[5],
"content_set_id": row[6], "cert_name": row[7], "ca_cert": row[8],
"cert": row[9], "key": row[10]}
cur.close()
return repos
def _import_basearch(self, basearch):
cur = self.conn.cursor()
cur.execute("select id from arch where name = %s", (basearch,))
arch_id = cur.fetchone()
if not arch_id:
cur.execute("insert into arch (name) values(%s) returning id", (basearch,))
arch_id = cur.fetchone()
cur.close()
self.conn.commit()
return arch_id[0]
def _import_certificate(self, cert_name, ca_cert, cert, key):
if not key:
key = None
cur = self.conn.cursor()
cur.execute("select id from certificate where name = %s", (cert_name,))
cert_id = cur.fetchone()
if not cert_id:
cur.execute("""insert into certificate (name, ca_cert, cert, key)
values (%s, %s, %s, %s) returning id""", (cert_name, ca_cert, cert, key,))
cert_id = cur.fetchone()
else:
cur.execute("update certificate set ca_cert = %s, cert = %s, key = %s where name = %s",
(ca_cert, cert, key, cert_name,))
cur.close()
self.conn.commit()
return cert_id[0]
def cleanup_unused_data(self):
"""
Deletes packages and errata not associated with any repo etc.
"""
cur = self.conn.cursor()
cur.execute("""select p.id from package p where not exists (
select 1 from pkg_repo pr where pr.pkg_id = p.id
)
""")
packages_to_delete = [pkg_id for pkg_id in cur.fetchall()]
cur.execute("""select e.id from errata e where not exists (
select 1 from errata_repo er where er.errata_id = e.id
)
""")
updates_to_delete = [update_id for update_id in cur.fetchall()]
if packages_to_delete:
cur.execute("""delete from pkg_errata pe where pe.pkg_id in %s""", (tuple(packages_to_delete),))
cur.execute("""delete from package p where p.id in %s""", (tuple(packages_to_delete),))
if updates_to_delete:
cur.execute("""delete from pkg_errata pe where pe.errata_id in %s""", (tuple(updates_to_delete),))
cur.execute("""delete from errata_cve ec where ec.errata_id in %s""", (tuple(updates_to_delete),))
cur.execute("""delete from errata_refs er where er.errata_id in %s""", (tuple(updates_to_delete),))
cur.execute("""delete from errata e where e.id in %s""", (tuple(updates_to_delete),))
cur.close()
self.conn.commit()
def delete_content_set(self, content_set_label):
"""
Deletes repositories and their content from DB.
"""
content_set_id = self.content_set_to_db_id[content_set_label]
cur = self.conn.cursor()
cur.execute("""select id from repo where content_set_id = %s""", (content_set_id,))
repo_ids = [repo_id for repo_id in cur.fetchall()]
for repo_id in repo_ids:
cur.execute("delete from pkg_repo where repo_id = %s", (repo_id,))
cur.execute("delete from errata_repo where repo_id = %s", (repo_id,))
cur.execute("delete from repo where id = %s", (repo_id,))
cur.execute("delete from content_set where id = %s", (content_set_id,))
cur.close()
self.conn.commit()
def import_repository(self, repo):
"""
Imports or updates repository record in DB.
"""
if repo.ca_cert:
cert_id = self._import_certificate(repo.cert_name, repo.ca_cert, repo.cert, repo.key)
else:
cert_id = None
if repo.basearch:
basearch_id = self._import_basearch(repo.basearch)
else:
basearch_id = None
content_set_id = self.content_set_to_db_id[repo.content_set]
cur = self.conn.cursor()
cur.execute("""select id from repo where content_set_id = %s
and ((%s is null and basearch_id is null) or basearch_id = %s)
and ((%s is null and releasever is null) or releasever = %s)
""",
(content_set_id, basearch_id, basearch_id, repo.releasever, repo.releasever))
repo_id = cur.fetchone()
if not repo_id:
cur.execute("""insert into repo (url, content_set_id, basearch_id, releasever,
revision, eol, certificate_id)
values (%s, %s, %s, %s, %s, false, %s) returning id""",
(repo.repo_url, content_set_id, basearch_id, repo.releasever,
repo.get_revision(), cert_id,))
repo_id = cur.fetchone()
else:
# Update repository timestamp
cur.execute("""update repo set revision = %s, certificate_id = %s, content_set_id = %s,
basearch_id = %s, releasever = %s
where id = %s""", (repo.get_revision(), cert_id, content_set_id, basearch_id,
repo.releasever, repo_id[0],))
cur.close()
self.conn.commit()
return repo_id[0]
def store(self, repository):
"""
Store single repository content into DB.
First, basic repository info is processed, then all packages, then all updates.
Some steps may be skipped if given data doesn't exist or are already synced.
"""
self.logger.info("Syncing repository: %s", ", ".join(filter(None, (repository.content_set, repository.basearch,
repository.releasever))))
repo_id = self.import_repository(repository)
self.package_store.store(repo_id, repository.list_packages())
self.module_store.store(repo_id, repository.list_modules())
self.update_store.store(repo_id, repository.list_updates())
class RepositoryStore:
"""
Class providing interface for listing repositories stored in DB and storing repositories one by one.
"""
def __init__(self):
self.content_set_to_db_id = {}
self.logger = SimpleLogger()
self.conn = DatabaseHandler.get_connection()
self.package_store = PackageStore()
self.update_store = UpdateStore()
def set_content_set_db_mapping(self, content_set_to_db_id):
"""Set content set to DB is mapping from product_store"""
self.content_set_to_db_id = content_set_to_db_id
def _get_content_set_id(self, repo):
if repo.content_set in self.content_set_to_db_id:
return self.content_set_to_db_id[repo.content_set]
return None
def list_repositories(self):
"""List repositories stored in DB. Dictionary with repository label as key is returned."""
cur = self.conn.cursor()
cur.execute(
"""select r.id, r.label, r.url, r.revision, cs.id, cs.label, c.name, c.ca_cert, c.cert, c.key
from repo r
left join certificate c on r.certificate_id = c.id
left join content_set cs on r.content_set_id = cs.id""")
repos = {}
for row in cur.fetchall():
# repo_label -> repo_id, repo_url, repo_revision
repos[row[1]] = {
"id": row[0],
"url": row[2],
"revision": row[3],
"content_set_id": row[4],
"content_set": row[5],
"cert_name": row[6],
"ca_cert": row[7],
"cert": row[8],
"key": row[9]
}
cur.close()
return repos
def _import_certificate(self, cert_name, ca_cert, cert, key):
cur = self.conn.cursor()
cur.execute("select id from certificate where name = %s",
(cert_name, ))
cert_id = cur.fetchone()
if not cert_id:
cur.execute(
"""insert into certificate (name, ca_cert, cert, key)
values (%s, %s, %s, %s) returning id""", (
cert_name,
ca_cert,
cert,
key,
))
cert_id = cur.fetchone()
else:
cur.execute(
"update certificate set ca_cert = %s, cert = %s, key = %s where name = %s",
(
ca_cert,
cert,
key,
cert_name,
))
cur.close()
self.conn.commit()
return cert_id[0]
def _import_repository(self, repo):
if repo.ca_cert:
cert_id = self._import_certificate(repo.cert_name, repo.ca_cert,
repo.cert, repo.key)
else:
cert_id = None
cur = self.conn.cursor()
cur.execute("select id from repo where label = %s",
(repo.repo_label, ))
repo_id = cur.fetchone()
content_set_id = self._get_content_set_id(repo)
if not repo_id:
cur.execute(
"""insert into repo (label, url, revision, eol, certificate_id, content_set_id)
values (%s, %s, to_timestamp(%s), false, %s, %s) returning id""",
(
repo.repo_label,
repo.repo_url,
repo.repomd.get_revision(),
cert_id,
content_set_id,
))
repo_id = cur.fetchone()
else:
# Update repository timestamp
cur.execute(
"""update repo set revision = to_timestamp(%s), certificate_id = %s, content_set_id = %s
where id = %s""", (
repo.repomd.get_revision(),
cert_id,
content_set_id,
repo_id[0],
))
cur.close()
self.conn.commit()
return repo_id[0]
def store(self, repository):
"""
Store single repository into DB.
First, basic repository info is processed, then all packages, then all updates.
Some steps may be skipped if given data doesn't exist or are already synced.
"""
self.logger.log("Syncing repository: %s" % repository.repo_label)
repo_id = self._import_repository(repository)
self.package_store.store(repo_id, repository.list_packages())
self.update_store.store(repo_id, repository.list_updates())
def __init__(self):
self.content_set_to_db_id = {}
self.logger = SimpleLogger()
self.conn = DatabaseHandler.get_connection()
self.package_store = PackageStore()
self.update_store = UpdateStore()
class OvalStore(ObjectStore): # pylint: disable=too-many-instance-attributes
"""
Class providing interface for fetching/importing OVAL data from/into the DB.
"""
OVAL_FEED_UPDATED_KEY = 'redhatovalfeed:updated'
# Not in DB table like other operations because we don't need this information further
SUPPORTED_ARCH_OPERATIONS = ["equals", "pattern match"]
def __init__(self):
super().__init__()
self.cpe_store = CpeStore()
self.package_store = PackageStore()
self.evr_operation_map = self._prepare_table_map(
cols=["name"], table="oval_operation_evr")
self.cve_map = self._prepare_table_map(cols=["name"], table="cve")
self.errata_map = self._prepare_table_map(cols=["name"],
table="errata")
self.oval_check_map = self._prepare_table_map(
cols=["name"], table="oval_check_rpminfo")
self.oval_check_existence_map = self._prepare_table_map(
cols=["name"], table="oval_check_existence_rpminfo")
self.oval_object_map = self._prepare_table_map(
cols=["oval_id"], table="oval_rpminfo_object")
self.oval_object_version_map = self._prepare_table_map(
cols=["oval_id"], table="oval_rpminfo_object", to_col="version")
self.oval_state_map = self._prepare_table_map(
cols=["oval_id"], table="oval_rpminfo_state")
self.oval_state_version_map = self._prepare_table_map(
cols=["oval_id"], table="oval_rpminfo_state", to_col="version")
self.oval_test_map = self._prepare_table_map(cols=["oval_id"],
table="oval_rpminfo_test")
self.oval_test_version_map = self._prepare_table_map(
cols=["oval_id"], table="oval_rpminfo_test", to_col="version")
self.oval_definition_map = self._prepare_table_map(
cols=["oval_id"], table="oval_definition")
self.oval_definition_version_map = self._prepare_table_map(
cols=["oval_id"], table="oval_definition", to_col="version")
self.oval_definition_type_map = self._prepare_table_map(
cols=["name"], table="oval_definition_type")
self.oval_criteria_operator_map = self._prepare_table_map(
cols=["name"], table="oval_criteria_operator")
def list_oval_definitions(self):
"""List oval definitions and their timestamps stored in DB. Dictionary with oval id as key is returned."""
cur = self.conn.cursor()
cur.execute("""select oval_id, updated from oval_file""")
return dict(cur.fetchall())
def save_lastmodified(self, lastmodified):
"""Store OVAL file timestamp."""
lastmodified = format_datetime(lastmodified)
cur = self.conn.cursor()
# Update timestamp
cur.execute("update metadata set value = %s where key = %s", (
lastmodified,
self.OVAL_FEED_UPDATED_KEY,
))
if cur.rowcount < 1:
cur.execute("insert into metadata (key, value) values (%s, %s)",
(self.OVAL_FEED_UPDATED_KEY, lastmodified))
cur.close()
self.conn.commit()
def _save_oval_file_updated(self, oval_id, updated):
cur = self.conn.cursor()
# Update timestamp
cur.execute(
"update oval_file set updated = %s where oval_id = %s returning id",
(
updated,
oval_id,
))
if cur.rowcount < 1:
cur.execute(
"insert into oval_file (oval_id, updated) values (%s, %s) returning id",
(
oval_id,
updated,
))
db_id_row = cur.fetchone()
cur.close()
self.conn.commit()
return db_id_row[0]
def _populate_data(self, entity, data, import_check_func, query,
refresh_maps_func):
"""Generic method to populate table with OVAL entity (objects, states, tests, etc.)."""
to_import = []
# Append only here, not delete rows, same item may be referenced from multiple files
for item in data:
row = import_check_func(item)
if row:
to_import.append(row)
self.logger.debug("OVAL %s to import: %d", entity, len(to_import))
if to_import:
try:
cur = self.conn.cursor()
execute_values(cur, query, to_import, page_size=len(to_import))
refresh_maps_func(cur)
self.conn.commit()
except Exception: # pylint: disable=broad-except
self.logger.exception("Failure while inserting %s data: ",
entity)
self.conn.rollback()
finally:
cur.close()
def _populate_associations(self,
entity_one,
entity_many,
ent_one_id_col,
ent_many_id_col,
table_name,
ent_one_id,
ent_many_data,
import_check_map,
missing_ok=False):
"""Associate/disassociate many_entities (objects, states, archs) with entity_one (file, state)."""
try:
cur = self.conn.cursor()
associated_with_entity_one = set()
cur.execute(
f"select {ent_many_id_col} from {table_name} where {ent_one_id_col} = %s",
(ent_one_id, ))
for row in cur.fetchall():
associated_with_entity_one.add(row[0])
self.logger.debug("OVAL %s associated with %s %s: %d", entity_many,
entity_one, ent_one_id,
len(associated_with_entity_one))
to_associate = []
for item in ent_many_data:
item_id = import_check_map.get(item["id"])
if not item_id:
if not missing_ok:
self.logger.warning("Item (%s) not found: %s",
entity_many, item["id"])
continue
if item_id in associated_with_entity_one:
associated_with_entity_one.remove(item_id)
else:
to_associate.append(item_id)
self.logger.debug("New OVAL %s to associate with %s %s: %d",
entity_many, entity_one, ent_one_id,
len(to_associate))
self.logger.debug("OVAL %s to disassociate with %s %s: %d",
entity_many, entity_one, ent_one_id,
len(associated_with_entity_one))
if to_associate:
execute_values(
cur,
f"insert into {table_name} ({ent_one_id_col}, {ent_many_id_col}) values %s",
[(ent_one_id, item_id) for item_id in to_associate],
page_size=len(to_associate))
if associated_with_entity_one:
cur.execute(
f"delete from {table_name} where {ent_one_id_col} = %s and {ent_many_id_col} in %s",
(
ent_one_id,
tuple(associated_with_entity_one),
))
self.conn.commit()
except Exception: # pylint: disable=broad-except
self.logger.exception(
"Failure while (dis)associating OVAL %s with %s %s: ",
entity_many, entity_one, ent_one_id)
self.conn.rollback()
finally:
cur.close()
def _object_import_check(self, item):
"""Check if object is in DB, return None if it's up to date and row to import otherwise."""
if item["version"] <= self.oval_object_version_map.get(item["id"], -1):
return None
name_id = self.package_store.package_name_map.get(item["name"])
if not name_id:
self.logger.warning("Package name not found: %s", item["name"])
return None
return item["id"], name_id, item["version"]
def _object_refresh_maps(self, cur):
"""Add imported data to caches."""
for obj_id, oval_id, version in cur.fetchall():
self.oval_object_map[oval_id] = obj_id
self.oval_object_version_map[oval_id] = version
def _populate_objects(self, oval_file_id, objects):
query = """insert into oval_rpminfo_object (oval_id, package_name_id, version) values %s
on conflict (oval_id) do update set
package_name_id = EXCLUDED.package_name_id, version = EXCLUDED.version
returning id, oval_id, version"""
# Populate missing package names
self.package_store.populate_dep_table(
"package_name", {obj["name"]
for obj in objects},
self.package_store.package_name_map)
self._populate_data("objects", objects, self._object_import_check,
query, self._object_refresh_maps)
self._populate_associations("file", "objects", "file_id",
"rpminfo_object_id",
"oval_file_rpminfo_object", oval_file_id,
objects, self.oval_object_map)
def _state_import_check(self, item):
"""Check if state is in DB, return None if it's up to date and row to import otherwise."""
if item["version"] <= self.oval_state_version_map.get(item["id"], -1):
return None
evr_id = evr_operation_id = None
if item['evr'] is not None:
epoch, version, release = item['evr']
evr_id = self.package_store.evr_map.get((epoch, version, release))
if not evr_id:
self.logger.warning("EVR not found: %s, %s, %s", epoch,
version, release)
return None
if item["evr_operation"] is not None:
evr_operation_id = self.evr_operation_map.get(
item["evr_operation"])
if not evr_operation_id:
self.logger.warning("Unsupported EVR operation: %s",
item["evr_operation"])
return None
return item["id"], evr_id, evr_operation_id, item["version"]
def _state_refresh_maps(self, cur):
"""Add imported data to caches."""
for state_id, oval_id, version in cur.fetchall():
self.oval_state_map[oval_id] = state_id
self.oval_state_version_map[oval_id] = version
def _populate_states(self, oval_file_id, states):
query = """insert into oval_rpminfo_state (oval_id, evr_id, evr_operation_id, version) values %s
on conflict (oval_id) do update set
evr_id = EXCLUDED.evr_id, evr_operation_id = EXCLUDED.evr_operation_id,
version = EXCLUDED.version
returning id, oval_id, version"""
# Parse EVR first
for state in states:
if state['evr'] is not None:
# FIXME: as an input to common.rpm.parse_rpm_name, we don't have function to parse evr only
fake_nevra = f"pn-{state['evr']}.noarch"
_, epoch, version, release, _ = parse_rpm_name(fake_nevra)
state['evr'] = (epoch, version, release)
# Populate missing EVRs
self.package_store.populate_evrs(
{state['evr']
for state in states if state['evr'] is not None})
self._populate_data("states", states, self._state_import_check, query,
self._state_refresh_maps)
self._populate_associations("file", "states", "file_id",
"rpminfo_state_id",
"oval_file_rpminfo_state", oval_file_id,
states, self.oval_state_map)
for state in states:
if state["arch_operation"] is not None and state[
"arch_operation"] not in self.SUPPORTED_ARCH_OPERATIONS:
self.logger.warning("Unsupported arch operation: %s",
state["arch_operation"])
continue
if state[
"id"] in self.oval_state_map: # Make sure state is imported
# Simplified logic, can contain any regex but RH oval files contains only logical OR
archs = []
if state["arch"] is not None:
archs.extend([{
"id": arch
} for arch in state["arch"].split("|")])
self._populate_associations("state", "archs",
"rpminfo_state_id", "arch_id",
"oval_rpminfo_state_arch",
self.oval_state_map[state["id"]],
archs, self.package_store.arch_map)
def _test_import_check(self, item):
"""Check if test is in DB, return None if it's up to date and row to import otherwise."""
if item["version"] <= self.oval_test_version_map.get(item["id"], -1):
return None
rpminfo_object_id = self.oval_object_map.get(item["object"])
if not rpminfo_object_id:
self.logger.warning("OVAL object not found: %s", item["object"])
return None
check_id = self.oval_check_map.get(item["check"])
if not check_id:
self.logger.warning("OVAL check not found: %s", item["check"])
return None
check_existence_id = self.oval_check_existence_map.get(
item["check_existence"])
if not check_existence_id:
self.logger.warning("OVAL check_existence not found: %s",
item["check_existence"])
return None
return item[
"id"], rpminfo_object_id, check_id, check_existence_id, item[
"version"]
def _test_refresh_maps(self, cur):
"""Add imported data to caches."""
for test_id, oval_id, version in cur.fetchall():
self.oval_test_map[oval_id] = test_id
self.oval_test_version_map[oval_id] = version
def _populate_tests(self, oval_file_id, tests):
query = """insert into oval_rpminfo_test
(oval_id, rpminfo_object_id, check_id, check_existence_id, version) values %s
on conflict (oval_id) do update set
rpminfo_object_id = EXCLUDED.rpminfo_object_id, check_id = EXCLUDED.check_id,
check_existence_id = EXCLUDED.check_existence_id, version = EXCLUDED.version
returning id, oval_id, version"""
self._populate_data("tests", tests, self._test_import_check, query,
self._test_refresh_maps)
self._populate_associations("file", "tests", "file_id",
"rpminfo_test_id",
"oval_file_rpminfo_test", oval_file_id,
tests, self.oval_test_map)
for test in tests:
if test["id"] in self.oval_test_map: # Make sure test is imported
states = [{"id": state} for state in test["states"]]
self._populate_associations("test", "states",
"rpminfo_test_id",
"rpminfo_state_id",
"oval_rpminfo_test_state",
self.oval_test_map[test["id"]],
states, self.oval_state_map)
def _populate_definition_criteria(self, cur, criteria):
operator_id = self.oval_criteria_operator_map.get(criteria["operator"])
if not operator_id:
self.logger.warning("OVAL criteria operator not found: %s",
criteria["operator"])
return None
cur.execute(
"insert into oval_criteria (operator_id) values (%s) returning id",
(operator_id, ))
criteria_id = cur.fetchone()[0]
dependencies_to_import = []
for test in criteria["criterions"]:
test_id = self.oval_test_map.get(test)
if test_id: # Unsuported test type may not be imported (rpmverifyfile etc.)
dependencies_to_import.append(
(criteria_id, None, test_id)) # dep_criteria_id is null
for child_criteria in criteria["criteria"]:
child_criteria_id = self._populate_definition_criteria(
cur, child_criteria) # Recursion
dependencies_to_import.append(
(criteria_id, child_criteria_id, None)) # test_id is null
# Import dependencies
if dependencies_to_import:
execute_values(cur,
"""insert into oval_criteria_dependency
(parent_criteria_id, dep_criteria_id, dep_test_id)
values %s""",
dependencies_to_import,
page_size=len(dependencies_to_import))
return criteria_id
def _definition_import_check(self, item):
"""Check if definition is in DB, return None if it's up to date and row to import otherwise."""
if item["version"] <= self.oval_definition_version_map.get(
item["id"], -1):
return None
definition_type = self.oval_definition_type_map.get(item["type"])
if not definition_type:
self.logger.warning("OVAL definition type not found: %s",
item["type"])
return None
criteria_id = None
if item["criteria"]:
try:
cur = self.conn.cursor()
criteria_id = self._populate_definition_criteria(
cur, item["criteria"])
self.conn.commit()
except Exception: # pylint: disable=broad-except
self.logger.exception("Failure while inserting criteria: ")
self.conn.rollback()
return item["id"], definition_type, criteria_id, item["version"]
def _definition_refresh_maps(self, cur):
"""Add imported data to caches."""
for definition_id, oval_id, version in cur.fetchall():
self.oval_definition_map[oval_id] = definition_id
self.oval_definition_version_map[oval_id] = version
def _populate_definitions(self, oval_file_id, definitions):
query = """insert into oval_definition
(oval_id, definition_type_id, criteria_id, version) values %s
on conflict (oval_id) do update set
definition_type_id = EXCLUDED.definition_type_id, criteria_id = EXCLUDED.criteria_id,
version = EXCLUDED.version
returning id, oval_id, version"""
self._populate_data("definitions", definitions,
self._definition_import_check, query,
self._definition_refresh_maps)
self._populate_associations("file", "definitions", "file_id",
"definition_id", "oval_file_definition",
oval_file_id, definitions,
self.oval_definition_map)
for definition in definitions:
if definition[
"id"] in self.oval_definition_map: # Make sure definition is imported
cves = [{"id": cve} for cve in definition["cves"]]
advisories = [{
"id": advisory
} for advisory in definition["advisories"]]
cpes = [{"id": cpe} for cpe in definition["cpes"]]
# Store missing CPEs (they are often substrings of CPEs already in DB)
self.cpe_store.populate_cpes({
cpe: None
for cpe in definition["cpes"]
if cpe not in self.cpe_store.cpe_label_to_id
})
tests = []
criteria = definition["criteria"]
criteria_stack = []
while criteria is not None:
tests.extend(criteria["criterions"])
criteria_stack.extend(criteria["criteria"])
if criteria_stack:
criteria = criteria_stack.pop()
else:
criteria = None
if not cves:
self.logger.warning(
"OVAL definition has empty CVE list: %s",
definition["id"])
if not advisories and definition["type"] != "vulnerability":
self.logger.warning(
"OVAL definition has empty errata list: %s",
definition["id"])
if not tests:
self.logger.warning(
"OVAL definition has empty test list: %s",
definition["id"])
tests = [{
"id": test
} for test in list(set(tests))] # Make unique
definition_id = self.oval_definition_map[definition["id"]]
self._populate_associations("definition", "cves",
"definition_id", "cve_id",
"oval_definition_cve",
definition_id, cves, self.cve_map)
self._populate_associations("definition", "advisories",
"definition_id", "errata_id",
"oval_definition_errata",
definition_id, advisories,
self.errata_map)
self._populate_associations("definition", "cpes",
"definition_id", "cpe_id",
"oval_definition_cpe",
definition_id, cpes,
self.cpe_store.cpe_label_to_id)
self._populate_associations(
"definition",
"tests",
"definition_id",
"rpminfo_test_id",
"oval_definition_test",
definition_id,
tests,
self.oval_test_map,
missing_ok=True
) # Don't log unsupported test types (rpmverifyfile etc.)
def store(self, oval_file):
"""Store single OVAL definitions file into DB."""
oval_file_id = self._save_oval_file_updated(oval_file.oval_id,
oval_file.updated)
self._populate_objects(oval_file_id, oval_file.objects)
self._populate_states(oval_file_id, oval_file.states)
self._populate_tests(oval_file_id, oval_file.tests)
self._populate_definitions(oval_file_id, oval_file.definitions)
