def __init__(self): super().__init__() self.cpe_store = CpeStore() self.package_store = PackageStore() self.evr_operation_map = self._prepare_table_map( cols=["name"], table="oval_operation_evr") self.cve_map = self._prepare_table_map(cols=["name"], table="cve") self.errata_map = self._prepare_table_map(cols=["name"], table="errata") self.oval_check_map = self._prepare_table_map( cols=["name"], table="oval_check_rpminfo") self.oval_check_existence_map = self._prepare_table_map( cols=["name"], table="oval_check_existence_rpminfo") self.oval_object_map = self._prepare_table_map( cols=["oval_id"], table="oval_rpminfo_object") self.oval_object_version_map = self._prepare_table_map( cols=["oval_id"], table="oval_rpminfo_object", to_col="version") self.oval_state_map = self._prepare_table_map( cols=["oval_id"], table="oval_rpminfo_state") self.oval_state_version_map = self._prepare_table_map( cols=["oval_id"], table="oval_rpminfo_state", to_col="version") self.oval_test_map = self._prepare_table_map(cols=["oval_id"], table="oval_rpminfo_test") self.oval_test_version_map = self._prepare_table_map( cols=["oval_id"], table="oval_rpminfo_test", to_col="version") self.oval_definition_map = self._prepare_table_map( cols=["oval_id"], table="oval_definition") self.oval_definition_version_map = self._prepare_table_map( cols=["oval_id"], table="oval_definition", to_col="version") self.oval_definition_type_map = self._prepare_table_map( cols=["name"], table="oval_definition_type") self.oval_criteria_operator_map = self._prepare_table_map( cols=["name"], table="oval_criteria_operator")
def __init__(self): self.logger = get_logger(__name__) self.conn = DatabaseHandler.get_connection() self.module_store = ModulesStore() self.package_store = PackageStore() self.update_store = UpdateStore() self.content_set_to_db_id = self._prepare_content_set_map()
class RepositoryStore: """ Class providing interface for listing repositories stored in DB and storing repositories one by one. """ def __init__(self): self.logger = get_logger(__name__) self.conn = DatabaseHandler.get_connection() self.module_store = ModulesStore() self.package_store = PackageStore() self.update_store = UpdateStore() self.content_set_to_db_id = self._prepare_content_set_map() def _prepare_content_set_map(self): cur = self.conn.cursor() cur.execute("""select id, label from content_set""") content_sets = {} for cs_id, cs_label in cur.fetchall(): content_sets[cs_label] = cs_id cur.close() return content_sets def list_repositories(self): """List repositories stored in DB. Dictionary with repository label as key is returned.""" cur = self.conn.cursor() cur.execute("""select cs.label, a.name, r.releasever, r.id, r.url, r.revision, cs.id, c.name, c.ca_cert, c.cert, c.key from repo r left join arch a on r.basearch_id = a.id left join certificate c on r.certificate_id = c.id left join content_set cs on r.content_set_id = cs.id""") repos = {} for row in cur.fetchall(): # (content_set_label, repo_arch, repo_releasever) -> repo_id, repo_url, repo_revision... repos[(row[0], row[1], row[2])] = {"id": row[3], "url": row[4], "revision": row[5], "content_set_id": row[6], "cert_name": row[7], "ca_cert": row[8], "cert": row[9], "key": row[10]} cur.close() return repos def _import_basearch(self, basearch): cur = self.conn.cursor() try: cur.execute("select id from arch where name = %s", (basearch,)) arch_id = cur.fetchone() if not arch_id: cur.execute("insert into arch (name) values(%s) returning id", (basearch,)) arch_id = cur.fetchone() self.conn.commit() except Exception: self.logger.exception("Failed to import basearch.") self.conn.rollback() raise finally: cur.close() return arch_id[0] def _import_certificate(self, cert_name, ca_cert, cert, key): if not key: key = None cur = self.conn.cursor() try: cur.execute("select id from certificate where name = %s", (cert_name,)) cert_id = cur.fetchone() if not cert_id: cur.execute("""insert into certificate (name, ca_cert, cert, key) values (%s, %s, %s, %s) returning id""", (cert_name, ca_cert, cert, key,)) cert_id = cur.fetchone() else: cur.execute("update certificate set ca_cert = %s, cert = %s, key = %s where name = %s", (ca_cert, cert, key, cert_name,)) self.conn.commit() except Exception: self.logger.exception("Failed to import certificate.") self.conn.rollback() raise finally: cur.close() return cert_id[0] def cleanup_unused_data(self): """ Deletes packages and errata not associated with any repo etc. """ cur = self.conn.cursor() try: cur.execute("""select p.id from package p where not exists ( select 1 from pkg_repo pr where pr.pkg_id = p.id ) and p.source_package_id is not null """) packages_to_delete = cur.fetchall() cur.execute("""select e.id from errata e where not exists ( select 1 from errata_repo er where er.errata_id = e.id ) """) updates_to_delete = cur.fetchall() if packages_to_delete: cur.execute("""delete from pkg_errata pe where pe.pkg_id in %s""", (tuple(packages_to_delete),)) cur.execute("""delete from module_rpm_artifact mra where mra.pkg_id in %s""", (tuple(packages_to_delete),)) cur.execute("""delete from package p where p.id in %s""", (tuple(packages_to_delete),)) if updates_to_delete: cur.execute("""delete from pkg_errata pe where pe.errata_id in %s""", (tuple(updates_to_delete),)) cur.execute("""delete from errata_cve ec where ec.errata_id in %s""", (tuple(updates_to_delete),)) cur.execute("""delete from errata_refs er where er.errata_id in %s""", (tuple(updates_to_delete),)) cur.execute("""delete from errata e where e.id in %s""", (tuple(updates_to_delete),)) self.conn.commit() except Exception: # pylint: disable=broad-except self.logger.exception("Failed to clean up unused data.") self.conn.rollback() finally: cur.close() def delete_content_set(self, content_set_label): """ Deletes repositories and their content from DB. """ content_set_id = self.content_set_to_db_id[content_set_label] cur = self.conn.cursor() try: cur.execute("""select id from repo where content_set_id = %s""", (content_set_id,)) repo_ids = cur.fetchall() for repo_id in repo_ids: cur.execute("select id from module where repo_id = %s", (repo_id,)) module_ids = cur.fetchall() if module_ids: cur.execute("select id from module_stream where module_id in %s", (tuple(module_ids),)) module_stream_ids = cur.fetchall() if module_stream_ids: cur.execute("""delete from module_profile_pkg where profile_id in (select id from module_profile where stream_id in %s)""", (tuple(module_stream_ids),)) cur.execute("delete from module_rpm_artifact where stream_id in %s", (tuple(module_stream_ids),)) cur.execute("delete from pkg_errata where module_stream_id in %s", (tuple(module_stream_ids),)) cur.execute("delete from module_profile where stream_id in %s", (tuple(module_stream_ids),)) cur.execute("delete from module_stream where module_id in %s", (tuple(module_ids),)) cur.execute("delete from module where repo_id = %s", (repo_id,)) cur.execute("delete from pkg_repo where repo_id = %s", (repo_id,)) cur.execute("delete from errata_repo where repo_id = %s", (repo_id,)) cur.execute("delete from repo where id = %s", (repo_id,)) cur.execute("delete from content_set where id = %s", (content_set_id,)) self.conn.commit() except Exception: # pylint: disable=broad-except self.logger.exception("Failed to delete content set.") self.conn.rollback() finally: cur.close() def import_repository(self, repo): """ Imports or updates repository record in DB. """ if repo.ca_cert: # will raise exception if db error occurs cert_id = self._import_certificate(repo.cert_name, repo.ca_cert, repo.cert, repo.key) else: cert_id = None if repo.basearch: # will raise exception if db error occurs basearch_id = self._import_basearch(repo.basearch) else: basearch_id = None cur = self.conn.cursor() try: content_set_id = self.content_set_to_db_id[repo.content_set] cur.execute("""select id, revision from repo where content_set_id = %s and ((%s is null and basearch_id is null) or basearch_id = %s) and ((%s is null and releasever is null) or releasever = %s) """, (content_set_id, basearch_id, basearch_id, repo.releasever, repo.releasever)) db_repo = cur.fetchone() if not db_repo: cur.execute("""insert into repo (url, content_set_id, basearch_id, releasever, revision, eol, certificate_id) values (%s, %s, %s, %s, %s, false, %s) returning id, revision""", (repo.repo_url, content_set_id, basearch_id, repo.releasever, repo.get_revision(), cert_id,)) db_repo = cur.fetchone() else: revision = repo.get_revision() # if revision in repo object is None, re-use current revision from DB (don't update) # this method is called from 2 different places with 2 different states of repo object if not revision: revision = db_repo[1] cur.execute("""update repo set revision = %s, url = %s, certificate_id = %s where id = %s""", (revision, repo.repo_url, cert_id, db_repo[0],)) self.conn.commit() return db_repo[0] except Exception: self.logger.exception("Failed to import or update repository.") self.conn.rollback() raise finally: cur.close() def store(self, repository): """ Store single repository content into DB. First, basic repository info is processed, then all packages, then all updates. Some steps may be skipped if given data doesn't exist or are already synced. """ try: repo_id = self.import_repository(repository) self.package_store.store(repo_id, repository.list_packages()) self.module_store.store(repo_id, repository.list_modules()) self.update_store.store(repo_id, repository.list_updates()) except Exception: # pylint: disable=broad-except # exception already logged. pass
class RepositoryStore: """ Class providing interface for listing repositories stored in DB and storing repositories one by one. """ def __init__(self): self.logger = get_logger(__name__) self.conn = DatabaseHandler.get_connection() self.module_store = ModulesStore() self.package_store = PackageStore() self.update_store = UpdateStore() self.content_set_to_db_id = self._prepare_content_set_map() def _prepare_content_set_map(self): cur = self.conn.cursor() cur.execute("""select id, label from content_set""") content_sets = {} for cs_id, cs_label in cur.fetchall(): content_sets[cs_label] = cs_id cur.close() return content_sets def list_repositories(self): """List repositories stored in DB. Dictionary with repository label as key is returned.""" cur = self.conn.cursor() cur.execute("""select cs.label, a.name, r.releasever, r.id, r.url, r.revision, cs.id, c.name, c.ca_cert, c.cert, c.key from repo r left join arch a on r.basearch_id = a.id left join certificate c on r.certificate_id = c.id left join content_set cs on r.content_set_id = cs.id""") repos = {} for row in cur.fetchall(): # (content_set_label, repo_arch, repo_releasever) -> repo_id, repo_url, repo_revision... repos[(row[0], row[1], row[2])] = {"id": row[3], "url": row[4], "revision": row[5], "content_set_id": row[6], "cert_name": row[7], "ca_cert": row[8], "cert": row[9], "key": row[10]} cur.close() return repos def _import_basearch(self, basearch): cur = self.conn.cursor() cur.execute("select id from arch where name = %s", (basearch,)) arch_id = cur.fetchone() if not arch_id: cur.execute("insert into arch (name) values(%s) returning id", (basearch,)) arch_id = cur.fetchone() cur.close() self.conn.commit() return arch_id[0] def _import_certificate(self, cert_name, ca_cert, cert, key): if not key: key = None cur = self.conn.cursor() cur.execute("select id from certificate where name = %s", (cert_name,)) cert_id = cur.fetchone() if not cert_id: cur.execute("""insert into certificate (name, ca_cert, cert, key) values (%s, %s, %s, %s) returning id""", (cert_name, ca_cert, cert, key,)) cert_id = cur.fetchone() else: cur.execute("update certificate set ca_cert = %s, cert = %s, key = %s where name = %s", (ca_cert, cert, key, cert_name,)) cur.close() self.conn.commit() return cert_id[0] def cleanup_unused_data(self): """ Deletes packages and errata not associated with any repo etc. """ cur = self.conn.cursor() cur.execute("""select p.id from package p where not exists ( select 1 from pkg_repo pr where pr.pkg_id = p.id ) """) packages_to_delete = [pkg_id for pkg_id in cur.fetchall()] cur.execute("""select e.id from errata e where not exists ( select 1 from errata_repo er where er.errata_id = e.id ) """) updates_to_delete = [update_id for update_id in cur.fetchall()] if packages_to_delete: cur.execute("""delete from pkg_errata pe where pe.pkg_id in %s""", (tuple(packages_to_delete),)) cur.execute("""delete from package p where p.id in %s""", (tuple(packages_to_delete),)) if updates_to_delete: cur.execute("""delete from pkg_errata pe where pe.errata_id in %s""", (tuple(updates_to_delete),)) cur.execute("""delete from errata_cve ec where ec.errata_id in %s""", (tuple(updates_to_delete),)) cur.execute("""delete from errata_refs er where er.errata_id in %s""", (tuple(updates_to_delete),)) cur.execute("""delete from errata e where e.id in %s""", (tuple(updates_to_delete),)) cur.close() self.conn.commit() def delete_content_set(self, content_set_label): """ Deletes repositories and their content from DB. """ content_set_id = self.content_set_to_db_id[content_set_label] cur = self.conn.cursor() cur.execute("""select id from repo where content_set_id = %s""", (content_set_id,)) repo_ids = [repo_id for repo_id in cur.fetchall()] for repo_id in repo_ids: cur.execute("delete from pkg_repo where repo_id = %s", (repo_id,)) cur.execute("delete from errata_repo where repo_id = %s", (repo_id,)) cur.execute("delete from repo where id = %s", (repo_id,)) cur.execute("delete from content_set where id = %s", (content_set_id,)) cur.close() self.conn.commit() def import_repository(self, repo): """ Imports or updates repository record in DB. """ if repo.ca_cert: cert_id = self._import_certificate(repo.cert_name, repo.ca_cert, repo.cert, repo.key) else: cert_id = None if repo.basearch: basearch_id = self._import_basearch(repo.basearch) else: basearch_id = None content_set_id = self.content_set_to_db_id[repo.content_set] cur = self.conn.cursor() cur.execute("""select id from repo where content_set_id = %s and ((%s is null and basearch_id is null) or basearch_id = %s) and ((%s is null and releasever is null) or releasever = %s) """, (content_set_id, basearch_id, basearch_id, repo.releasever, repo.releasever)) repo_id = cur.fetchone() if not repo_id: cur.execute("""insert into repo (url, content_set_id, basearch_id, releasever, revision, eol, certificate_id) values (%s, %s, %s, %s, %s, false, %s) returning id""", (repo.repo_url, content_set_id, basearch_id, repo.releasever, repo.get_revision(), cert_id,)) repo_id = cur.fetchone() else: # Update repository timestamp cur.execute("""update repo set revision = %s, certificate_id = %s, content_set_id = %s, basearch_id = %s, releasever = %s where id = %s""", (repo.get_revision(), cert_id, content_set_id, basearch_id, repo.releasever, repo_id[0],)) cur.close() self.conn.commit() return repo_id[0] def store(self, repository): """ Store single repository content into DB. First, basic repository info is processed, then all packages, then all updates. Some steps may be skipped if given data doesn't exist or are already synced. """ self.logger.info("Syncing repository: %s", ", ".join(filter(None, (repository.content_set, repository.basearch, repository.releasever)))) repo_id = self.import_repository(repository) self.package_store.store(repo_id, repository.list_packages()) self.module_store.store(repo_id, repository.list_modules()) self.update_store.store(repo_id, repository.list_updates())
class RepositoryStore: """ Class providing interface for listing repositories stored in DB and storing repositories one by one. """ def __init__(self): self.content_set_to_db_id = {} self.logger = SimpleLogger() self.conn = DatabaseHandler.get_connection() self.package_store = PackageStore() self.update_store = UpdateStore() def set_content_set_db_mapping(self, content_set_to_db_id): """Set content set to DB is mapping from product_store""" self.content_set_to_db_id = content_set_to_db_id def _get_content_set_id(self, repo): if repo.content_set in self.content_set_to_db_id: return self.content_set_to_db_id[repo.content_set] return None def list_repositories(self): """List repositories stored in DB. Dictionary with repository label as key is returned.""" cur = self.conn.cursor() cur.execute( """select r.id, r.label, r.url, r.revision, cs.id, cs.label, c.name, c.ca_cert, c.cert, c.key from repo r left join certificate c on r.certificate_id = c.id left join content_set cs on r.content_set_id = cs.id""") repos = {} for row in cur.fetchall(): # repo_label -> repo_id, repo_url, repo_revision repos[row[1]] = { "id": row[0], "url": row[2], "revision": row[3], "content_set_id": row[4], "content_set": row[5], "cert_name": row[6], "ca_cert": row[7], "cert": row[8], "key": row[9] } cur.close() return repos def _import_certificate(self, cert_name, ca_cert, cert, key): cur = self.conn.cursor() cur.execute("select id from certificate where name = %s", (cert_name, )) cert_id = cur.fetchone() if not cert_id: cur.execute( """insert into certificate (name, ca_cert, cert, key) values (%s, %s, %s, %s) returning id""", ( cert_name, ca_cert, cert, key, )) cert_id = cur.fetchone() else: cur.execute( "update certificate set ca_cert = %s, cert = %s, key = %s where name = %s", ( ca_cert, cert, key, cert_name, )) cur.close() self.conn.commit() return cert_id[0] def _import_repository(self, repo): if repo.ca_cert: cert_id = self._import_certificate(repo.cert_name, repo.ca_cert, repo.cert, repo.key) else: cert_id = None cur = self.conn.cursor() cur.execute("select id from repo where label = %s", (repo.repo_label, )) repo_id = cur.fetchone() content_set_id = self._get_content_set_id(repo) if not repo_id: cur.execute( """insert into repo (label, url, revision, eol, certificate_id, content_set_id) values (%s, %s, to_timestamp(%s), false, %s, %s) returning id""", ( repo.repo_label, repo.repo_url, repo.repomd.get_revision(), cert_id, content_set_id, )) repo_id = cur.fetchone() else: # Update repository timestamp cur.execute( """update repo set revision = to_timestamp(%s), certificate_id = %s, content_set_id = %s where id = %s""", ( repo.repomd.get_revision(), cert_id, content_set_id, repo_id[0], )) cur.close() self.conn.commit() return repo_id[0] def store(self, repository): """ Store single repository into DB. First, basic repository info is processed, then all packages, then all updates. Some steps may be skipped if given data doesn't exist or are already synced. """ self.logger.log("Syncing repository: %s" % repository.repo_label) repo_id = self._import_repository(repository) self.package_store.store(repo_id, repository.list_packages()) self.update_store.store(repo_id, repository.list_updates())
def __init__(self): self.content_set_to_db_id = {} self.logger = SimpleLogger() self.conn = DatabaseHandler.get_connection() self.package_store = PackageStore() self.update_store = UpdateStore()
class OvalStore(ObjectStore): # pylint: disable=too-many-instance-attributes """ Class providing interface for fetching/importing OVAL data from/into the DB. """ OVAL_FEED_UPDATED_KEY = 'redhatovalfeed:updated' # Not in DB table like other operations because we don't need this information further SUPPORTED_ARCH_OPERATIONS = ["equals", "pattern match"] def __init__(self): super().__init__() self.cpe_store = CpeStore() self.package_store = PackageStore() self.evr_operation_map = self._prepare_table_map( cols=["name"], table="oval_operation_evr") self.cve_map = self._prepare_table_map(cols=["name"], table="cve") self.errata_map = self._prepare_table_map(cols=["name"], table="errata") self.oval_check_map = self._prepare_table_map( cols=["name"], table="oval_check_rpminfo") self.oval_check_existence_map = self._prepare_table_map( cols=["name"], table="oval_check_existence_rpminfo") self.oval_object_map = self._prepare_table_map( cols=["oval_id"], table="oval_rpminfo_object") self.oval_object_version_map = self._prepare_table_map( cols=["oval_id"], table="oval_rpminfo_object", to_col="version") self.oval_state_map = self._prepare_table_map( cols=["oval_id"], table="oval_rpminfo_state") self.oval_state_version_map = self._prepare_table_map( cols=["oval_id"], table="oval_rpminfo_state", to_col="version") self.oval_test_map = self._prepare_table_map(cols=["oval_id"], table="oval_rpminfo_test") self.oval_test_version_map = self._prepare_table_map( cols=["oval_id"], table="oval_rpminfo_test", to_col="version") self.oval_definition_map = self._prepare_table_map( cols=["oval_id"], table="oval_definition") self.oval_definition_version_map = self._prepare_table_map( cols=["oval_id"], table="oval_definition", to_col="version") self.oval_definition_type_map = self._prepare_table_map( cols=["name"], table="oval_definition_type") self.oval_criteria_operator_map = self._prepare_table_map( cols=["name"], table="oval_criteria_operator") def list_oval_definitions(self): """List oval definitions and their timestamps stored in DB. Dictionary with oval id as key is returned.""" cur = self.conn.cursor() cur.execute("""select oval_id, updated from oval_file""") return dict(cur.fetchall()) def save_lastmodified(self, lastmodified): """Store OVAL file timestamp.""" lastmodified = format_datetime(lastmodified) cur = self.conn.cursor() # Update timestamp cur.execute("update metadata set value = %s where key = %s", ( lastmodified, self.OVAL_FEED_UPDATED_KEY, )) if cur.rowcount < 1: cur.execute("insert into metadata (key, value) values (%s, %s)", (self.OVAL_FEED_UPDATED_KEY, lastmodified)) cur.close() self.conn.commit() def _save_oval_file_updated(self, oval_id, updated): cur = self.conn.cursor() # Update timestamp cur.execute( "update oval_file set updated = %s where oval_id = %s returning id", ( updated, oval_id, )) if cur.rowcount < 1: cur.execute( "insert into oval_file (oval_id, updated) values (%s, %s) returning id", ( oval_id, updated, )) db_id_row = cur.fetchone() cur.close() self.conn.commit() return db_id_row[0] def _populate_data(self, entity, data, import_check_func, query, refresh_maps_func): """Generic method to populate table with OVAL entity (objects, states, tests, etc.).""" to_import = [] # Append only here, not delete rows, same item may be referenced from multiple files for item in data: row = import_check_func(item) if row: to_import.append(row) self.logger.debug("OVAL %s to import: %d", entity, len(to_import)) if to_import: try: cur = self.conn.cursor() execute_values(cur, query, to_import, page_size=len(to_import)) refresh_maps_func(cur) self.conn.commit() except Exception: # pylint: disable=broad-except self.logger.exception("Failure while inserting %s data: ", entity) self.conn.rollback() finally: cur.close() def _populate_associations(self, entity_one, entity_many, ent_one_id_col, ent_many_id_col, table_name, ent_one_id, ent_many_data, import_check_map, missing_ok=False): """Associate/disassociate many_entities (objects, states, archs) with entity_one (file, state).""" try: cur = self.conn.cursor() associated_with_entity_one = set() cur.execute( f"select {ent_many_id_col} from {table_name} where {ent_one_id_col} = %s", (ent_one_id, )) for row in cur.fetchall(): associated_with_entity_one.add(row[0]) self.logger.debug("OVAL %s associated with %s %s: %d", entity_many, entity_one, ent_one_id, len(associated_with_entity_one)) to_associate = [] for item in ent_many_data: item_id = import_check_map.get(item["id"]) if not item_id: if not missing_ok: self.logger.warning("Item (%s) not found: %s", entity_many, item["id"]) continue if item_id in associated_with_entity_one: associated_with_entity_one.remove(item_id) else: to_associate.append(item_id) self.logger.debug("New OVAL %s to associate with %s %s: %d", entity_many, entity_one, ent_one_id, len(to_associate)) self.logger.debug("OVAL %s to disassociate with %s %s: %d", entity_many, entity_one, ent_one_id, len(associated_with_entity_one)) if to_associate: execute_values( cur, f"insert into {table_name} ({ent_one_id_col}, {ent_many_id_col}) values %s", [(ent_one_id, item_id) for item_id in to_associate], page_size=len(to_associate)) if associated_with_entity_one: cur.execute( f"delete from {table_name} where {ent_one_id_col} = %s and {ent_many_id_col} in %s", ( ent_one_id, tuple(associated_with_entity_one), )) self.conn.commit() except Exception: # pylint: disable=broad-except self.logger.exception( "Failure while (dis)associating OVAL %s with %s %s: ", entity_many, entity_one, ent_one_id) self.conn.rollback() finally: cur.close() def _object_import_check(self, item): """Check if object is in DB, return None if it's up to date and row to import otherwise.""" if item["version"] <= self.oval_object_version_map.get(item["id"], -1): return None name_id = self.package_store.package_name_map.get(item["name"]) if not name_id: self.logger.warning("Package name not found: %s", item["name"]) return None return item["id"], name_id, item["version"] def _object_refresh_maps(self, cur): """Add imported data to caches.""" for obj_id, oval_id, version in cur.fetchall(): self.oval_object_map[oval_id] = obj_id self.oval_object_version_map[oval_id] = version def _populate_objects(self, oval_file_id, objects): query = """insert into oval_rpminfo_object (oval_id, package_name_id, version) values %s on conflict (oval_id) do update set package_name_id = EXCLUDED.package_name_id, version = EXCLUDED.version returning id, oval_id, version""" # Populate missing package names self.package_store.populate_dep_table( "package_name", {obj["name"] for obj in objects}, self.package_store.package_name_map) self._populate_data("objects", objects, self._object_import_check, query, self._object_refresh_maps) self._populate_associations("file", "objects", "file_id", "rpminfo_object_id", "oval_file_rpminfo_object", oval_file_id, objects, self.oval_object_map) def _state_import_check(self, item): """Check if state is in DB, return None if it's up to date and row to import otherwise.""" if item["version"] <= self.oval_state_version_map.get(item["id"], -1): return None evr_id = evr_operation_id = None if item['evr'] is not None: epoch, version, release = item['evr'] evr_id = self.package_store.evr_map.get((epoch, version, release)) if not evr_id: self.logger.warning("EVR not found: %s, %s, %s", epoch, version, release) return None if item["evr_operation"] is not None: evr_operation_id = self.evr_operation_map.get( item["evr_operation"]) if not evr_operation_id: self.logger.warning("Unsupported EVR operation: %s", item["evr_operation"]) return None return item["id"], evr_id, evr_operation_id, item["version"] def _state_refresh_maps(self, cur): """Add imported data to caches.""" for state_id, oval_id, version in cur.fetchall(): self.oval_state_map[oval_id] = state_id self.oval_state_version_map[oval_id] = version def _populate_states(self, oval_file_id, states): query = """insert into oval_rpminfo_state (oval_id, evr_id, evr_operation_id, version) values %s on conflict (oval_id) do update set evr_id = EXCLUDED.evr_id, evr_operation_id = EXCLUDED.evr_operation_id, version = EXCLUDED.version returning id, oval_id, version""" # Parse EVR first for state in states: if state['evr'] is not None: # FIXME: as an input to common.rpm.parse_rpm_name, we don't have function to parse evr only fake_nevra = f"pn-{state['evr']}.noarch" _, epoch, version, release, _ = parse_rpm_name(fake_nevra) state['evr'] = (epoch, version, release) # Populate missing EVRs self.package_store.populate_evrs( {state['evr'] for state in states if state['evr'] is not None}) self._populate_data("states", states, self._state_import_check, query, self._state_refresh_maps) self._populate_associations("file", "states", "file_id", "rpminfo_state_id", "oval_file_rpminfo_state", oval_file_id, states, self.oval_state_map) for state in states: if state["arch_operation"] is not None and state[ "arch_operation"] not in self.SUPPORTED_ARCH_OPERATIONS: self.logger.warning("Unsupported arch operation: %s", state["arch_operation"]) continue if state[ "id"] in self.oval_state_map: # Make sure state is imported # Simplified logic, can contain any regex but RH oval files contains only logical OR archs = [] if state["arch"] is not None: archs.extend([{ "id": arch } for arch in state["arch"].split("|")]) self._populate_associations("state", "archs", "rpminfo_state_id", "arch_id", "oval_rpminfo_state_arch", self.oval_state_map[state["id"]], archs, self.package_store.arch_map) def _test_import_check(self, item): """Check if test is in DB, return None if it's up to date and row to import otherwise.""" if item["version"] <= self.oval_test_version_map.get(item["id"], -1): return None rpminfo_object_id = self.oval_object_map.get(item["object"]) if not rpminfo_object_id: self.logger.warning("OVAL object not found: %s", item["object"]) return None check_id = self.oval_check_map.get(item["check"]) if not check_id: self.logger.warning("OVAL check not found: %s", item["check"]) return None check_existence_id = self.oval_check_existence_map.get( item["check_existence"]) if not check_existence_id: self.logger.warning("OVAL check_existence not found: %s", item["check_existence"]) return None return item[ "id"], rpminfo_object_id, check_id, check_existence_id, item[ "version"] def _test_refresh_maps(self, cur): """Add imported data to caches.""" for test_id, oval_id, version in cur.fetchall(): self.oval_test_map[oval_id] = test_id self.oval_test_version_map[oval_id] = version def _populate_tests(self, oval_file_id, tests): query = """insert into oval_rpminfo_test (oval_id, rpminfo_object_id, check_id, check_existence_id, version) values %s on conflict (oval_id) do update set rpminfo_object_id = EXCLUDED.rpminfo_object_id, check_id = EXCLUDED.check_id, check_existence_id = EXCLUDED.check_existence_id, version = EXCLUDED.version returning id, oval_id, version""" self._populate_data("tests", tests, self._test_import_check, query, self._test_refresh_maps) self._populate_associations("file", "tests", "file_id", "rpminfo_test_id", "oval_file_rpminfo_test", oval_file_id, tests, self.oval_test_map) for test in tests: if test["id"] in self.oval_test_map: # Make sure test is imported states = [{"id": state} for state in test["states"]] self._populate_associations("test", "states", "rpminfo_test_id", "rpminfo_state_id", "oval_rpminfo_test_state", self.oval_test_map[test["id"]], states, self.oval_state_map) def _populate_definition_criteria(self, cur, criteria): operator_id = self.oval_criteria_operator_map.get(criteria["operator"]) if not operator_id: self.logger.warning("OVAL criteria operator not found: %s", criteria["operator"]) return None cur.execute( "insert into oval_criteria (operator_id) values (%s) returning id", (operator_id, )) criteria_id = cur.fetchone()[0] dependencies_to_import = [] for test in criteria["criterions"]: test_id = self.oval_test_map.get(test) if test_id: # Unsuported test type may not be imported (rpmverifyfile etc.) dependencies_to_import.append( (criteria_id, None, test_id)) # dep_criteria_id is null for child_criteria in criteria["criteria"]: child_criteria_id = self._populate_definition_criteria( cur, child_criteria) # Recursion dependencies_to_import.append( (criteria_id, child_criteria_id, None)) # test_id is null # Import dependencies if dependencies_to_import: execute_values(cur, """insert into oval_criteria_dependency (parent_criteria_id, dep_criteria_id, dep_test_id) values %s""", dependencies_to_import, page_size=len(dependencies_to_import)) return criteria_id def _definition_import_check(self, item): """Check if definition is in DB, return None if it's up to date and row to import otherwise.""" if item["version"] <= self.oval_definition_version_map.get( item["id"], -1): return None definition_type = self.oval_definition_type_map.get(item["type"]) if not definition_type: self.logger.warning("OVAL definition type not found: %s", item["type"]) return None criteria_id = None if item["criteria"]: try: cur = self.conn.cursor() criteria_id = self._populate_definition_criteria( cur, item["criteria"]) self.conn.commit() except Exception: # pylint: disable=broad-except self.logger.exception("Failure while inserting criteria: ") self.conn.rollback() return item["id"], definition_type, criteria_id, item["version"] def _definition_refresh_maps(self, cur): """Add imported data to caches.""" for definition_id, oval_id, version in cur.fetchall(): self.oval_definition_map[oval_id] = definition_id self.oval_definition_version_map[oval_id] = version def _populate_definitions(self, oval_file_id, definitions): query = """insert into oval_definition (oval_id, definition_type_id, criteria_id, version) values %s on conflict (oval_id) do update set definition_type_id = EXCLUDED.definition_type_id, criteria_id = EXCLUDED.criteria_id, version = EXCLUDED.version returning id, oval_id, version""" self._populate_data("definitions", definitions, self._definition_import_check, query, self._definition_refresh_maps) self._populate_associations("file", "definitions", "file_id", "definition_id", "oval_file_definition", oval_file_id, definitions, self.oval_definition_map) for definition in definitions: if definition[ "id"] in self.oval_definition_map: # Make sure definition is imported cves = [{"id": cve} for cve in definition["cves"]] advisories = [{ "id": advisory } for advisory in definition["advisories"]] cpes = [{"id": cpe} for cpe in definition["cpes"]] # Store missing CPEs (they are often substrings of CPEs already in DB) self.cpe_store.populate_cpes({ cpe: None for cpe in definition["cpes"] if cpe not in self.cpe_store.cpe_label_to_id }) tests = [] criteria = definition["criteria"] criteria_stack = [] while criteria is not None: tests.extend(criteria["criterions"]) criteria_stack.extend(criteria["criteria"]) if criteria_stack: criteria = criteria_stack.pop() else: criteria = None if not cves: self.logger.warning( "OVAL definition has empty CVE list: %s", definition["id"]) if not advisories and definition["type"] != "vulnerability": self.logger.warning( "OVAL definition has empty errata list: %s", definition["id"]) if not tests: self.logger.warning( "OVAL definition has empty test list: %s", definition["id"]) tests = [{ "id": test } for test in list(set(tests))] # Make unique definition_id = self.oval_definition_map[definition["id"]] self._populate_associations("definition", "cves", "definition_id", "cve_id", "oval_definition_cve", definition_id, cves, self.cve_map) self._populate_associations("definition", "advisories", "definition_id", "errata_id", "oval_definition_errata", definition_id, advisories, self.errata_map) self._populate_associations("definition", "cpes", "definition_id", "cpe_id", "oval_definition_cpe", definition_id, cpes, self.cpe_store.cpe_label_to_id) self._populate_associations( "definition", "tests", "definition_id", "rpminfo_test_id", "oval_definition_test", definition_id, tests, self.oval_test_map, missing_ok=True ) # Don't log unsupported test types (rpmverifyfile etc.) def store(self, oval_file): """Store single OVAL definitions file into DB.""" oval_file_id = self._save_oval_file_updated(oval_file.oval_id, oval_file.updated) self._populate_objects(oval_file_id, oval_file.objects) self._populate_states(oval_file_id, oval_file.states) self._populate_tests(oval_file_id, oval_file.tests) self._populate_definitions(oval_file_id, oval_file.definitions)