def _populate_states(self, oval_file_id, states):
self.oval_state_map = self._prepare_table_map(cols=["file_id", "oval_id"],
to_cols=["id", "evr_id", "evr_operation_id", "version"],
table="oval_rpminfo_state",
where=f"file_id = {oval_file_id}")
# Parse EVR first
for state in states:
if state['evr'] is not None:
# FIXME: as an input to common.rpm.parse_rpm_name, we don't have function to parse evr only
fake_nevra = f"pn-{state['evr']}.noarch"
_, epoch, version, release, _ = parse_rpm_name(fake_nevra)
state['evr'] = (epoch, version, release)
# Populate missing EVRs
self.package_store.populate_evrs({state['evr'] for state in states if state['evr'] is not None})
# Insert/update data
self._populate_data("states", oval_file_id, states, self._state_import_check, "oval_rpminfo_state",
["file_id", "oval_id", "evr_id", "evr_operation_id", "version"], self._state_refresh_maps,
self._states_to_delete, "(%s::int, %s::int, %s, %s::int, %s::int, %s::int)")
for state in states:
if state["arch_operation"] is not None and state["arch_operation"] not in self.SUPPORTED_ARCH_OPERATIONS:
self.logger.warning("Unsupported arch operation: %s", state["arch_operation"])
continue
if (oval_file_id, state["id"]) in self.oval_state_map: # Make sure state is imported
# Simplified logic, can contain any regex but RH oval files contains only logical OR
archs = []
if state["arch"] is not None:
archs.extend([{"id": arch} for arch in state["arch"].split("|")])
self._populate_associations("state", "archs", "rpminfo_state_id", "arch_id",
"oval_rpminfo_state_arch",
self.oval_state_map[(oval_file_id, state["id"])][0],
archs, self.package_store.arch_map)
def test_parse_3_epoch(self):
"""Test parsing valid rpm name with epoch preceeding package name."""
name, epoch, ver, rel, arch = rpm_utils.parse_rpm_name(
'3:Agda-2.5.2-9.fc27.x86_64.rpm')
self.assertEqual("3", epoch)
self.assertEqual("Agda", name)
self.assertEqual("2.5.2", ver)
self.assertEqual("9.fc27", rel)
self.assertEqual("x86_64", arch)
def test_parse_4_epoch(self):
"""Test parsing valid rpm name with epoch preceeding version."""
name, epoch, ver, rel, arch = rpm_utils.parse_rpm_name(
'perl-DBD-Pg-2:3.7.4-2.module+el8+2517+b1471f1c.x86_64')
self.assertEqual("2", epoch)
self.assertEqual("perl-DBD-Pg", name)
self.assertEqual("3.7.4", ver)
self.assertEqual("2.module+el8+2517+b1471f1c", rel)
self.assertEqual("x86_64", arch)
def test_parse_5_epoch(self):
"""Test parsing valid rpm name with no epoch and specify default epoch"""
name, epoch, ver, rel, arch = rpm_utils.parse_rpm_name(
'389-ds-base-1.3.7.8-1.fc27.src.rpm', default_epoch="1")
self.assertEqual("1", epoch)
self.assertEqual("389-ds-base", name)
self.assertEqual("1.3.7.8", ver)
self.assertEqual("1.fc27", rel)
self.assertEqual("src", arch)
def test_split_packagename(self):
"""Test splitting package name into N,E,V,R,A."""
pkg_name = "bash-0:4.2.46-20.el7_2.x86_64.rpm"
name, epoch, version, release, arch = rpm_utils.parse_rpm_name(pkg_name, default_epoch='0')
assert name == "bash"
assert epoch == "0"
assert version == "4.2.46"
assert release == "20.el7_2"
assert arch == "x86_64"
def test_parse_2_rpm(self):
"""Test parsing valid rpm name."""
name, epoch, ver, rel, arch = rpm_utils.parse_rpm_name(
'Agda-2.5.2-9.fc27.x86_64.rpm')
self.assertEqual(None, epoch)
self.assertEqual("Agda", name)
self.assertEqual("2.5.2", ver)
self.assertEqual("9.fc27", rel)
self.assertEqual("x86_64", arch)
def test_parse_1_srpm(self):
"""Test parsing valid srpm name."""
name, epoch, ver, rel, arch = rpm_utils.parse_rpm_name(
'389-ds-base-1.3.7.8-1.fc27.src.rpm')
self.assertEqual(None, epoch)
self.assertEqual("389-ds-base", name)
self.assertEqual("1.3.7.8", ver)
self.assertEqual("1.fc27", rel)
self.assertEqual("src", arch)
def test_parse_6_invalid_rpmname(self):
"""Test parsing invalid rpm name."""
with self.assertRaises(rpm_utils.RPMParseException):
rpm_utils.parse_rpm_name('foo', raise_exception=True)
with self.assertRaises(rpm_utils.RPMParseException):
rpm_utils.parse_rpm_name('foo.rpm', raise_exception=True)
with self.assertRaises(rpm_utils.RPMParseException):
rpm_utils.parse_rpm_name('foo-1.3.x86.rpm', raise_exception=True)
with self.assertRaises(rpm_utils.RPMParseException):
rpm_utils.parse_rpm_name('2:389-ds-base-4:1.3.7.8-1.fc27.src.rpm',
raise_exception=True)
def _get_source_package_id(self, pkg):
source_package_id = None
if pkg["srpm"]:
name, epoch, ver, rel, arch = rpm_utils.parse_rpm_name(pkg["srpm"], default_epoch=pkg["epoch"],
raise_exception=True)
name_id = self.package_name_map[name]
evr_id = self.evr_map[(epoch, ver, rel)]
arch_id = self.arch_map[arch]
source_package_id = self.package_map[(name_id, evr_id, arch_id)]
return source_package_id
def _get_source_packages(packages):
unique_source_packages = set()
for pkg in packages:
if pkg["srpm"]:
unique_source_packages.add(rpm_utils.parse_rpm_name(pkg["srpm"], default_epoch=pkg["epoch"],
raise_exception=True))
source_packages = []
for name, epoch, ver, rel, arch in unique_source_packages:
source_packages.append(dict(name=name, epoch=epoch, ver=ver, rel=rel, arch=arch,
srpm=None, summary=None, description=None))
return source_packages
def test_parse_7_invalid_noraise(self):
"""Test parsing invalid rpm name."""
for name in [
'foo',
'foo.rpm',
'foo-1.3.x86.rpm',
'2:389-ds-base-4:1.3.7.8-1.fc27.src.rpm',
]:
name, epoch, ver, rel, arch = rpm_utils.parse_rpm_name(name)
self.assertEqual(None, epoch)
self.assertEqual("", name)
self.assertEqual("", ver)
self.assertEqual("", rel)
self.assertEqual("", arch)
def process_input_packages(self, data: dict) -> (dict, dict):
"""Parse input NEVRAs and filter out unknown (or without updates) package names."""
latest_only = data.get("latest_only", False)
packages_to_process = filter_package_list(data.get('package_list', None), latest_only)
filtered_packages_to_process = {}
update_list = {}
if packages_to_process is not None:
for pkg in packages_to_process:
update_list[pkg] = {}
name, epoch, ver, rel, arch = parse_rpm_name(pkg, default_epoch='0')
if name in self.db_cache.packagename2id:
if self.db_cache.packagename2id[name] in self.db_cache.updates_index:
filtered_packages_to_process[pkg] = {'parsed_nevra': (name, epoch, ver, rel, arch)}
return filtered_packages_to_process, update_list
def process_list(self, data):
""" Processes whole package_list and returns info. """
packages = data.get("package_list")
response = {"package_list": {}}
db_connection = self.db_pool.get_connection()
with db_connection.get_cursor() as cursor:
for package in packages:
name, epoch, version, release, arch = parse_rpm_name(package, default_epoch='0')
cursor.execute("""select distinct p.summary, p.description,
pn2.name, evr2.epoch, evr2.version, evr2.release, a2.name,
cs.label, cs.name, a.name, r.releasever,
pn3.name
from package p
left join package_name pn on pn.id = p.name_id
left join arch ap on p.arch_id = ap.id
left join evr on p.evr_id = evr.id
left join package p2 on p2.id = p.source_package_id
left join package_name pn2 on p2.name_id = pn2.id
left join evr evr2 on p2.evr_id = evr2.id
left join arch a2 on p2.arch_id = a2.id
left join pkg_repo pr on pr.pkg_id = p.id
left join repo r on r.id = pr.repo_id
left join content_set cs on cs.id = r.content_set_id
left join arch a on a.id = r.basearch_id
left join package p3 on p3.source_package_id = p.id
left join package_name pn3 on p3.name_id = pn3.id
where pn.name = '%s'
and ap.name = '%s'
and evr.epoch = '%s'
and evr.version = '%s'
and evr.release = '%s'
""" % (name, arch, epoch, version, release))
query = cursor.fetchall()
pkgs = {}
pkgs[package] = {}
for item in query:
pkgs[package] = {"summary": item[PACKAGE_SUMMARY],
"description": item[PACKAGE_DESCRIPTION],
"source_package": join_rpm_name(item[SOURCE_PACKAGE_NAME],
item[SOURCE_PACKAGE_EPOCH],
item[SOURCE_PACKAGE_VERSION],
item[SOURCE_PACKAGE_RELEASE],
item[SOURCE_PACKAGE_ARCH]),
"repositories": self._build_repositories(query),
"binary_package_list": self._build_binary_packages(query)
}
response["package_list"].update(pkgs)
self.db_pool.return_connection(db_connection)
return response
def _populate_rpm_artifacts(self, modules, repo_id):
cur = self.conn.cursor()
try: # pylint: disable=too-many-nested-blocks
nevras_in_repo = self._get_nevras_in_repo(repo_id)
to_associate = set()
for module in modules:
if 'artifacts' in module:
for artifact in module['artifacts']:
split_pkg_name = rpm_utils.parse_rpm_name(
artifact, default_epoch='0', raise_exception=True)
if split_pkg_name in nevras_in_repo:
to_associate.add((
nevras_in_repo[split_pkg_name],
module['stream_id'],
))
else:
self.logger.debug('Nevra %s missing in repo %s',
artifact, repo_id)
if to_associate:
execute_values(
cur,
"""select pkg_id, stream_id from module_rpm_artifact
inner join (values %s) t(pkg_id, stream_id)
using (pkg_id, stream_id)
""",
list(to_associate),
page_size=len(to_associate))
for r_pkg_id, r_stream_id in cur.fetchall():
to_associate.remove((
r_pkg_id,
r_stream_id,
))
if to_associate:
execute_values(
cur,
"""insert into module_rpm_artifact (pkg_id, stream_id)
values %s""",
list(to_associate),
page_size=len(to_associate))
self.conn.commit()
except Exception: # pylint: disable=broad-except
self.logger.exception("Failure while populating rpm artifacts")
self.conn.rollback()
finally:
cur.close()
def filter_package_list(package_list, latest_only=False):
"""
Filter packages with latest NEVRA
:param package_list: list of package NEVRAs
:param latest_only: boolean switch to return only latest NEVRA for given name.arch
:return: filtered list of NEVRAs
"""
if not latest_only:
return package_list
latest_pkgs = {}
for pkg in package_list:
name, epoch, ver, rel, arch = parse_rpm_name(pkg)
if (name, arch) in latest_pkgs:
latest = latest_pkgs[(name, arch)][0:3]
if rpm.labelCompare((epoch, ver, rel), latest) < 1: # pylint: disable=no-member
continue
latest_pkgs[(name, arch)] = (epoch, ver, rel, pkg)
return [val[3] for val in latest_pkgs.values()]
def process_nevras(self, data):
""" Method returns the list of repositories where packages belongs. """
packages = data.get("package_list")
response = {"data": {}}
db_connection = self.db_pool.get_connection()
with db_connection.get_cursor() as cursor:
for package in packages:
name, epoch, version, release, arch = parse_rpm_name(
package, default_epoch='0')
cursor.execute("""select distinct cs.name, cs.label
from package p
left join package_name pn on p.name_id = pn.id
left join evr e on p.evr_id = e.id
left join arch a on p.arch_id = a.id
left join pkg_repo pr on p.id = pr.pkg_id
left join repo r on r.id = pr.repo_id
left join content_set cs on cs.id = r.content_set_id
where pn.name = '%s'
and a.name = '%s'
and e.epoch = '%s'
and e.version = '%s'
and e.release = '%s'
""" % (name, arch, epoch, version, release))
response["data"][package] = []
for repository_query in cursor:
repository_data = {}
if repository_query[REPOSITORY_NAME] is not None:
repository_data["repo_name"] = repository_query[
REPOSITORY_NAME]
if repository_query[REPOSITORY_LABEL] is not None:
repository_data["repo_label"] = repository_query[
REPOSITORY_LABEL]
if repository_data:
response["data"][package].append(repository_data)
return response
def process_list(self, api_version, data): # pylint: disable=unused-argument
"""
Returns package details.
:param data: json request parsed into data structure
:returns: json response with package details
"""
packages = data.get('package_list', None)
# By default, don't include third party data
want_third_party = data.get('third_party', False)
packagelist = {}
response = {
'last_change':
utils.format_datetime(self.cache.dbchange['last_change'])
}
if not packages:
response['package_list'] = packagelist
return response
for pkg in packages:
packagedata = packagelist.setdefault(pkg, {})
is_third_party = False
name, epoch, ver, rel, arch = parse_rpm_name(pkg,
default_epoch='0')
if name in self.cache.packagename2id \
and (epoch, ver, rel) in self.cache.evr2id \
and arch in self.cache.arch2id:
name_id = self.cache.packagename2id[name]
evr_id = self.cache.evr2id[(epoch, ver, rel)]
arch_id = self.cache.arch2id[arch]
pkg_id = self.cache.nevra2pkgid.get((name_id, evr_id, arch_id),
None)
if pkg_id:
pkg_detail = self.cache.package_details[pkg_id]
packagedata['summary'] = self.cache.strings.get(
pkg_detail[PKG_SUMMARY_ID], None)
packagedata['description'] = self.cache.strings.get(
pkg_detail[PKG_DESC_ID], None)
packagedata['source_package'] = self._get_source_package(
pkg_detail)
packagedata['repositories'] = []
packagedata[
'package_list'] = self._get_built_binary_packages(
pkg_id)
if pkg_id in self.cache.pkgid2repoids:
for repo_id in self.cache.pkgid2repoids[pkg_id]:
repodetail = self.cache.repo_detail[repo_id]
is_third_party = is_third_party or bool(
repodetail[REPO_THIRD_PARTY])
repodata = {
'label':
repodetail[REPO_LABEL],
'name':
repodetail[REPO_NAME],
'basearch':
utils.none2empty(repodetail[REPO_BASEARCH]),
'releasever':
utils.none2empty(repodetail[REPO_RELEASEVER]),
}
packagedata['repositories'].append(repodata)
# If the package is third party, then remove it from result
if not want_third_party and is_third_party:
packagelist[pkg] = {}
response['package_list'] = packagelist
return response