def _populate_rpm_artifacts(self, modules, repo_id):
cur = self.conn.cursor()
try: # pylint: disable=too-many-nested-blocks
nevras_in_repo = self._get_nevras_in_repo(repo_id)
to_associate = set()
for module in modules:
if 'artifacts' in module:
for artifact in module['artifacts']:
split_pkg_name = rpm.parse_rpm_name(artifact, default_epoch='0')
if split_pkg_name in nevras_in_repo:
to_associate.add((nevras_in_repo[split_pkg_name], module['stream_id'],))
else:
self.logger.info('Nevra %s missing in repo %s', artifact, repo_id)
if to_associate:
execute_values(cur,
"""select pkg_id, stream_id from module_rpm_artifact
inner join (values %s) t(pkg_id, stream_id)
using (pkg_id, stream_id)
""", list(to_associate), page_size=len(to_associate))
for r_pkg_id, r_stream_id in cur.fetchall():
to_associate.remove((r_pkg_id, r_stream_id,))
if to_associate:
execute_values(cur,
"""insert into module_rpm_artifact (pkg_id, stream_id)
values %s""",
list(to_associate), page_size=len(to_associate))
self.conn.commit()
except Exception: # pylint: disable=broad-except
self.logger.exception("Failure while populating rpm artifacts")
self.conn.rollback()
finally:
cur.close()
def test_parse_5_epoch(self):
"""Test parsing valid rpm name with no epoch and specify default epoch"""
name, epoch, ver, rel, arch = rpm.parse_rpm_name('389-ds-base-1.3.7.8-1.fc27.src.rpm', default_epoch="1")
self.assertEqual("1", epoch)
self.assertEqual("389-ds-base", name)
self.assertEqual("1.3.7.8", ver)
self.assertEqual("1.fc27", rel)
self.assertEqual("src", arch)
def test_parse_4_epoch(self):
"""Test parsing valid rpm name with epoch preceeding version."""
name, epoch, ver, rel, arch = rpm.parse_rpm_name('perl-DBD-Pg-2:3.7.4-2.module+el8+2517+b1471f1c.x86_64')
self.assertEqual("2", epoch)
self.assertEqual("perl-DBD-Pg", name)
self.assertEqual("3.7.4", ver)
self.assertEqual("2.module+el8+2517+b1471f1c", rel)
self.assertEqual("x86_64", arch)
def test_parse_3_epoch(self):
"""Test parsing valid rpm name with epoch preceeding package name."""
name, epoch, ver, rel, arch = rpm.parse_rpm_name('3:Agda-2.5.2-9.fc27.x86_64.rpm')
self.assertEqual("3", epoch)
self.assertEqual("Agda", name)
self.assertEqual("2.5.2", ver)
self.assertEqual("9.fc27", rel)
self.assertEqual("x86_64", arch)
def test_parse_2_rpm(self):
"""Test parsing valid rpm name."""
name, epoch, ver, rel, arch = rpm.parse_rpm_name('Agda-2.5.2-9.fc27.x86_64.rpm')
self.assertEqual(None, epoch)
self.assertEqual("Agda", name)
self.assertEqual("2.5.2", ver)
self.assertEqual("9.fc27", rel)
self.assertEqual("x86_64", arch)
def test_parse_1_srpm(self):
"""Test parsing valid srpm name."""
name, epoch, ver, rel, arch = rpm.parse_rpm_name('389-ds-base-1.3.7.8-1.fc27.src.rpm')
self.assertEqual(None, epoch)
self.assertEqual("389-ds-base", name)
self.assertEqual("1.3.7.8", ver)
self.assertEqual("1.fc27", rel)
self.assertEqual("src", arch)
def test_parse_6_invalid_rpmname(self):
"""Test parsing invalid rpm name."""
with self.assertRaises(rpm.RPMParseException):
rpm.parse_rpm_name('foo')
with self.assertRaises(rpm.RPMParseException):
rpm.parse_rpm_name('foo.rpm')
with self.assertRaises(rpm.RPMParseException):
rpm.parse_rpm_name('foo-1.3.x86.rpm')
with self.assertRaises(rpm.RPMParseException):
rpm.parse_rpm_name('2:389-ds-base-4:1.3.7.8-1.fc27.src.rpm')
def test_split_packagename(self):
"""Test splitting package name into N,E,V,R,A."""
pkg_name = "bash-0:4.2.46-20.el7_2.x86_64.rpm"
name, epoch, version, release, arch = rpm.parse_rpm_name(pkg_name, default_epoch='0')
assert name == "bash"
assert epoch == "0"
assert version == "4.2.46"
assert release == "20.el7_2"
assert arch == "x86_64"
def _get_source_package_id(self, pkg):
source_package_id = None
if pkg["srpm"]:
name, epoch, ver, rel, arch = rpm.parse_rpm_name(
pkg["srpm"], default_epoch=pkg["epoch"])
name_id = self.package_name_map[name]
evr_id = self.evr_map[(epoch, ver, rel)]
arch_id = self.arch_map[arch]
source_package_id = self.package_map[(name_id, evr_id, arch_id)]
return source_package_id
def _get_source_packages(packages):
unique_source_packages = set()
for pkg in packages:
if pkg["srpm"]:
unique_source_packages.add(rpm.parse_rpm_name(pkg["srpm"], default_epoch=pkg["epoch"],
raise_exception=True))
source_packages = []
for name, epoch, ver, rel, arch in unique_source_packages:
source_packages.append(dict(name=name, epoch=epoch, ver=ver, rel=rel, arch=arch,
srpm=None, summary=None, description=None))
return source_packages
def test_parse_7_invalid_noraise(self):
"""Test parsing invalid rpm name."""
for name in [
'foo',
'foo.rpm',
'foo-1.3.x86.rpm',
'2:389-ds-base-4:1.3.7.8-1.fc27.src.rpm',
]:
name, epoch, ver, rel, arch = rpm.parse_rpm_name(name)
self.assertEqual(None, epoch)
self.assertEqual("", name)
self.assertEqual("", ver)
self.assertEqual("", rel)
self.assertEqual("", arch)
def process_list(self, data):
""" Processes whole package_list and returns info. """
packages = data.get("package_list")
response = {"package_list": {}}
db_connection = self.db_pool.get_connection()
with db_connection.get_cursor() as cursor:
for package in packages:
name, epoch, version, release, arch = parse_rpm_name(package, default_epoch='0')
cursor.execute("""select distinct p.summary, p.description,
pn2.name, evr2.epoch, evr2.version, evr2.release, a2.name,
cs.label, cs.name, a.name, r.releasever,
pn3.name
from package p
left join package_name pn on pn.id = p.name_id
left join arch ap on p.arch_id = ap.id
left join evr on p.evr_id = evr.id
left join package p2 on p2.id = p.source_package_id
left join package_name pn2 on p2.name_id = pn2.id
left join evr evr2 on p2.evr_id = evr2.id
left join arch a2 on p2.arch_id = a2.id
left join pkg_repo pr on pr.pkg_id = p.id
left join repo r on r.id = pr.repo_id
left join content_set cs on cs.id = r.content_set_id
left join arch a on a.id = r.basearch_id
left join package p3 on p3.source_package_id = p.id
left join package_name pn3 on p3.name_id = pn3.id
where pn.name = '%s'
and ap.name = '%s'
and evr.epoch = '%s'
and evr.version = '%s'
and evr.release = '%s'
""" % (name, arch, epoch, version, release))
query = cursor.fetchall()
pkgs = {}
pkgs[package] = {}
for item in query:
pkgs[package] = {"summary": item[PACKAGE_SUMMARY],
"description": item[PACKAGE_DESCRIPTION],
"source_package": join_rpm_name(item[SOURCE_PACKAGE_NAME],
item[SOURCE_PACKAGE_EPOCH],
item[SOURCE_PACKAGE_VERSION],
item[SOURCE_PACKAGE_RELEASE],
item[SOURCE_PACKAGE_ARCH]),
"repositories": self._build_repositories(query),
"binary_package_list": self._build_binary_packages(query)
}
response["package_list"].update(pkgs)
self.db_pool.return_connection(db_connection)
return response
def process_input_packages(self, data: dict) -> (dict, dict):
"""Parse input NEVRAs and filter out unknown (or without updates) package names."""
latest_only = data.get("latest_only", False)
packages_to_process = filter_package_list(
data.get('package_list', None), latest_only)
filtered_packages_to_process = {}
update_list = {}
if packages_to_process is not None:
for pkg in packages_to_process:
update_list[pkg] = {}
name, epoch, ver, rel, arch = parse_rpm_name(pkg,
default_epoch='0')
if name in self.db_cache.packagename2id:
if self.db_cache.packagename2id[
name] in self.db_cache.updates_index:
filtered_packages_to_process[pkg] = {
'parsed_nevra': (name, epoch, ver, rel, arch)
}
return filtered_packages_to_process, update_list
def _state_import_check(self, item):
"""Check if state is in DB, return None if it's up to date and row to import otherwise."""
if item["version"] <= self.oval_state_version_map.get(item["id"], -1):
return None
evr_id = evr_operation_id = None
if item['evr'] is not None:
# FIXME: as an input to common.rpm.parse_rpm_name, we don't have function to parse evr only
fake_nevra = f"pn-{item['evr']}.noarch"
_, epoch, version, release, _ = parse_rpm_name(fake_nevra)
evr_id = self.evr_map.get((epoch, version, release))
if not evr_id:
self.logger.warning("EVR not found: %s, %s, %s", epoch, version, release)
return None
if item["evr_operation"] is not None:
evr_operation_id = self.evr_operation_map.get(item["evr_operation"])
if not evr_operation_id:
self.logger.warning("Unsupported EVR operation: %s", item["evr_operation"])
return None
return item["id"], evr_id, evr_operation_id, item["version"]
def _populate_states(self, oval_file_id, states):
query = """insert into oval_rpminfo_state (oval_id, evr_id, evr_operation_id, version) values %s
on conflict (oval_id) do update set
evr_id = EXCLUDED.evr_id, evr_operation_id = EXCLUDED.evr_operation_id,
version = EXCLUDED.version
returning id, oval_id, version"""
# Parse EVR first
for state in states:
if state['evr'] is not None:
# FIXME: as an input to common.rpm.parse_rpm_name, we don't have function to parse evr only
fake_nevra = f"pn-{state['evr']}.noarch"
_, epoch, version, release, _ = parse_rpm_name(fake_nevra)
state['evr'] = (epoch, version, release)
# Populate missing EVRs
self.package_store.populate_evrs(
{state['evr']
for state in states if state['evr'] is not None})
self._populate_data("states", states, self._state_import_check, query,
self._state_refresh_maps)
self._populate_associations("file", "states", "file_id",
"rpminfo_state_id",
"oval_file_rpminfo_state", oval_file_id,
states, self.oval_state_map)
for state in states:
if state["arch_operation"] is not None and state[
"arch_operation"] not in self.SUPPORTED_ARCH_OPERATIONS:
self.logger.warning("Unsupported arch operation: %s",
state["arch_operation"])
continue
if state[
"id"] in self.oval_state_map: # Make sure state is imported
# Simplified logic, can contain any regex but RH oval files contains only logical OR
archs = []
if state["arch"] is not None:
archs.extend([{
"id": arch
} for arch in state["arch"].split("|")])
self._populate_associations("state", "archs",
"rpminfo_state_id", "arch_id",
"oval_rpminfo_state_arch",
self.oval_state_map[state["id"]],
archs, self.package_store.arch_map)
def process_list(self, api_version, data): # pylint: disable=unused-argument
"""
Returns package details.
:param data: json request parsed into data structure
:returns: json response with package details
"""
packages = data.get('package_list', None)
# By default, don't include third party data
want_third_party = data.get('third_party', False)
packagelist = {}
if not packages:
return packagelist
for pkg in packages:
packagedata = packagelist.setdefault(pkg, {})
is_third_party = False
name, epoch, ver, rel, arch = parse_rpm_name(pkg,
default_epoch='0')
if name in self.cache.packagename2id \
and (epoch, ver, rel) in self.cache.evr2id \
and arch in self.cache.arch2id:
name_id = self.cache.packagename2id[name]
evr_id = self.cache.evr2id[(epoch, ver, rel)]
arch_id = self.cache.arch2id[arch]
pkg_id = self.cache.nevra2pkgid.get((name_id, evr_id, arch_id),
None)
if pkg_id:
pkg_detail = self.cache.package_details[pkg_id]
packagedata['summary'] = self.cache.strings.get(
pkg_detail[PKG_SUMMARY_ID], None)
packagedata['description'] = self.cache.strings.get(
pkg_detail[PKG_DESC_ID], None)
packagedata['source_package'] = self._get_source_package(
pkg_detail)
packagedata['repositories'] = []
packagedata[
'package_list'] = self._get_built_binary_packages(
pkg_id)
if pkg_id in self.cache.pkgid2repoids:
for repo_id in self.cache.pkgid2repoids[pkg_id]:
repodetail = self.cache.repo_detail[repo_id]
is_third_party = is_third_party or bool(
repodetail[REPO_THIRD_PARTY])
repodata = {
'label':
repodetail[REPO_LABEL],
'name':
repodetail[REPO_NAME],
'basearch':
utils.none2empty(repodetail[REPO_BASEARCH]),
'releasever':
utils.none2empty(repodetail[REPO_RELEASEVER]),
}
packagedata['repositories'].append(repodata)
# If the package is third party, then remove it from result
if not want_third_party and is_third_party:
del packagelist[pkg]
response = {'package_list': packagelist}
return response
