def keep_n_latest_packages(self, packages, n=1):
"""
Keep n latest non-modular packages,
modular packages are kept only if they are referenced by some of remaining modules
Parameter packages: sorted list of Packages objects, oldest goes first
"""
packages_to_keep = []
non_modular_pkgs = []
for package in packages:
if package.is_modular:
for module_name_stream, packages_ref_by_module in \
self.repos.pkgs_from_modules.items():
pkgs_filenames_from_modules = [
pkg.filename for pkg in packages_ref_by_module
]
if package.filename in pkgs_filenames_from_modules \
and module_name_stream in self.repos.modules:
# this skips modular pkgs that are not referenced by module
packages_to_keep.append(package)
else:
non_modular_pkgs.append(package)
# filter non-modular pkgs per arches, there can be rpms with different arches
# for package in one repository
pkgs_per_arch = defaultdict(list)
for pkg in non_modular_pkgs:
_, _, _, _, arch = split_filename(pkg.filename)
pkgs_per_arch[arch].append(pkg)
latest_pkgs_per_arch = []
for pkgs in pkgs_per_arch.values():
latest_pkgs_per_arch += pkgs[-n:]
packages[:] = latest_pkgs_per_arch + packages_to_keep
def get_blacklisted_packages(self, package_list):
"""
Finds blacklisted packages in output sets
"""
blacklisted_pkgs = []
for pattern_name, globbing, pattern_arch in self._parse_blacklist_config(
):
for package in package_list:
name, _, _, _, arch = split_filename(package.filename)
blacklisted = False
if globbing:
if name.startswith(pattern_name):
blacklisted = True
else:
if name == pattern_name:
blacklisted = True
if pattern_arch:
if arch != pattern_arch:
blacklisted = False
if blacklisted:
blacklisted_pkgs.append(package)
return blacklisted_pkgs
def test_keep_n_newest_packages_multi_arch(mock_ubipop_runner):
packages = [
get_test_pkg(
name="tomcatjss",
filename="tomcatjss-7.3.6-1.noarch.rpm",
),
get_test_pkg(
name="tomcatjss",
filename="tomcatjss-7.3.6-1.x86_64.rpm",
),
get_test_pkg(
name="tomcatjss",
filename="tomcatjss-7.3.6-1.i686.rpm",
),
get_test_pkg(
name="tomcatjss",
filename="tomcatjss-7.3.5-1.i686.rpm",
),
]
packages.sort()
mock_ubipop_runner.keep_n_latest_packages(packages)
assert len(packages) == 3
arches_expected = ['noarch', 'x86_64', 'i686']
arches_current = []
for pkg in packages:
_, _, _, _, arch = split_filename(pkg.filename)
arches_current.append(arch)
assert sorted(arches_current) == sorted(arches_expected)
def _modular_rpms_filenames(self, modules):
config_map = {}
for module_config in self._ubi_config:
key = module_config.name + module_config.stream
config_map[key] = module_config.profiles
filenames = set()
for module in modules:
key = module.name + module.stream
pkgs_names = []
# get rpm names from the modulemd profiles
for profile in config_map.get(key) or []:
if module.profiles:
pkgs_names.extend(module.profiles.get(profile) or [])
for filename in module.artifacts_filenames:
# skip source rpms
if filename.endswith(".src.rpm"):
continue
# if need to take only some packages from profiles
if pkgs_names:
name, _, _, _, _ = split_filename(filename)
if name not in pkgs_names:
continue
filenames.add(filename)
return filenames
def __init__(self, name, filename, sourcerpm_filename=None, is_modular=False):
self.name = name
self.filename = filename
self.sourcerpm_filename = sourcerpm_filename
self.is_modular = is_modular
# return name, ver, rel, epoch, arch
_, self.version, self.release, self.epoch, _ = split_filename(self.filename)
self.evr_tuple = (self.epoch, self.version, self.release)
def _f(*args, **kwargs):
if debug_only and "debug" not in args[0].repo_id:
return
if len(args) > 1 and args[1] == package_name_or_filename_or_list:
return [get_test_pkg(name=args[1], filename=args[1] + '.rpm')]
if isinstance(package_name_or_filename_or_list, list):
if kwargs['filename'] in package_name_or_filename_or_list:
return [
get_test_pkg(name=split_filename(kwargs['filename'])[0],
filename=kwargs['filename']),
]
if 'filename' in kwargs and package_name_or_filename_or_list == kwargs[
'filename']:
return [
get_test_pkg(name=split_filename(kwargs['filename'])[0],
filename=kwargs['filename']),
]
def get_packages_from_module(self, input_modules, package_name=None):
"""
Gathers packages from module.
"""
ret_rpms = []
ret_debug_rpms = []
regex = r'\d+:'
reg = re.compile(regex)
for module in input_modules:
for rpm_nevra in module.packages:
rpm_without_epoch = reg.sub('', rpm_nevra)
name, _, _, _, arch = split_filename(rpm_without_epoch)
# skip source package, they are taken from pkgs metadata
if arch == 'src':
continue
if package_name and name != package_name:
continue
rpm_filename = rpm_without_epoch + '.rpm'
# Check existence of rpm in binary rpm repos
rpms = []
for in_repo_rpm in self.repos.in_repos.rpm:
res = self.pulp.search_rpms(in_repo_rpm, filename=rpm_filename)
if res:
rpms.extend(res)
if rpms:
rpms[0].is_modular = True
ret_rpms.append(rpms[0])
else:
# Check existence of rpm in debug repos
debug_rpms = []
for in_repo_debug in self.repos.in_repos.debug:
res = self.pulp.search_rpms(in_repo_debug, filename=rpm_filename)
if res:
debug_rpms.extend(res)
if debug_rpms:
debug_rpms[0].is_modular = True
ret_debug_rpms.append(debug_rpms[0])
else:
_LOG.warning("RPM %s is unavailable in input repos %s %s, skipping",
rpm_filename,
[r.repo_id for r in self.repos.in_repos.rpm],
[r.repo_id for r in self.repos.in_repos.debug]
)
return ret_rpms, ret_debug_rpms
def keep_n_latest_packages(self, packages, n=1):
"""
Keep n latest non-modular packages.
Modular packages are kept only if they are referenced by some of
remaining modules.
Arguments:
packages (List[Package]): Sorted, oldest goes first
Keyword arguments:
n (int): Number of non-modular package versions to keep
Returns:
None. The packages list is changed in-place
"""
packages_to_keep = []
filenames_to_keep = set() # from modular packages
# Use a queue of n elements per arch
pkgs_per_arch = defaultdict(lambda: deque(maxlen=n))
# Set of package filenames from modules. Modular packages in
# packages list are kept if referenced here.
modular_packages_filenames = set()
for module_name_stream, packages_in_module in self.repos.pkgs_from_modules.items():
if module_name_stream in self.repos.modules:
modular_packages_filenames |= set([pkg.filename for pkg in packages_in_module])
for package in packages:
if not package.is_modular:
# filter non-modular pkgs per arches, there can be rpms
# with different arches for package in one repository
_, _, _, _, arch = split_filename(package.filename)
pkgs_per_arch[arch].append(package)
elif (package.filename in modular_packages_filenames
and package.filename not in filenames_to_keep):
# this skips modular pkgs that are not referenced by module
packages_to_keep.append(package)
filenames_to_keep.add(package.filename)
# Packages already included from modules are skipped
latest_pkgs_per_arch = [pkg for pkg in chain.from_iterable(pkgs_per_arch.values())
if pkg.filename not in filenames_to_keep]
packages[:] = latest_pkgs_per_arch + packages_to_keep
def clean_name(name):
return split_filename(name)[0]
