def keep_n_latest_packages(self, packages, n=1): """ Keep n latest non-modular packages, modular packages are kept only if they are referenced by some of remaining modules Parameter packages: sorted list of Packages objects, oldest goes first """ packages_to_keep = [] non_modular_pkgs = [] for package in packages: if package.is_modular: for module_name_stream, packages_ref_by_module in \ self.repos.pkgs_from_modules.items(): pkgs_filenames_from_modules = [ pkg.filename for pkg in packages_ref_by_module ] if package.filename in pkgs_filenames_from_modules \ and module_name_stream in self.repos.modules: # this skips modular pkgs that are not referenced by module packages_to_keep.append(package) else: non_modular_pkgs.append(package) # filter non-modular pkgs per arches, there can be rpms with different arches # for package in one repository pkgs_per_arch = defaultdict(list) for pkg in non_modular_pkgs: _, _, _, _, arch = split_filename(pkg.filename) pkgs_per_arch[arch].append(pkg) latest_pkgs_per_arch = [] for pkgs in pkgs_per_arch.values(): latest_pkgs_per_arch += pkgs[-n:] packages[:] = latest_pkgs_per_arch + packages_to_keep
def get_blacklisted_packages(self, package_list): """ Finds blacklisted packages in output sets """ blacklisted_pkgs = [] for pattern_name, globbing, pattern_arch in self._parse_blacklist_config( ): for package in package_list: name, _, _, _, arch = split_filename(package.filename) blacklisted = False if globbing: if name.startswith(pattern_name): blacklisted = True else: if name == pattern_name: blacklisted = True if pattern_arch: if arch != pattern_arch: blacklisted = False if blacklisted: blacklisted_pkgs.append(package) return blacklisted_pkgs
def test_keep_n_newest_packages_multi_arch(mock_ubipop_runner): packages = [ get_test_pkg( name="tomcatjss", filename="tomcatjss-7.3.6-1.noarch.rpm", ), get_test_pkg( name="tomcatjss", filename="tomcatjss-7.3.6-1.x86_64.rpm", ), get_test_pkg( name="tomcatjss", filename="tomcatjss-7.3.6-1.i686.rpm", ), get_test_pkg( name="tomcatjss", filename="tomcatjss-7.3.5-1.i686.rpm", ), ] packages.sort() mock_ubipop_runner.keep_n_latest_packages(packages) assert len(packages) == 3 arches_expected = ['noarch', 'x86_64', 'i686'] arches_current = [] for pkg in packages: _, _, _, _, arch = split_filename(pkg.filename) arches_current.append(arch) assert sorted(arches_current) == sorted(arches_expected)
def _modular_rpms_filenames(self, modules): config_map = {} for module_config in self._ubi_config: key = module_config.name + module_config.stream config_map[key] = module_config.profiles filenames = set() for module in modules: key = module.name + module.stream pkgs_names = [] # get rpm names from the modulemd profiles for profile in config_map.get(key) or []: if module.profiles: pkgs_names.extend(module.profiles.get(profile) or []) for filename in module.artifacts_filenames: # skip source rpms if filename.endswith(".src.rpm"): continue # if need to take only some packages from profiles if pkgs_names: name, _, _, _, _ = split_filename(filename) if name not in pkgs_names: continue filenames.add(filename) return filenames
def __init__(self, name, filename, sourcerpm_filename=None, is_modular=False): self.name = name self.filename = filename self.sourcerpm_filename = sourcerpm_filename self.is_modular = is_modular # return name, ver, rel, epoch, arch _, self.version, self.release, self.epoch, _ = split_filename(self.filename) self.evr_tuple = (self.epoch, self.version, self.release)
def _f(*args, **kwargs): if debug_only and "debug" not in args[0].repo_id: return if len(args) > 1 and args[1] == package_name_or_filename_or_list: return [get_test_pkg(name=args[1], filename=args[1] + '.rpm')] if isinstance(package_name_or_filename_or_list, list): if kwargs['filename'] in package_name_or_filename_or_list: return [ get_test_pkg(name=split_filename(kwargs['filename'])[0], filename=kwargs['filename']), ] if 'filename' in kwargs and package_name_or_filename_or_list == kwargs[ 'filename']: return [ get_test_pkg(name=split_filename(kwargs['filename'])[0], filename=kwargs['filename']), ]
def get_packages_from_module(self, input_modules, package_name=None): """ Gathers packages from module. """ ret_rpms = [] ret_debug_rpms = [] regex = r'\d+:' reg = re.compile(regex) for module in input_modules: for rpm_nevra in module.packages: rpm_without_epoch = reg.sub('', rpm_nevra) name, _, _, _, arch = split_filename(rpm_without_epoch) # skip source package, they are taken from pkgs metadata if arch == 'src': continue if package_name and name != package_name: continue rpm_filename = rpm_without_epoch + '.rpm' # Check existence of rpm in binary rpm repos rpms = [] for in_repo_rpm in self.repos.in_repos.rpm: res = self.pulp.search_rpms(in_repo_rpm, filename=rpm_filename) if res: rpms.extend(res) if rpms: rpms[0].is_modular = True ret_rpms.append(rpms[0]) else: # Check existence of rpm in debug repos debug_rpms = [] for in_repo_debug in self.repos.in_repos.debug: res = self.pulp.search_rpms(in_repo_debug, filename=rpm_filename) if res: debug_rpms.extend(res) if debug_rpms: debug_rpms[0].is_modular = True ret_debug_rpms.append(debug_rpms[0]) else: _LOG.warning("RPM %s is unavailable in input repos %s %s, skipping", rpm_filename, [r.repo_id for r in self.repos.in_repos.rpm], [r.repo_id for r in self.repos.in_repos.debug] ) return ret_rpms, ret_debug_rpms
def keep_n_latest_packages(self, packages, n=1): """ Keep n latest non-modular packages. Modular packages are kept only if they are referenced by some of remaining modules. Arguments: packages (List[Package]): Sorted, oldest goes first Keyword arguments: n (int): Number of non-modular package versions to keep Returns: None. The packages list is changed in-place """ packages_to_keep = [] filenames_to_keep = set() # from modular packages # Use a queue of n elements per arch pkgs_per_arch = defaultdict(lambda: deque(maxlen=n)) # Set of package filenames from modules. Modular packages in # packages list are kept if referenced here. modular_packages_filenames = set() for module_name_stream, packages_in_module in self.repos.pkgs_from_modules.items(): if module_name_stream in self.repos.modules: modular_packages_filenames |= set([pkg.filename for pkg in packages_in_module]) for package in packages: if not package.is_modular: # filter non-modular pkgs per arches, there can be rpms # with different arches for package in one repository _, _, _, _, arch = split_filename(package.filename) pkgs_per_arch[arch].append(package) elif (package.filename in modular_packages_filenames and package.filename not in filenames_to_keep): # this skips modular pkgs that are not referenced by module packages_to_keep.append(package) filenames_to_keep.add(package.filename) # Packages already included from modules are skipped latest_pkgs_per_arch = [pkg for pkg in chain.from_iterable(pkgs_per_arch.values()) if pkg.filename not in filenames_to_keep] packages[:] = latest_pkgs_per_arch + packages_to_keep
def clean_name(name): return split_filename(name)[0]