def perl_rebuild_main(options, out, err): path = pjoin(options.domain.root, "usr/lib/perl5", options.new_version) if not os.path.exists(path): perl_rebuild.error( f"version {options.new_version} doesn't seem to be installed; " f"can't find it at {path!r}") base = pjoin(options.domain.root, "/usr/lib/perl5") potential_perl_versions = [ x.replace(".", "\\.") for x in listdir_dirs(base) if x.startswith("5.") and x != options.new_version] if len(potential_perl_versions) == 1: subpattern = potential_perl_versions[0] else: subpattern = "(?:%s)" % ("|".join(potential_perl_versions),) matcher = re.compile( "/usr/lib(?:64|32)?/perl5/(?:%s|vendor_perl/%s)" % (subpattern, subpattern)).match for pkg in options.domain.all_installed_repos: contents = getattr(pkg, 'contents', ()) if not contents: continue # scan just directories... for fsobj in contents.iterdirs(): if matcher(fsobj.location): out.write(str(pkg.unversioned_atom)) break return 0
def category_dirs(self): try: return frozenset(map(intern, filterfalse( self.false_categories.__contains__, (x for x in listdir_dirs(self.base) if not x.startswith('.'))))) except EnvironmentError as e: logger.error(f"failed listing categories: {e}") return ()
def _get_categories(self, *optional_category): # return if optional_category is passed... cause it's not yet supported if optional_category: return {} try: return tuple(x for x in listdir_dirs(self.base) if x.lower() != "all") except EnvironmentError as e: raise_from(KeyError("failed fetching categories: %s" % str(e)))
def start(self): self.licenses = set() if isinstance(self.options.target_repo,SlavedTree): if 'licenses' in listdir_dirs(self.options.target_repo.location): self.licenses.update(listdir_files(pjoin(self.options.target_repo.location,"licenses"))) else: for license_dir in self.options.license_dirs: self.licenses.update(listdir_files(license_dir))
def _get_categories(self, *optional_category): # return if optional_category is passed... cause it's not yet supported if optional_category: return {} try: try: return tuple(x for x in listdir_dirs(self.location) if not x.startswith('.')) except EnvironmentError as e: raise KeyError(f"failed fetching categories: {e}") from e finally: pass
def _get_categories(self, *optional_category): # return if optional_category is passed... cause it's not yet supported if optional_category: return {} try: try: return tuple(x for x in listdir_dirs(self.location) if not x.startswith('.')) except EnvironmentError as e: compatibility.raise_from(KeyError("failed fetching categories: %s" % str(e))) finally: pass
def _get_packages(self, category): cpath = pjoin(self.base, category.lstrip(os.path.sep)) try: return tuple(listdir_dirs(cpath)) except FileNotFoundError: if category in self.categories: # ignore it, since it's PMS mandated that it be allowed. return () except EnvironmentError as e: category = pjoin(self.base, category.lstrip(os.path.sep)) raise KeyError( f'failed fetching packages for category {category}: {e}') from e
def _get_packages(self, category): cpath = pjoin(self.base, category.lstrip(os.path.sep)) try: return tuple(filterfalse( self.false_packages.__contains__, listdir_dirs(cpath))) except FileNotFoundError: if category in self.categories: # ignore it, since it's PMS mandated that it be allowed. return () except EnvironmentError as e: category = pjoin(self.base, category.lstrip(os.path.sep)) raise KeyError( f'failed fetching packages for category {category}: {e}') from e
def _get_packages(self, category): cpath = pjoin(self.base, category.lstrip(os.path.sep)) try: return tuple(ifilterfalse( self.false_packages.__contains__, listdir_dirs(cpath))) except EnvironmentError as e: if e.errno == errno.ENOENT: if category in self.categories: # ignore it, since it's PMS mandated that it be allowed. return () raise_from(KeyError( "failed fetching packages for category %s: %s" % (pjoin(self.base, category.lstrip(os.path.sep)), str(e))))
def _get_categories(self, *optional_category): # why the auto return? current porttrees don't allow/support # categories deeper then one dir. if optional_category: #raise KeyError return () cats = self.hardcoded_categories if cats is not None: return cats try: return tuple(imap(intern, ifilterfalse( self.false_categories.__contains__, (x for x in listdir_dirs(self.base) if x[0:1] != ".")))) except EnvironmentError as e: raise_from(KeyError("failed fetching categories: %s" % str(e)))
def _get_packages(self, category): cpath = pjoin(self.location, category.lstrip(os.path.sep)) l = set() d = {} bad = False try: for x in listdir_dirs(cpath): if x.startswith(".tmp.") or x.endswith(".lockfile") \ or x.startswith("-MERGING-"): continue try: pkg = versioned_CPV(category + "/" + x) except InvalidCPV: bad = True if bad or not pkg.fullver: if '-scm' in x: bad = 'scm' elif '-try' in x: bad = 'try' else: raise InvalidCPV( "%s/%s: no version component" % (category, x)) logger.error( "merged -%s pkg detected: %s/%s. " "throwing exception due to -%s not being a valid" " version component. Silently ignoring that " "specific version is not viable either since it " "would result in pkgcore stomping whatever it was " "that -%s version merged. " "This is why embrace and extend is bad, mm'kay. " "Use the offending pkg manager that merged it to " "unmerge it." % (bad, category, x, bad, bad)) raise InvalidCPV( "%s/%s: -%s version component is " "not standard." % (category, x, bad)) l.add(pkg.package) d.setdefault((category, pkg.package), []).append(pkg.fullver) except EnvironmentError as e: compatibility.raise_from(KeyError( "failed fetching packages for category %s: %s" % (pjoin(self.location, category.lstrip(os.path.sep)), str(e)))) self._versions_tmp_cache.update(d) return tuple(l)
def _get_categories(self, *optional_category): # why the auto return? current porttrees don't allow/support # categories deeper then one dir. if optional_category: # raise KeyError return () categories = set() for repo in self.trees: if repo.hardcoded_categories is not None: categories.update(repo.hardcoded_categories) if categories: return tuple(categories) try: return tuple(imap(intern, ifilterfalse( self.false_categories.__contains__, (x for x in listdir_dirs(self.base) if x[0:1] != ".")))) except EnvironmentError as e: raise_from(KeyError("failed fetching categories: %s" % str(e)))
def _get_categories(self, *optional_category): # why the auto return? current porttrees don't allow/support # categories deeper then one dir. if optional_category: # raise KeyError return () categories = set() for repo in self.trees: if repo.hardcoded_categories is not None: categories.update(repo.hardcoded_categories) if categories: return tuple(categories) try: return tuple(map(intern, filterfalse( self.false_categories.__contains__, (x for x in listdir_dirs(self.base) if x[0:1] != ".")))) except EnvironmentError as e: raise KeyError(f"failed fetching categories: {e}") from e
def _get_packages(self, category): cpath = pjoin(self.location, category.lstrip(os.path.sep)) l = set() d = {} bad = False try: for x in listdir_dirs(cpath): if x.startswith(".tmp.") or x.endswith(".lockfile") \ or x.startswith("-MERGING-"): continue try: pkg = versioned_CPV(category + "/" + x) except InvalidCPV: bad = True if bad or not pkg.fullver: if '-scm' in x: bad = 'scm' elif '-try' in x: bad = 'try' else: raise InvalidCPV("%s/%s: no version component" % (category, x)) logger.error( "merged -%s pkg detected: %s/%s. " "throwing exception due to -%s not being a valid" " version component. Silently ignoring that " "specific version is not viable either since it " "would result in pkgcore stomping whatever it was " "that -%s version merged. " "This is why embrace and extend is bad, mm'kay. " "Use the offending pkg manager that merged it to " "unmerge it.", bad, category, x, bad, bad) raise InvalidCPV("%s/%s: -%s version component is " "not standard." % (category, x, bad)) l.add(pkg.package) d.setdefault((category, pkg.package), []).append(pkg.fullver) except EnvironmentError as e: compatibility.raise_from( KeyError( "failed fetching packages for category %s: %s" % (pjoin( self.location, category.lstrip(os.path.sep)), str(e)))) self._versions_tmp_cache.update(d) return tuple(l)
def _get_packages(self, category): cpath = pjoin(self.location, category.lstrip(os.path.sep)) l = set() d = {} bad = False try: for x in listdir_dirs(cpath): if x.startswith(".tmp.") or x.endswith(".lockfile") \ or x.startswith("-MERGING-"): continue try: pkg = versioned_CPV(f'{category}/{x}') except InvalidCPV: bad = True if bad or not pkg.fullver: if '-scm' in x: bad = 'scm' elif '-try' in x: bad = 'try' else: raise InvalidCPV( f"{category}/{x}: no version component") logger.error( f'merged -{bad} pkg detected: {category}/{x}. ' f'throwing exception due to -{bad} not being a valid' ' version component. Silently ignoring that ' 'specific version is not viable either since it ' 'would result in pkgcore stomping whatever it was ' f'that -{bad} version merged. ' 'Use the offending pkg manager that merged it to ' 'unmerge it.') raise InvalidCPV( f"{category}/{x}: -{bad} version component is not standard." ) l.add(pkg.package) d.setdefault((category, pkg.package), []).append(pkg.fullver) except EnvironmentError as e: category = pjoin(self.location, category.lstrip(os.path.sep)) raise KeyError( f'failed fetching packages for category {category}: {e}' ) from e self._versions_tmp_cache.update(d) return tuple(l)
def _tmp_validate_args(parser, namespace): tmpdir = namespace.domain.pm_tmpdir dirs = () files = () if namespace.restrict and not namespace.wipe_all: # create a fake repo from tmpdir entries and pull matches from it pkg_map = {} for pkg_build_dir in glob.glob(pjoin(tmpdir, '*', '*')): try: pkg = atom_mod.atom( '=' + pkg_build_dir[len(tmpdir):].lstrip(os.path.sep)) except atom_mod.MalformedAtom: continue pkg_map.setdefault(pkg.category, {}).setdefault(pkg.package, []).append(pkg.fullver) repo = SimpleTree(pkg_map) def _remove_dir_and_empty_parent(d): """Remove a given directory tree and its parent directory, if empty.""" shutil.rmtree(d) try: os.rmdir(os.path.dirname(d)) except OSError as e: # POSIX specifies either ENOTEMPTY or EEXIST for non-empty dir # in particular, Solaris uses EEXIST in that case. if e.errno not in (errno.ENOTEMPTY, errno.EEXIST): raise removal_func = partial(_remove_dir_and_empty_parent) dirs = ((removal_func, pjoin(tmpdir, pkg.cpvstr)) for pkg in repo.itermatch(namespace.restrict)) else: # not in a configured repo dir, remove all tmpdir entries dir_removal_func = partial(shutil.rmtree) dirs = ((dir_removal_func, pjoin(tmpdir, d)) for d in listdir_dirs(tmpdir)) file_removal_func = partial(os.remove) files = ((file_removal_func, pjoin(tmpdir, f)) for f in listdir_files(tmpdir)) namespace.remove = chain(dirs, files)
def _get_packages(self, category): cpath = pjoin(self.location, category.lstrip(os.path.sep)) l = set() d = {} bad = False try: for x in listdir_dirs(cpath): if x.startswith(".tmp.") or x.endswith(".lockfile") \ or x.startswith("-MERGING-"): continue try: pkg = versioned_CPV(f'{category}/{x}') except InvalidCPV: bad = True if bad or not pkg.fullver: if '-scm' in x: bad = 'scm' elif '-try' in x: bad = 'try' else: raise InvalidCPV(f"{category}/{x}: no version component") logger.error( f'merged -{bad} pkg detected: {category}/{x}. ' f'throwing exception due to -{bad} not being a valid' ' version component. Silently ignoring that ' 'specific version is not viable either since it ' 'would result in pkgcore stomping whatever it was ' f'that -{bad} version merged. ' 'Use the offending pkg manager that merged it to ' 'unmerge it.') raise InvalidCPV( f"{category}/{x}: -{bad} version component is not standard.") l.add(pkg.package) d.setdefault((category, pkg.package), []).append(pkg.fullver) except EnvironmentError as e: category = pjoin(self.location, category.lstrip(os.path.sep)) raise KeyError(f'failed fetching packages for category {category}: {e}') from e self._versions_tmp_cache.update(d) return tuple(l)
def finish(self): # don't check for unknown category dirs on overlays if self.options.gentoo_repo: category_dirs = set( filterfalse(self.repo.false_categories.__contains__, (x for x in listdir_dirs(self.repo.location) if x[0] != '.'))) unknown_categories = category_dirs.difference(self.repo.categories) if unknown_categories: yield UnknownCategories(sorted(unknown_categories)) arches_without_profiles = set(self.arches) - set( self.repo.profiles.arches()) if arches_without_profiles: yield ArchesWithoutProfiles(sorted(arches_without_profiles)) root_profile_dirs = {'embedded'} available_profile_dirs = set() for root, _dirs, _files in os.walk(self.profiles_dir): d = root[len(self.profiles_dir):].lstrip('/') if d: available_profile_dirs.add(d) available_profile_dirs -= self.non_profile_dirs | root_profile_dirs profile_reports = [] report_profile_warnings = lambda x: profile_reports.append( ProfileWarning(x)) report_profile_errors = lambda x: profile_reports.append( ProfileError(x)) # don't check for acceptable profile statuses on overlays if self.options.gentoo_repo: known_profile_statuses = self.known_profile_statuses else: known_profile_statuses = None # forcibly parse profiles.desc and convert log warnings/errors into reports with patch('pkgcore.log.logger.error', report_profile_errors), \ patch('pkgcore.log.logger.warning', report_profile_warnings): profiles = Profiles.parse(self.profiles_dir, self.repo.repo_id, known_status=known_profile_statuses, known_arch=self.arches) yield from profile_reports seen_profile_dirs = set() lagging_profile_eapi = defaultdict(list) for p in profiles: try: profile = profiles_mod.ProfileStack( pjoin(self.profiles_dir, p.path)) except profiles_mod.ProfileError: yield NonexistentProfilePath(p.path) continue for parent in profile.stack: seen_profile_dirs.update( dir_parents(parent.path[len(self.profiles_dir):])) # flag lagging profile EAPIs -- assumes EAPIs are sequentially # numbered which should be the case for the gentoo repo if (self.options.gentoo_repo and str(profile.eapi) < str(parent.eapi)): lagging_profile_eapi[profile].append(parent) for profile, parents in lagging_profile_eapi.items(): parent = parents[-1] yield LaggingProfileEapi(profile.name, str(profile.eapi), parent.name, str(parent.eapi)) unused_profile_dirs = available_profile_dirs - seen_profile_dirs if unused_profile_dirs: yield UnusedProfileDirs(sorted(unused_profile_dirs))