def GetPackageInfo(self, packages, use_all=False, use_workon_only=False): """Get information about packages. Args: packages: list of package name fragments. These will be mapped to canonical portage atoms via the same process as StartWorkingOnPackages(). use_all: True iff instead of the provided package list, we should just stop working on all currently worked on atoms for the system in question. use_workon_only: True iff instead of the provided package list, we should stop working on all currently worked on atoms that define only a -9999 ebuild. Returns: Returns a list of PackageInfo tuples. """ if use_all or use_workon_only: # You can't use info to find the source code from Chrome, since that # workflow is different. ebuilds = self._GetWorkonEbuilds(filter_workon=use_workon_only, include_chrome=False) else: atoms = self._GetCanonicalAtoms(packages) ebuilds = [self._FindEbuildForPackage(atom) for atom in atoms] ebuild_to_repos = {} for ebuild in ebuilds: workon_vars = portage_util.EBuild.GetCrosWorkonVars( ebuild, portage_util.EbuildToCP(ebuild)) projects = workon_vars.project if workon_vars else [] ebuild_to_repos[ebuild] = projects repository_to_source_path = {} repo_list_result = cros_build_lib.RunCommand('repo list', shell=True, enter_chroot=True, capture_output=True, print_cmd=False) for line in repo_list_result.output.splitlines(): pieces = line.split(' : ') if len(pieces) != 2: logging.debug('Ignoring malformed repo list output line: "%s"', line) continue source_path, repository = pieces repository_to_source_path[repository] = source_path result = [] for ebuild in ebuilds: package = portage_util.EbuildToCP(ebuild) repos = ebuild_to_repos.get(ebuild, []) src_paths = [repository_to_source_path.get(repo) for repo in repos] src_paths = [path for path in src_paths if path] result.append(PackageInfo(package, repos, src_paths)) result.sort() return result
def GetPackageInfo(self, packages, use_all=False, use_workon_only=False): """Get information about packages. Args: packages: list of package name fragments. These will be mapped to canonical portage atoms via the same process as StartWorkingOnPackages(). use_all: True iff instead of the provided package list, we should just stop working on all currently worked on atoms for the system in question. use_workon_only: True iff instead of the provided package list, we should stop working on all currently worked on atoms that define only a -9999 ebuild. Returns: Returns a list of PackageInfo tuples. """ if use_all or use_workon_only: # You can't use info to find the source code from Chrome, since that # workflow is different. ebuilds = self._GetWorkonEbuilds(filter_workon=use_workon_only, include_chrome=False) else: atoms = self._GetCanonicalAtoms(packages) ebuilds = [self._FindEbuildForPackage(atom) for atom in atoms] build_root = self._src_root src_root = os.path.join(build_root, 'src') manifest = git.ManifestCheckout.Cached(build_root) ebuild_to_repos = {} ebuild_to_src_paths = collections.defaultdict(list) for ebuild in ebuilds: workon_vars = portage_util.EBuild.GetCrosWorkonVars( ebuild, portage_util.EbuildToCP(ebuild)) projects = workon_vars.project if workon_vars else [] ebuild_to_repos[ebuild] = projects ebuild_obj = portage_util.EBuild(ebuild) if ebuild_obj.is_blacklisted: # blacklisted ebuilds may have source infos incorrectly defined since # they are not validated by bots continue src_paths = ebuild_obj.GetSourceInfo(src_root, manifest).srcdirs src_paths = [ os.path.relpath(path, build_root) for path in src_paths ] ebuild_to_src_paths[ebuild] = src_paths result = [] for ebuild in ebuilds: package = portage_util.EbuildToCP(ebuild) repos = ebuild_to_repos.get(ebuild, []) src_paths = ebuild_to_src_paths.get(ebuild, []) result.append(PackageInfo(package, repos, src_paths)) result.sort() return result
def _GetCanonicalAtom(self, package): """Transform a package name or name fragment to the canonical atom. If there a multiple atoms that a package name fragment could map to, picks an arbitrary one and prints a warning. Args: package: string package name or fragment of a name. Returns: string canonical atom name (e.g. 'sys-apps/dbus') """ # Attempt to not hit portage if at all possible for speed. if package in self._GetWorkedOnAtoms(): return package # Ask portage directly what it thinks about that package. ebuild_path = self._FindEbuildForPackage(package) # If portage didn't know about that package, try and autocomplete it. if ebuild_path is None: possible_ebuilds = [] for ebuild in self._GetWorkonEbuilds(filter_on_arch=False): if package in ebuild: possible_ebuilds.append(ebuild) if not possible_ebuilds: logging.warning('Could not find canonical package for "%s"', package) return None if len(possible_ebuilds) > 1: logging.warning('Multiple autocompletes found:') for possible_ebuild in possible_ebuilds: logging.warning(' %s', possible_ebuild) autocompleted_package = portage_util.EbuildToCP( possible_ebuilds[0]) # Sanity check to avoid infinite loop. if package == autocompleted_package: logging.error('Resolved %s to itself', package) return None logging.info('Autocompleted "%s" to: "%s"', package, autocompleted_package) return self._GetCanonicalAtom(autocompleted_package) if not _IsWorkonEbuild(True, ebuild_path): logging.warning( '"%s" is a -9999 ebuild, but does not inherit from cros-workon?', ebuild_path) return None return portage_util.EbuildToCP(ebuild_path)
def _AtomsToEbuilds(self, atoms): """Maps from a list of CP atoms to a list of corresponding -9999 ebuilds. Args: atoms: iterable of portage atoms (e.g. ['sys-apps/dbus']). Returns: list of ebuilds corresponding to those atoms. """ atoms_to_ebuilds = dict([(atom, None) for atom in atoms]) for overlay in self._overlays: ebuild_paths = glob.glob( os.path.join(overlay, '*-*', '*', '*-9999.ebuild')) for ebuild_path in ebuild_paths: atom = portage_util.EbuildToCP(ebuild_path) if atom in atoms_to_ebuilds: atoms_to_ebuilds[atom] = ebuild_path ebuilds = [] for atom, ebuild in atoms_to_ebuilds.iteritems(): if ebuild is None: raise WorkonError('Could not find ebuild for atom %s' % atom) ebuilds.append(ebuild) return ebuilds
def _IsWorkonEbuild(include_chrome, ebuild_path, ebuild_contents=None): """Returns True iff the ebuild at |ebuild_path| is a workon ebuild. This means roughly that the ebuild is compatible with our cros_workon based system. For most packages, this means that it inherits the cros-workon overlay. Args: include_chrome: True iff we should include Chrome and chromium-source packages. ebuild_path: path an ebuild in question. ebuild_contents: None, or the contents of the ebuild at |ebuild_path|. If None, _IsWorkonEbuild will read the contents of the ebuild when necessary. Returns: True iff the ebuild can be used with cros_workon. """ # TODO(rcui): remove special casing of chromeos-chrome here when we make it # inherit from cros-workon / chromium-source class (chromium-os:19259). if (include_chrome and portage_util.EbuildToCP(ebuild_path) == constants.CHROME_CP): return True workon_eclasses = 'cros-workon' if include_chrome: workon_eclasses += '|chromium-source' ebuild_contents = ebuild_contents or osutils.ReadFile(ebuild_path) if re.search('^inherit .*(%s)' % workon_eclasses, ebuild_contents, re.M): return True return False
def StartWorkingOnPackages(self, packages, use_all=False, use_workon_only=False): """Mark a list of packages as being worked on locally. Args: packages: list of package name fragments. While each fragment could be a a complete portage atom, this helper will attempt to infer intent by looking for fragments in a list of all possible atoms for the system in question. use_all: True iff we should ignore the package list, and instead consider all possible atoms that we could mark as worked on locally. use_workon_only: True iff we should ignore the package list, and instead consider all possible atoms for the system in question that define only the -9999 ebuild. """ if not os.path.exists(self._sysroot): raise WorkonError('Sysroot %s is not setup.' % self._sysroot) if use_all or use_workon_only: ebuilds = self._GetWorkonEbuilds(filter_workon=use_workon_only) atoms = [portage_util.EbuildToCP(ebuild) for ebuild in ebuilds] else: atoms = self._GetCanonicalAtoms(packages) atoms = set(atoms) # Read out what atoms we're already working on. existing_atoms = self._GetWorkedOnAtoms() # Warn the user if they're requested to work on an atom that's already # marked as being worked on. for atom in atoms & existing_atoms: logging.warning('Already working on %s', atom) # If we have no new atoms to work on, we can quit now. new_atoms = atoms - existing_atoms if not new_atoms: return # Write out all these atoms to the appropriate files. current_atoms = new_atoms | existing_atoms self._SetWorkedOnAtoms(current_atoms) self._AddProjectsToPartialManifests(new_atoms) # Legacy scripts used single quotes in their output, and we carry on this # honorable tradition. logging.info("Started working on '%s' for '%s'", ' '.join(new_atoms), self._system)
def _GetLiveAtoms(self, filter_workon=False): """Get a list of atoms currently marked as being locally compiled. Args: filter_workon: True iff the list should be filtered to only those atoms without a stable version (i.e. the -9999 ebuild is the only ebuild). Returns: list of canonical portage atoms. """ atoms = self._GetWorkedOnAtoms() if filter_workon: ebuilds = _FilterWorkonOnlyEbuilds(self._AtomsToEbuilds(atoms)) return [portage_util.EbuildToCP(ebuild) for ebuild in ebuilds] return atoms
def ListAtoms(self, use_all=False, use_workon_only=False): """Returns a list of interesting atoms. By default, return a list of the atoms marked as being locally worked on for the system in question. Args: use_all: If true, return a list of all atoms we could possibly work on for the system in question. use_workon_only: If true, return a list of all atoms we could possibly work on that have no stable ebuild. Returns: a list of atoms (e.g. ['chromeos-base/shill', 'sys-apps/dbus']). """ if use_workon_only or use_all: ebuilds = self._GetWorkonEbuilds(filter_workon=use_workon_only) packages = [portage_util.EbuildToCP(ebuild) for ebuild in ebuilds] else: packages = self._GetLiveAtoms() return sorted(packages)
def _GetCanonicalAtom(self, package, find_stale=False): """Transform a package name or name fragment to the canonical atom. If there a multiple atoms that a package name fragment could map to, picks an arbitrary one and prints a warning. Args: package: string package name or fragment of a name. find_stale: if True, allow stale (missing) worked on package. Returns: string canonical atom name (e.g. 'sys-apps/dbus') """ # Attempt to not hit portage if at all possible for speed. if package in self._GetWorkedOnAtoms(): return package # Ask portage directly what it thinks about that package. ebuild_path = self._FindEbuildForPackage(package) # If portage didn't know about that package, try and autocomplete it. if ebuild_path is None: possible_ebuilds = set() for ebuild in (portage_util.EbuildToCP(ebuild) for ebuild in self._GetWorkonEbuilds( filter_on_arch=False)): if package in ebuild: possible_ebuilds.add(ebuild) # Also autocomplete from the worked-on list, in case the ebuild was # deleted. if find_stale: for ebuild in self._GetWorkedOnAtoms(): if package in ebuild: possible_ebuilds.add(ebuild) if not possible_ebuilds: logging.warning('Could not find canonical package for "%s"', package) return None # We want some consistent order for making our selection below. possible_ebuilds = sorted(possible_ebuilds) if len(possible_ebuilds) > 1: logging.warning('Multiple autocompletes found:') for possible_ebuild in possible_ebuilds: logging.warning(' %s', possible_ebuild) autocompleted_package = portage_util.EbuildToCP( possible_ebuilds[0]) # Sanity check to avoid infinite loop. if package == autocompleted_package: logging.error('Resolved %s to itself', package) return None logging.info('Autocompleted "%s" to: "%s"', package, autocompleted_package) return self._GetCanonicalAtom(autocompleted_package) if not _IsWorkonEbuild(True, ebuild_path): msg = ( 'In order to cros_workon a package, it must have a -9999 ebuild ' 'that inherits from cros-workon.\n') if '-9999' in ebuild_path: msg += ('"%s" is a -9999 ebuild, make sure it inherits from ' 'cros-workon.\n' % ebuild_path) else: msg += '"%s" is not a -9999 ebuild.\n' % ebuild_path logging.warning(msg) return None return portage_util.EbuildToCP(ebuild_path)