def _fetch_epoch_mapping(self): epoch_map = self.distro.get_dependency_config("epoch_map", quiet=True) if not epoch_map: epoch_map = {} built_epochs = {} for name in self.python_names: if name in epoch_map: built_epochs[name] = epoch_map.pop(name) else: built_epochs[name] = self.OPENSTACK_EPOCH # Exclude names from the epoch map that we never downloaded in the # first place (since these are not useful and should not be set in # the first place). try: raw_downloaded = sh.load_file(self.build_requires_filename) downloaded_reqs = pip_helper.parse_requirements(raw_downloaded) except IOError as e: if e.errno != errno.ENOENT: raise else: downloaded_names = set([req.key for req in downloaded_reqs]) tmp_epoch_map = {} for (name, epoch) in six.iteritems(epoch_map): if name.lower() in downloaded_names: tmp_epoch_map[name] = epoch else: LOG.debug("Discarding %s:%s from the epoch mapping since" " it was not part of the downloaded build" " requirements", name, epoch) epoch_map = tmp_epoch_map epoch_map.update(built_epochs) return epoch_map
def _fetch_epoch_mapping(self): epoch_map = self.distro.get_dependency_config("epoch_map", quiet=True) if not epoch_map: epoch_map = {} built_epochs = {} for name in self.python_names: if name in epoch_map: built_epochs[name] = epoch_map.pop(name) else: built_epochs[name] = self.OPENSTACK_EPOCH # Exclude names from the epoch map that we never downloaded in the # first place (since these are not useful and should not be set in # the first place). try: raw_downloaded = sh.load_file(self.build_requires_filename) downloaded_reqs = pip_helper.parse_requirements(raw_downloaded) except IOError as e: if e.errno != errno.ENOENT: raise else: downloaded_names = set([req.key for req in downloaded_reqs]) tmp_epoch_map = {} for (name, epoch) in six.iteritems(epoch_map): if name.lower() in downloaded_names: tmp_epoch_map[name] = epoch else: LOG.debug( "Discarding %s:%s from the epoch mapping since" " it was not part of the downloaded build" " requirements", name, epoch) epoch_map = tmp_epoch_map epoch_map.update(built_epochs) return epoch_map
def _extract_pip_requires(self, fn): if not sh.isfile(fn): return [] LOG.debug("Resolving dependencies from %s.", colorizer.quote(fn)) pips_needed = pip_helper.parse_requirements(sh.load_file(fn)) matchings = [] for req in pips_needed: (pkg_info, from_pip) = self._match_pip_requires(req) matchings.append({"requirement": req, "package": pkg_info, "from_pip": from_pip, "needed_by": fn}) return matchings
def _extract_pip_requires(self, fn): if not sh.isfile(fn): return [] LOG.debug("Resolving dependencies from %s.", colorizer.quote(fn)) pips_needed = pip_helper.parse_requirements(sh.load_file(fn)) matchings = [] for req in pips_needed: (pkg_info, from_pip) = self._match_pip_requires(req) matchings.append({ 'requirement': req, 'package': pkg_info, 'from_pip': from_pip, 'needed_by': fn, }) return matchings
def main(): if len(sys.argv) < 3: print("%s distro_yaml root_dir ..." % sys.argv[0]) return 1 root_dirs = sys.argv[2:] yaml_fn = sh.abspth(sys.argv[1]) requires_files = [] for d in root_dirs: all_contents = sh.listdir(d, recursive=True, files_only=True) requires_files = [sh.abspth(f) for f in all_contents if re.search(r"(test|pip)[-]requires$", f, re.I)] requires_files = sorted(list(set(requires_files))) requirements = [] source_requirements = {} for fn in requires_files: source_requirements[fn] = [] for req in pip_helper.parse_requirements(sh.load_file(fn)): requirements.append(req.key.lower().strip()) source_requirements[fn].append(req.key.lower().strip()) print("Comparing pips/pip2pkgs in %s to those found in %s" % (yaml_fn, root_dirs)) for fn in sorted(requires_files): print(" + " + str(fn)) requirements = set(requirements) print("All known requirements:") for r in sorted(requirements): print("+ " + str(r)) distro_yaml = utils.load_yaml(yaml_fn) components = distro_yaml.get('components', {}) all_known_names = [] components_pips = {} for (c, details) in components.items(): components_pips[c] = [] pip2pkgs = details.get('pip_to_package', []) pips = details.get('pips', []) known_names = [] for item in pip2pkgs: known_names.append(item['name'].lower().strip()) for item in pips: known_names.append(item['name'].lower().strip()) components_pips[c].extend(known_names) all_known_names.extend(known_names) all_known_names = sorted(list(set(all_known_names))) not_needed = [] for n in all_known_names: if n not in requirements: not_needed.append(n) if not_needed: print("The following distro yaml mappings may not be needed:") for n in sorted(not_needed): msg = " + %s (" % (n) # Find which components said they need this... for (c, known_names) in components_pips.items(): if n in known_names: msg += c + "," msg += ")" print(msg) not_found = [] for n in requirements: name = n.lower().strip() if name not in all_known_names: not_found.append(name) not_found = sorted(list(set(not_found))) if not_found: print("The following distro yaml mappings may be required but were not found:") for n in sorted(not_found): msg = " + %s" % (n) msg += " (" # Find which file/s said they need this... for (fn, reqs) in source_requirements.items(): matched = False for r in reqs: if r.lower().strip() == name: matched = True if matched: msg += fn + "," msg += ")" print(msg) return len(not_found) + len(not_needed)
def main(): if len(sys.argv) < 3: print("%s distro_yaml root_dir ..." % sys.argv[0]) return 1 root_dirs = sys.argv[2:] yaml_fn = sh.abspth(sys.argv[1]) requires_files = [] for d in root_dirs: all_contents = sh.listdir(d, recursive=True, files_only=True) requires_files = [ sh.abspth(f) for f in all_contents if re.search(r"(test|pip)[-]requires$", f, re.I) ] requires_files = sorted(list(set(requires_files))) requirements = [] source_requirements = {} for fn in requires_files: source_requirements[fn] = [] for req in pip_helper.parse_requirements(sh.load_file(fn)): requirements.append(req.key.lower().strip()) source_requirements[fn].append(req.key.lower().strip()) print("Comparing pips/pip2pkgs in %s to those found in %s" % (yaml_fn, root_dirs)) for fn in sorted(requires_files): print(" + " + str(fn)) requirements = set(requirements) print("All known requirements:") for r in sorted(requirements): print("+ " + str(r)) distro_yaml = utils.load_yaml(yaml_fn) components = distro_yaml.get('components', {}) all_known_names = [] components_pips = {} for (c, details) in components.items(): components_pips[c] = [] pip2pkgs = details.get('pip_to_package', []) pips = details.get('pips', []) known_names = [] for item in pip2pkgs: known_names.append(item['name'].lower().strip()) for item in pips: known_names.append(item['name'].lower().strip()) components_pips[c].extend(known_names) all_known_names.extend(known_names) all_known_names = sorted(list(set(all_known_names))) not_needed = [] for n in all_known_names: if n not in requirements: not_needed.append(n) if not_needed: print("The following distro yaml mappings may not be needed:") for n in sorted(not_needed): msg = " + %s (" % (n) # Find which components said they need this... for (c, known_names) in components_pips.items(): if n in known_names: msg += c + "," msg += ")" print(msg) not_found = [] for n in requirements: name = n.lower().strip() if name not in all_known_names: not_found.append(name) not_found = sorted(list(set(not_found))) if not_found: print( "The following distro yaml mappings may be required but were not found:" ) for n in sorted(not_found): msg = " + %s" % (n) msg += " (" # Find which file/s said they need this... for (fn, reqs) in source_requirements.items(): matched = False for r in reqs: if r.lower().strip() == name: matched = True if matched: msg += fn + "," msg += ")" print(msg) return len(not_found) + len(not_needed)
def _fetch_epoch_mapping(self): epoch_map = self.distro.get_dependency_config("epoch_map", quiet=True) if not epoch_map: epoch_map = {} epoch_skips = self.distro.get_dependency_config("epoch_skips", quiet=True) if not epoch_skips: epoch_skips = _DEFAULT_SKIP_EPOCHS if not isinstance(epoch_skips, (list, tuple)): epoch_skips = [i.strip() for i in epoch_skips.split(",")] built_epochs = {} for name in self.python_names: if name in epoch_map: built_epochs[name] = str(epoch_map.pop(name)) else: built_epochs[name] = str(self.OPENSTACK_EPOCH) # Ensure epochs set by a yum searching (that are not in the list of # epochs to provide) are correctly set when building dependent # packages... keep_names = set() try: yum_satisfies = sh.load_file(self.yum_satisfies_filename) except IOError as e: if e.errno != errno.ENOENT: raise else: for line in yum_satisfies.splitlines(): raw_req_rpm = utils.parse_json(line) req = pip_helper.extract_requirement(raw_req_rpm['requirement']) if req.key in epoch_map: LOG.debug("Ensuring manually set epoch is retained for" " requirement '%s' with epoch %s", req, epoch_map[req.key]) keep_names.add(req.key) else: rpm_info = raw_req_rpm['rpm'] rpm_epoch = rpm_info.get('epoch') if rpm_epoch and str(rpm_epoch) not in epoch_skips: LOG.debug("Adding in yum satisfiable package %s for" " requirement '%s' with epoch %s from repo %s", rpm_info['name'], req, rpm_epoch, rpm_info['repo']) keep_names.add(req.key) epoch_map[req.key] = str(rpm_epoch) # Exclude names from the epoch map that we never downloaded in the # first place or that we did not just set automatically (since these # are not useful and should not be set in the first place). try: raw_downloaded = sh.load_file(self.build_requires_filename) downloaded_reqs = pip_helper.parse_requirements(raw_downloaded) except IOError as e: if e.errno != errno.ENOENT: raise else: downloaded_names = set([req.key for req in downloaded_reqs]) tmp_epoch_map = {} for (name, epoch) in six.iteritems(epoch_map): name = name.lower() if name in downloaded_names or name in keep_names: tmp_epoch_map[name] = str(epoch) else: LOG.debug("Discarding %s:%s from the epoch mapping since" " it was not part of the downloaded (or automatically" " included) build requirements", name, epoch) epoch_map = tmp_epoch_map epoch_map.update(built_epochs) return epoch_map