def _fetch_epoch_mapping(self):
epoch_map = self.distro.get_dependency_config("epoch_map", quiet=True)
if not epoch_map:
epoch_map = {}
built_epochs = {}
for name in self.python_names:
if name in epoch_map:
built_epochs[name] = epoch_map.pop(name)
else:
built_epochs[name] = self.OPENSTACK_EPOCH
# Exclude names from the epoch map that we never downloaded in the
# first place (since these are not useful and should not be set in
# the first place).
try:
raw_downloaded = sh.load_file(self.build_requires_filename)
downloaded_reqs = pip_helper.parse_requirements(raw_downloaded)
except IOError as e:
if e.errno != errno.ENOENT:
raise
else:
downloaded_names = set([req.key for req in downloaded_reqs])
tmp_epoch_map = {}
for (name, epoch) in six.iteritems(epoch_map):
if name.lower() in downloaded_names:
tmp_epoch_map[name] = epoch
else:
LOG.debug("Discarding %s:%s from the epoch mapping since"
" it was not part of the downloaded build"
" requirements", name, epoch)
epoch_map = tmp_epoch_map
epoch_map.update(built_epochs)
return epoch_map
def _fetch_epoch_mapping(self):
epoch_map = self.distro.get_dependency_config("epoch_map", quiet=True)
if not epoch_map:
epoch_map = {}
built_epochs = {}
for name in self.python_names:
if name in epoch_map:
built_epochs[name] = epoch_map.pop(name)
else:
built_epochs[name] = self.OPENSTACK_EPOCH
# Exclude names from the epoch map that we never downloaded in the
# first place (since these are not useful and should not be set in
# the first place).
try:
raw_downloaded = sh.load_file(self.build_requires_filename)
downloaded_reqs = pip_helper.parse_requirements(raw_downloaded)
except IOError as e:
if e.errno != errno.ENOENT:
raise
else:
downloaded_names = set([req.key for req in downloaded_reqs])
tmp_epoch_map = {}
for (name, epoch) in six.iteritems(epoch_map):
if name.lower() in downloaded_names:
tmp_epoch_map[name] = epoch
else:
LOG.debug(
"Discarding %s:%s from the epoch mapping since"
" it was not part of the downloaded build"
" requirements", name, epoch)
epoch_map = tmp_epoch_map
epoch_map.update(built_epochs)
return epoch_map
def _extract_pip_requires(self, fn):
if not sh.isfile(fn):
return []
LOG.debug("Resolving dependencies from %s.", colorizer.quote(fn))
pips_needed = pip_helper.parse_requirements(sh.load_file(fn))
matchings = []
for req in pips_needed:
(pkg_info, from_pip) = self._match_pip_requires(req)
matchings.append({"requirement": req, "package": pkg_info, "from_pip": from_pip, "needed_by": fn})
return matchings
def _extract_pip_requires(self, fn):
if not sh.isfile(fn):
return []
LOG.debug("Resolving dependencies from %s.", colorizer.quote(fn))
pips_needed = pip_helper.parse_requirements(sh.load_file(fn))
matchings = []
for req in pips_needed:
(pkg_info, from_pip) = self._match_pip_requires(req)
matchings.append({
'requirement': req,
'package': pkg_info,
'from_pip': from_pip,
'needed_by': fn,
})
return matchings
def main():
if len(sys.argv) < 3:
print("%s distro_yaml root_dir ..." % sys.argv[0])
return 1
root_dirs = sys.argv[2:]
yaml_fn = sh.abspth(sys.argv[1])
requires_files = []
for d in root_dirs:
all_contents = sh.listdir(d, recursive=True, files_only=True)
requires_files = [sh.abspth(f) for f in all_contents
if re.search(r"(test|pip)[-]requires$", f, re.I)]
requires_files = sorted(list(set(requires_files)))
requirements = []
source_requirements = {}
for fn in requires_files:
source_requirements[fn] = []
for req in pip_helper.parse_requirements(sh.load_file(fn)):
requirements.append(req.key.lower().strip())
source_requirements[fn].append(req.key.lower().strip())
print("Comparing pips/pip2pkgs in %s to those found in %s" % (yaml_fn, root_dirs))
for fn in sorted(requires_files):
print(" + " + str(fn))
requirements = set(requirements)
print("All known requirements:")
for r in sorted(requirements):
print("+ " + str(r))
distro_yaml = utils.load_yaml(yaml_fn)
components = distro_yaml.get('components', {})
all_known_names = []
components_pips = {}
for (c, details) in components.items():
components_pips[c] = []
pip2pkgs = details.get('pip_to_package', [])
pips = details.get('pips', [])
known_names = []
for item in pip2pkgs:
known_names.append(item['name'].lower().strip())
for item in pips:
known_names.append(item['name'].lower().strip())
components_pips[c].extend(known_names)
all_known_names.extend(known_names)
all_known_names = sorted(list(set(all_known_names)))
not_needed = []
for n in all_known_names:
if n not in requirements:
not_needed.append(n)
if not_needed:
print("The following distro yaml mappings may not be needed:")
for n in sorted(not_needed):
msg = " + %s (" % (n)
# Find which components said they need this...
for (c, known_names) in components_pips.items():
if n in known_names:
msg += c + ","
msg += ")"
print(msg)
not_found = []
for n in requirements:
name = n.lower().strip()
if name not in all_known_names:
not_found.append(name)
not_found = sorted(list(set(not_found)))
if not_found:
print("The following distro yaml mappings may be required but were not found:")
for n in sorted(not_found):
msg = " + %s" % (n)
msg += " ("
# Find which file/s said they need this...
for (fn, reqs) in source_requirements.items():
matched = False
for r in reqs:
if r.lower().strip() == name:
matched = True
if matched:
msg += fn + ","
msg += ")"
print(msg)
return len(not_found) + len(not_needed)
def main():
if len(sys.argv) < 3:
print("%s distro_yaml root_dir ..." % sys.argv[0])
return 1
root_dirs = sys.argv[2:]
yaml_fn = sh.abspth(sys.argv[1])
requires_files = []
for d in root_dirs:
all_contents = sh.listdir(d, recursive=True, files_only=True)
requires_files = [
sh.abspth(f) for f in all_contents
if re.search(r"(test|pip)[-]requires$", f, re.I)
]
requires_files = sorted(list(set(requires_files)))
requirements = []
source_requirements = {}
for fn in requires_files:
source_requirements[fn] = []
for req in pip_helper.parse_requirements(sh.load_file(fn)):
requirements.append(req.key.lower().strip())
source_requirements[fn].append(req.key.lower().strip())
print("Comparing pips/pip2pkgs in %s to those found in %s" %
(yaml_fn, root_dirs))
for fn in sorted(requires_files):
print(" + " + str(fn))
requirements = set(requirements)
print("All known requirements:")
for r in sorted(requirements):
print("+ " + str(r))
distro_yaml = utils.load_yaml(yaml_fn)
components = distro_yaml.get('components', {})
all_known_names = []
components_pips = {}
for (c, details) in components.items():
components_pips[c] = []
pip2pkgs = details.get('pip_to_package', [])
pips = details.get('pips', [])
known_names = []
for item in pip2pkgs:
known_names.append(item['name'].lower().strip())
for item in pips:
known_names.append(item['name'].lower().strip())
components_pips[c].extend(known_names)
all_known_names.extend(known_names)
all_known_names = sorted(list(set(all_known_names)))
not_needed = []
for n in all_known_names:
if n not in requirements:
not_needed.append(n)
if not_needed:
print("The following distro yaml mappings may not be needed:")
for n in sorted(not_needed):
msg = " + %s (" % (n)
# Find which components said they need this...
for (c, known_names) in components_pips.items():
if n in known_names:
msg += c + ","
msg += ")"
print(msg)
not_found = []
for n in requirements:
name = n.lower().strip()
if name not in all_known_names:
not_found.append(name)
not_found = sorted(list(set(not_found)))
if not_found:
print(
"The following distro yaml mappings may be required but were not found:"
)
for n in sorted(not_found):
msg = " + %s" % (n)
msg += " ("
# Find which file/s said they need this...
for (fn, reqs) in source_requirements.items():
matched = False
for r in reqs:
if r.lower().strip() == name:
matched = True
if matched:
msg += fn + ","
msg += ")"
print(msg)
return len(not_found) + len(not_needed)
def _fetch_epoch_mapping(self):
epoch_map = self.distro.get_dependency_config("epoch_map", quiet=True)
if not epoch_map:
epoch_map = {}
epoch_skips = self.distro.get_dependency_config("epoch_skips",
quiet=True)
if not epoch_skips:
epoch_skips = _DEFAULT_SKIP_EPOCHS
if not isinstance(epoch_skips, (list, tuple)):
epoch_skips = [i.strip() for i in epoch_skips.split(",")]
built_epochs = {}
for name in self.python_names:
if name in epoch_map:
built_epochs[name] = str(epoch_map.pop(name))
else:
built_epochs[name] = str(self.OPENSTACK_EPOCH)
# Ensure epochs set by a yum searching (that are not in the list of
# epochs to provide) are correctly set when building dependent
# packages...
keep_names = set()
try:
yum_satisfies = sh.load_file(self.yum_satisfies_filename)
except IOError as e:
if e.errno != errno.ENOENT:
raise
else:
for line in yum_satisfies.splitlines():
raw_req_rpm = utils.parse_json(line)
req = pip_helper.extract_requirement(raw_req_rpm['requirement'])
if req.key in epoch_map:
LOG.debug("Ensuring manually set epoch is retained for"
" requirement '%s' with epoch %s", req,
epoch_map[req.key])
keep_names.add(req.key)
else:
rpm_info = raw_req_rpm['rpm']
rpm_epoch = rpm_info.get('epoch')
if rpm_epoch and str(rpm_epoch) not in epoch_skips:
LOG.debug("Adding in yum satisfiable package %s for"
" requirement '%s' with epoch %s from repo %s",
rpm_info['name'], req, rpm_epoch, rpm_info['repo'])
keep_names.add(req.key)
epoch_map[req.key] = str(rpm_epoch)
# Exclude names from the epoch map that we never downloaded in the
# first place or that we did not just set automatically (since these
# are not useful and should not be set in the first place).
try:
raw_downloaded = sh.load_file(self.build_requires_filename)
downloaded_reqs = pip_helper.parse_requirements(raw_downloaded)
except IOError as e:
if e.errno != errno.ENOENT:
raise
else:
downloaded_names = set([req.key for req in downloaded_reqs])
tmp_epoch_map = {}
for (name, epoch) in six.iteritems(epoch_map):
name = name.lower()
if name in downloaded_names or name in keep_names:
tmp_epoch_map[name] = str(epoch)
else:
LOG.debug("Discarding %s:%s from the epoch mapping since"
" it was not part of the downloaded (or automatically"
" included) build requirements", name, epoch)
epoch_map = tmp_epoch_map
epoch_map.update(built_epochs)
return epoch_map