def _anything_there(self, pip):
wanted_pip = extract_requirement(pip)
pip_there = self.helper.get_installed(wanted_pip.name)
if not pip_there:
# Nothing installed
return None
if wanted_pip.version is not None:
# Check if version wanted will work with whats installed
if str(wanted_pip.version) not in pip_there:
is_upgrading = False
for o in ['-U', '--upgrade']:
if o in pip.get('options', []):
is_upgrading = True
if is_upgrading and (wanted_pip.name not in self.upgraded):
# Upgrade should hopefully get that package to the right version....
LOG.warn(
"Upgrade is occuring for %s, even though %s is installed.",
wanted_pip, pip_there)
# Mark it so that we don't keep on flip-flopping on upgrading this
# package (ie install new, install old, install new....)
self.upgraded[wanted_pip.name] = wanted_pip
return None
else:
msg = ("Pip %s is already installed"
" and it is not compatible with desired"
" pip %s")
msg = msg % (pip_there, wanted_pip)
raise excp.DependencyException(msg)
return pip_there
def _verify_pip_requires(self):
all_pips = self.pip_requires
for details in all_pips:
req = details['requirement']
needed_by = details['needed_by']
pkg_info = details['package']
if not pkg_info:
raise excp.DependencyException(("Pip dependency '%s' needed by '%s' is not translatable to a listed"
" (from this or previously activated components) pip package"
' or a pip->package mapping!') % (req, needed_by))
def _verify_pip_requires(self):
all_pips = []
for fn in self.requires_files:
all_pips.extend(self._extract_pip_requires(fn))
for (requirement, (pkg_info, _from_pip)) in all_pips:
if not pkg_info:
raise excp.DependencyException((
"Pip dependency '%s' is not translatable to a listed"
" (from this or previously activated components) pip package"
' or a pip->package mapping!') % (requirement))
def _gather_pips_to_install(self, requires_files, extra_pips=None):
"""Analyze requires_files and extra_pips.
Updates `self.forced_pips` and `self.pips_to_install`.
Writes requirements to `self.gathered_requires_filename`.
"""
ignore_pips = set(self.python_names)
ignore_pips.update(self.ignore_pips)
forced_pips = set()
forced_distro_pips = self.distro.get_dependency_config("forced_pips",
quiet=True)
if forced_distro_pips:
forced_pips.update(forced_distro_pips)
compatibles, incompatibles = self.multipip.resolve(
extra_pips, requires_files, ignore_pips, forced_pips)
self.pips_to_install = compatibles
sh.write_file(self.gathered_requires_filename,
"\n".join(self.pips_to_install))
pip_requirements, raw_requirements = pip_helper.read_requirement_files(
[self.gathered_requires_filename])
pips_to_install = sorted(raw_requirements, cmp=sort_req)
utils.log_iterable(pips_to_install,
logger=LOG,
header="Full known python dependency list")
for (name, lines) in incompatibles.items():
LOG.warn("Incompatible requirements found for %s",
colorizer.quote(name, quote_color='red'))
for line in lines:
LOG.warn(line)
if not self.pips_to_install:
LOG.error("No valid dependencies found. Something went wrong.")
raise exc.DependencyException("No valid dependencies found")
# Translate those that we altered requirements for into a set of forced
# requirements file (and associated list).
self.forced_pips = []
forced_pip_keys = []
for req in [
pip_helper.extract_requirement(line)
for line in self.pips_to_install
]:
if req.key in incompatibles and req.key not in forced_pip_keys:
self.forced_pips.append(req)
forced_pip_keys.append(req.key)
self.forced_pips = sorted(self.forced_pips, cmp=sort_req)
forced_pips = [str(req) for req in self.forced_pips]
utils.log_iterable(forced_pips,
logger=LOG,
header="Automatically forced python dependencies")
sh.write_file(self.forced_requires_filename, "\n".join(forced_pips))
def _anything_there(self, pip):
wanted_pip = extract_requirement(pip)
pip_there = self.helper.get_installed(wanted_pip.name)
if not pip_there:
# Nothing installed
return None
if wanted_pip.version is not None:
# Check if version wanted will work with whats installed
if str(wanted_pip.version) not in pip_there:
msg = ("Pip %s is already installed"
" and it is not compatible with desired"
" pip %s")
msg = msg % (pip_there, wanted_pip)
raise excp.DependencyException(msg)
return pip_there
def validate_requirement(filename, source_req):
install_egg = None
for egg_info in own_eggs:
if egg_info['name'] == source_req.key:
install_egg = egg_info
break
if not install_egg:
return
# Ensure what we are about to install/create will actually work
# with the desired version. If it is not compatible then we should
# abort and someone should update the tag/branch in the origin
# file (or fix it via some other mechanism).
if install_egg['version'] not in source_req:
msg = ("Can not satisfy '%s' with '%s', version"
" conflict found in %s")
raise exc.DependencyException(
msg % (source_req, install_egg['req'], filename))
def _all_rpm_names(self):
# This file should have all the requirements (including test ones)
# that we need to install (and which should have been built as rpms
# in the previous build stages).
gathered_requires = sh.load_file(
self.gathered_requires_filename).splitlines()
gathered_requires = [
line.strip() for line in gathered_requires if line.strip()
]
req_names = []
reqs = []
for line in gathered_requires:
req = pip_helper.extract_requirement(line)
if req.key in req_names:
continue
req_names.append(req.key)
reqs.append(req)
rpm_names = self._convert_names_python2rpm(req_names)
# Ensure we select the right versions that is required and not a
# version that doesn't match the requirements.
desired_rpms = []
desired_rpm_names = set()
desired_rpms_formatted = []
def format_name(rpm_name, py_req):
full_name = str(rpm_name).strip()
if py_req is not None:
full_name += ",%s" % (py_req)
return full_name
def capture_rpm(rpm_name, py_req):
if rpm_name in desired_rpm_names or not rpm_name:
return
desired_rpms_formatted.append(format_name(rpm_name, py_req))
desired_rpms.append((rpm_name, py_req))
desired_rpm_names.add(rpm_name)
for (rpm_name, req) in zip(rpm_names, reqs):
capture_rpm(rpm_name, req)
for inst in self.instances:
if sh.isdir(inst.get_option("app_dir")):
req = None
rpm_name = None
try:
(rpm_name, _tpl) = self._get_template_and_rpm_name(inst)
req = inst.egg_info['req']
except AttributeError:
pass
capture_rpm(rpm_name, req)
for rpm_name in inst.package_names():
capture_rpm(rpm_name, None)
for rpm_name in self.requirements["requires"]:
capture_rpm(rpm_name, None)
cmd = [self.yumfind_executable, '-j']
desired_rpms_formatted = sorted(desired_rpms_formatted)
for p in desired_rpms_formatted:
cmd.extend(['-p', p])
header = "Validating %s required packages are still available" % (
len(desired_rpms))
utils.log_iterable(desired_rpms_formatted, header=header, logger=LOG)
rpms_located = []
rpm_names_located = set()
for matched in sh.execute(cmd)[0].splitlines():
matched = matched.strip()
if matched:
pkg = json.loads(matched)
if isinstance(pkg, dict):
rpm_names_located.add(pkg['name'])
rpms_located.append(pkg)
rpm_names_missing = desired_rpm_names - rpm_names_located
if rpm_names_missing:
# Include the python version required information (if applicable)
missing_formatted = []
for n in sorted(rpm_names_missing):
source_found = False
for (n2, py_req) in desired_rpms:
if n2 == n:
missing_formatted.append(format_name(n2, py_req))
source_found = True
break
if not source_found:
missing_formatted.append(format_name(n, None))
msg = "Could not find available rpm packages: %s"
msg = msg % (", ".join(missing_formatted))
raise excp.DependencyException(msg)
LOG.info("All %s required packages are still available!",
len(desired_rpms))
desired_rpms = []
for pkg in rpms_located:
LOG.debug("Found %s", pkg)
desired_rpms.append("%s,%s" % (pkg['name'], pkg['version']))
return list(sorted(desired_rpms))
def _gather_pips_to_install(self, requires_files, extra_pips=None):
"""Analyze requires_files and extra_pips.
Updates `self.forced_packages` and `self.pips_to_install`.
Writes requirements to `self.gathered_requires_filename`.
"""
extra_pips = extra_pips or []
cmdline = [
self.multipip_executable, "--skip-requirements-regex",
"python.*client", "--pip", self.pip_executable
]
cmdline = cmdline + extra_pips + ["-r"] + requires_files
cmdline.extend(["--ignore-package"])
cmdline.extend(OPENSTACK_PACKAGES)
cmdline.extend(SKIP_PACKAGE_NAMES)
cmdline.extend(self.python_names)
stdout, stderr = sh.execute(cmdline, check_exit_code=False)
self.pips_to_install = list(utils.splitlines_not_empty(stdout))
sh.write_file(self.gathered_requires_filename,
"\n".join(self.pips_to_install))
utils.log_iterable(sorted(self.pips_to_install),
logger=LOG,
header="Full known python dependency list")
incompatibles = collections.defaultdict(list)
if stderr:
current_name = ''
for line in stderr.strip().splitlines():
if line.endswith(": incompatible requirements"):
current_name = line.split(":", 1)[0].lower().strip()
if current_name not in incompatibles:
incompatibles[current_name] = []
else:
incompatibles[current_name].append(line)
for (name, lines) in incompatibles.items():
if not name:
continue
LOG.warn("Incompatible requirements found for %s",
colorizer.quote(name, quote_color='red'))
for line in lines:
LOG.warn(line)
if not self.pips_to_install:
LOG.error("No dependencies for OpenStack found."
"Something went wrong. Please check:")
LOG.error("'%s'" % "' '".join(cmdline))
raise exc.DependencyException(
"No dependencies for OpenStack found")
# Translate those that we altered requirements for into a set of forced
# requirements file (and associated list).
self.forced_packages = []
for req in [
pip_helper.extract_requirement(line)
for line in self.pips_to_install
]:
if req.key in incompatibles:
self.forced_packages.append(req)
sh.write_file(self.forced_requires_filename,
"\n".join([str(req) for req in self.forced_packages]))