def _call_multipip(self, requirements,
requires_files=None, ignore_requirements=None):
cmdline = [self._multipip_executable]
if requires_files:
cmdline.append("-r")
cmdline.extend(requires_files)
if ignore_requirements:
cmdline.append("--ignore-package")
cmdline.extend(ignore_requirements)
if requirements:
cmdline.append("--")
cmdline.extend(requirements)
(stdout, stderr) = sh.execute(cmdline, check_exit_code=False)
compatibles = list(utils.splitlines_not_empty(stdout))
incompatibles = collections.defaultdict(list)
current_name = ''
for line in stderr.strip().splitlines():
if line.endswith(": incompatible requirements"):
current_name = line.split(":", 1)[0].lower().strip()
if current_name not in incompatibles:
incompatibles[current_name] = []
else:
incompatibles[current_name].append(line)
cleaned_incompatibles = dict()
for (requirement, lines) in six.iteritems(incompatibles):
requirement = requirement.strip()
if not requirement:
continue
if not lines:
continue
cleaned_incompatibles[requirement] = lines
incompatibles = cleaned_incompatibles
return (compatibles, incompatibles)
def _call_multipip(self, requirements, requires_files=None, ignore_requirements=None):
cmdline = [self._multipip_executable]
if requires_files:
cmdline.append("-r")
cmdline.extend(requires_files)
if ignore_requirements:
cmdline.append("--ignore-package")
cmdline.extend(ignore_requirements)
if requirements:
cmdline.append("--")
cmdline.extend(requirements)
(stdout, stderr) = sh.execute(cmdline, check_exit_code=False)
compatibles = list(utils.splitlines_not_empty(stdout))
incompatibles = collections.defaultdict(list)
current_name = ""
for line in stderr.strip().splitlines():
if line.endswith(": incompatible requirements"):
current_name = line.split(":", 1)[0].lower().strip()
if current_name not in incompatibles:
incompatibles[current_name] = []
else:
incompatibles[current_name].append(line)
cleaned_incompatibles = dict()
for (requirement, lines) in six.iteritems(incompatibles):
requirement = requirement.strip()
if not requirement:
continue
if not lines:
continue
cleaned_incompatibles[requirement] = lines
incompatibles = cleaned_incompatibles
return (compatibles, incompatibles)
def _gather_pips_to_install(self, requires_files, extra_pips=None):
"""Analyze requires_files and extra_pips.
Updates `self.forced_packages` and `self.pips_to_install`.
Writes requirements to `self.gathered_requires_filename`.
"""
extra_pips = extra_pips or []
cmdline = [
self.multipip_executable,
"--skip-requirements-regex",
"python.*client",
"--pip",
self.pip_executable
]
cmdline = cmdline + extra_pips + ["-r"] + requires_files
cmdline.extend(["--ignore-package"])
cmdline.extend(OPENSTACK_PACKAGES)
cmdline.extend(SKIP_PACKAGE_NAMES)
cmdline.extend(self.python_names)
stdout, stderr = sh.execute(cmdline, check_exit_code=False)
self.pips_to_install = list(utils.splitlines_not_empty(stdout))
sh.write_file(self.gathered_requires_filename, "\n".join(self.pips_to_install))
utils.log_iterable(sorted(self.pips_to_install), logger=LOG,
header="Full known python dependency list")
incompatibles = collections.defaultdict(list)
if stderr:
current_name = ''
for line in stderr.strip().splitlines():
if line.endswith(": incompatible requirements"):
current_name = line.split(":", 1)[0].lower().strip()
if current_name not in incompatibles:
incompatibles[current_name] = []
else:
incompatibles[current_name].append(line)
for (name, lines) in incompatibles.items():
if not name:
continue
LOG.warn("Incompatible requirements found for %s",
colorizer.quote(name, quote_color='red'))
for line in lines:
LOG.warn(line)
if not self.pips_to_install:
LOG.error("No dependencies for OpenStack found."
"Something went wrong. Please check:")
LOG.error("'%s'" % "' '".join(cmdline))
raise exc.DependencyException("No dependencies for OpenStack found")
# Translate those that we altered requirements for into a set of forced
# requirements file (and associated list).
self.forced_packages = []
for req in [pip_helper.extract_requirement(line) for line in self.pips_to_install]:
if req.key in incompatibles:
self.forced_packages.append(req)
sh.write_file(self.forced_requires_filename,
"\n".join([str(req) for req in self.forced_packages]))
def filter_download_requires(self):
if not self.python_names:
return self.pips_to_install
cmdline = [
self.multipip_executable,
"--pip", self.pip_executable,
] + self.pips_to_install + [
"--ignore-packages",
] + self.python_names
output = sh.execute(cmdline)
pips_to_download = list(utils.splitlines_not_empty(output[0]))
return pips_to_download
def filter_download_requires(self):
if not self.python_names:
return self.pips_to_install
cmdline = [
self.multipip_executable,
"--pip",
self.pip_executable,
] + self.pips_to_install + [
"--ignore-packages",
] + self.python_names
output = sh.execute(cmdline)
pips_to_download = list(utils.splitlines_not_empty(output[0]))
return pips_to_download
def gather_pips_to_install(self, requires_files, extra_pips=None):
"""Analyze requires_files and extra_pips.
Updates `self.forced_packages` and `self.pips_to_install`.
Writes requirements to `self.gathered_requires_filename`.
"""
extra_pips = extra_pips or []
cmdline = [
self.multipip_executable,
"--skip-requirements-regex",
"python.*client",
"--pip",
self.pip_executable
]
cmdline = cmdline + extra_pips + ["-r"] + requires_files
cmdline.extend(["--ignore-package"])
cmdline.extend(OPENSTACK_PACKAGES)
cmdline.extend(self.python_names)
output = sh.execute(cmdline, check_exit_code=False)
self.pips_to_install = list(utils.splitlines_not_empty(output[0]))
conflict_descr = output[1].strip()
forced_keys = set()
if conflict_descr:
for line in conflict_descr.splitlines():
LOG.warning(line)
if line.endswith(": incompatible requirements"):
forced_keys.add(line.split(":", 1)[0].lower())
sh.write_file(self.gathered_requires_filename,
"\n".join(self.pips_to_install))
if not self.pips_to_install:
LOG.error("No dependencies for OpenStack found."
"Something went wrong. Please check:")
LOG.error("'%s'" % "' '".join(cmdline))
raise RuntimeError("No dependencies for OpenStack found")
utils.log_iterable(sorted(self.pips_to_install),
logger=LOG,
header="Full known python dependency list")
self.forced_packages = []
for line in self.pips_to_install:
req = pip_helper.extract_requirement(line)
if req.key in forced_keys:
self.forced_packages.append(req)
sh.write_file(self.forced_requires_filename,
"\n".join(str(req) for req in self.forced_packages))
def gather_pips_to_install(self, requires_files, extra_pips=None):
"""Analyze requires_files and extra_pips.
Updates `self.forced_packages` and `self.pips_to_install`.
Writes requirements to `self.gathered_requires_filename`.
"""
extra_pips = extra_pips or []
cmdline = [
self.multipip_executable, "--skip-requirements-regex",
"python.*client", "--pip", self.pip_executable
]
cmdline = cmdline + extra_pips + ["-r"] + requires_files
output = sh.execute(cmdline, check_exit_code=False)
conflict_descr = output[1].strip()
forced_keys = set()
if conflict_descr:
for line in conflict_descr.splitlines():
LOG.warning(line)
if line.endswith(": incompatible requirements"):
forced_keys.add(line.split(":", 1)[0].lower())
self.pips_to_install = [
pkg for pkg in utils.splitlines_not_empty(output[0])
if pkg.lower() not in OPENSTACK_PACKAGES
]
sh.write_file(self.gathered_requires_filename,
"\n".join(self.pips_to_install))
if not self.pips_to_install:
LOG.error("No dependencies for OpenStack found."
"Something went wrong. Please check:")
LOG.error("'%s'" % "' '".join(cmdline))
raise RuntimeError("No dependencies for OpenStack found")
utils.log_iterable(sorted(self.pips_to_install),
logger=LOG,
header="Full known python dependency list")
self.forced_packages = []
for pip in self.pips_to_install:
req = pkg_resources.Requirement.parse(pip)
if req.key in forced_keys:
self.forced_packages.append(req)
sh.write_file(self.forced_requires_filename,
"\n".join(str(req) for req in self.forced_packages))
def _gather_pips_to_install(self, requires_files, extra_pips=None):
"""Analyze requires_files and extra_pips.
Updates `self.forced_packages` and `self.pips_to_install`.
Writes requirements to `self.gathered_requires_filename`.
"""
extra_pips = extra_pips or []
cmdline = [
self.multipip_executable, "--skip-requirements-regex",
"python.*client", "--pip", self.pip_executable
]
cmdline = cmdline + extra_pips + ["-r"] + requires_files
cmdline.extend(["--ignore-package"])
cmdline.extend(OPENSTACK_PACKAGES)
cmdline.extend(SKIP_PACKAGE_NAMES)
cmdline.extend(self.python_names)
stdout, stderr = sh.execute(cmdline, check_exit_code=False)
self.pips_to_install = list(utils.splitlines_not_empty(stdout))
sh.write_file(self.gathered_requires_filename,
"\n".join(self.pips_to_install))
utils.log_iterable(sorted(self.pips_to_install),
logger=LOG,
header="Full known python dependency list")
incompatibles = collections.defaultdict(list)
if stderr:
current_name = ''
for line in stderr.strip().splitlines():
if line.endswith(": incompatible requirements"):
current_name = line.split(":", 1)[0].lower().strip()
if current_name not in incompatibles:
incompatibles[current_name] = []
else:
incompatibles[current_name].append(line)
for (name, lines) in incompatibles.items():
if not name:
continue
LOG.warn("Incompatible requirements found for %s",
colorizer.quote(name, quote_color='red'))
for line in lines:
LOG.warn(line)
if not self.pips_to_install:
LOG.error("No dependencies for OpenStack found."
"Something went wrong. Please check:")
LOG.error("'%s'" % "' '".join(cmdline))
raise exc.DependencyException(
"No dependencies for OpenStack found")
# Translate those that we altered requirements for into a set of forced
# requirements file (and associated list).
self.forced_packages = []
for req in [
pip_helper.extract_requirement(line)
for line in self.pips_to_install
]:
if req.key in incompatibles:
self.forced_packages.append(req)
sh.write_file(self.forced_requires_filename,
"\n".join([str(req) for req in self.forced_packages]))