def __init__(self, distro, root_dir, instances, opts=None):
self.distro = distro
self.root_dir = root_dir
self.instances = instances
self.opts = opts or {}
# Various paths we will use while operating
self.deps_dir = sh.joinpths(self.root_dir, "deps")
self.downloaded_flag_file = sh.joinpths(self.deps_dir, "pip-downloaded")
self.download_dir = sh.joinpths(self.deps_dir, "download")
self.log_dir = sh.joinpths(self.deps_dir, "output")
self.gathered_requires_filename = sh.joinpths(self.deps_dir, "pip-requires")
self.forced_requires_filename = sh.joinpths(self.deps_dir, "forced-requires")
self.download_requires_filename = sh.joinpths(self.deps_dir, "download-requires")
# Executables we require to operate
self.multipip_executable = sh.which("multipip", ["tools/"])
self.pip_executable = sh.which_first(['pip-python', 'pip'])
self.pipdownload_executable = sh.which("pip-download", ["tools/"])
# List of requirements
self.pips_to_install = []
self.forced_packages = []
# Instances to there app directory (with a setup.py inside)
self.package_dirs = self._get_package_dirs(instances)
# Instantiate this as late as we can.
self._python_names = None
# Track what file we create so they can be cleaned up on uninstall.
trace_fn = tr.trace_filename(self.root_dir, 'deps')
self.tracewriter = tr.TraceWriter(trace_fn, break_if_there=False)
self.tracereader = tr.TraceReader(trace_fn)
self.requirements = {}
for key in ("build-requires", "requires", "conflicts"):
req_set = set()
for inst in self.instances:
req_set |= set(pkg["name"]
for pkg in inst.get_option(key) or [])
self.requirements[key] = req_set
def __init__(self, distro, root_dir, instances, opts=None):
super(YumDependencyHandler, self).__init__(distro, root_dir, instances,
opts)
# Various paths we will use while operating
self.rpmbuild_dir = sh.joinpths(self.deps_dir, "rpmbuild")
self.deps_repo_dir = sh.joinpths(self.deps_dir, "openstack-deps")
self.deps_src_repo_dir = sh.joinpths(self.deps_dir,
"openstack-deps-sources")
self.rpm_sources_dir = sh.joinpths(self.rpmbuild_dir, "SOURCES")
self.anvil_repo_dir = sh.joinpths(self.root_dir, "repo")
# Executables we require to operate
self.py2rpm_executable = sh.which("py2rpm", ["tools/"])
self.rpmbuild_executable = sh.which("rpmbuild")
self.specprint_executable = sh.which('specprint', ["tools/"])
self.yumfind_executable = sh.which("yumfind", ["tools/"])
# We inspect yum for packages, this helper allows us to do this.
self.helper = yum_helper.Helper(self.log_dir)
# See if we are requested to run at a higher make parallelism level
self._jobs = self.JOBS
if 'jobs' in self.opts:
try:
self._jobs = int(self.opts.get('jobs', self.JOBS))
if self._jobs <= 0:
self._jobs = self.JOBS
except (TypeError, ValueError):
pass
def __init__(self, distro, root_dir,
instances, opts, group, prior_groups):
super(YumDependencyHandler, self).__init__(distro, root_dir,
instances, opts, group,
prior_groups)
# Various paths we will use while operating
self.rpmbuild_dir = sh.joinpths(self.deps_dir, "rpmbuild")
self.prebuild_dir = sh.joinpths(self.deps_dir, "prebuild")
self.deps_repo_dir = sh.joinpths(self.deps_dir, "openstack-deps")
self.deps_src_repo_dir = sh.joinpths(self.deps_dir, "openstack-deps-sources")
self.rpm_sources_dir = sh.joinpths(self.rpmbuild_dir, "SOURCES")
self.anvil_repo_dir = sh.joinpths(self.root_dir, "repo")
self.generated_srpms_filename = sh.joinpths(self.deps_dir, "generated-srpms-%s" % group)
self.build_requires_filename = sh.joinpths(self.deps_dir, "build-requires-%s" % group)
self.yum_satisfies_filename = sh.joinpths(self.deps_dir, "yum-satisfiable-%s" % group)
self.rpm_build_requires_filename = sh.joinpths(self.deps_dir, "rpm-build-requires-%s" % group)
# Executables we require to operate
self.rpmbuild_executable = sh.which("rpmbuild")
self.specprint_executable = sh.which('specprint', ["tools/"])
# We inspect yum for packages, this helper allows us to do this.
self.helper = yum_helper.Helper(self.log_dir, self.REPOS)
self.envra_helper = envra_helper.Helper()
# See if we are requested to run at a higher make parallelism level
try:
self.jobs = max(self.JOBS, int(self.opts.get('jobs')))
except (TypeError, ValueError):
self.jobs = self.JOBS
def __init__(self, log_dir):
# Executables we require to operate
self.yyoom_executable = sh.which("yyoom", ["tools/"])
# Executable logs will go into this directory
self._log_dir = log_dir
# Caches of installed and available packages
self._installed = None
self._available = None
def __init__(self, log_dir):
# Executables we require to operate
self.yyoom_executable = sh.which("yyoom", ["tools/"])
# Executable logs will go into this directory
self._log_dir = log_dir
# Caches of installed and available packages
self._installed = None
self._available = None
def __init__(self, log_dir, repos):
# Executables we require to operate
self.yyoom_executable = sh.which("yyoom", ["tools/"])
# Preferred repositories names
self._repos = repos
# Caches of installed and available packages
self._installed = None
self._available = None
self._logs_dir = log_dir
def __init__(self, log_dir, repos):
# Executables we require to operate
self.yyoom_executable = sh.which("yyoom", ["tools/"])
# Preferred repositories names
self._repos = repos
# Caches of installed and available packages
self._installed = None
self._available = None
self._logs_dir = log_dir
def __init__(self, epoch_map, package_map, arch_dependent, rpmbuild_dir, download_dir, deps_dir, log_dir):
self._py2rpm_executable = sh.which("py2rpm", ["tools/"])
self._epoch_map = epoch_map
self._package_map = package_map
self._arch_dependent = arch_dependent
# Various paths that are used during operating
self._rpmbuild_dir = rpmbuild_dir
self._download_dir = download_dir
self._deps_dir = deps_dir
self._log_dir = log_dir
def _yyoom(arglist):
executable = sh.which("yyoom", ["tools/"])
cmdline = [executable]
if LOG.logger.isEnabledFor(logging.DEBUG):
cmdline.append('--verbose')
cmdline.extend(arglist)
out = sh.execute(cmdline, stderr_fh=sys.stderr)[0].strip()
if out:
return json.loads(out)
return None
def __init__(self, epoch_map, package_map, arch_dependent, rpmbuild_dir,
download_dir, deps_dir, log_dir):
self._py2rpm_executable = sh.which("py2rpm", ["tools/"])
self._epoch_map = epoch_map
self._package_map = package_map
self._arch_dependent = arch_dependent
# Various paths that are used during operating
self._rpmbuild_dir = rpmbuild_dir
self._download_dir = download_dir
self._deps_dir = deps_dir
self._log_dir = log_dir
def __init__(self, distro, root_dir, instances, opts=None):
self.distro = distro
self.root_dir = root_dir
self.instances = instances
self.opts = opts or {}
# Various paths we will use while operating
self.deps_dir = sh.joinpths(self.root_dir, "deps")
self.downloaded_flag_file = sh.joinpths(self.deps_dir,
"pip-downloaded")
self.download_dir = sh.joinpths(self.deps_dir, "download")
self.log_dir = sh.joinpths(self.deps_dir, "output")
self.gathered_requires_filename = sh.joinpths(self.deps_dir,
"pip-requires")
self.forced_requires_filename = sh.joinpths(self.deps_dir,
"forced-requires")
self.download_requires_filename = sh.joinpths(self.deps_dir,
"download-requires")
# Executables we require to operate
self.multipip_executable = sh.which("multipip", ["tools/"])
self.pip_executable = sh.which_first(['pip-python', 'pip'])
self.pipdownload_executable = sh.which("pip-download", ["tools/"])
# List of requirements
self.pips_to_install = []
self.forced_packages = []
# Instances to there app directory (with a setup.py inside)
self.package_dirs = self._get_package_dirs(instances)
# Instantiate this as late as we can.
self._python_names = None
# Track what file we create so they can be cleaned up on uninstall.
trace_fn = tr.trace_filename(self.root_dir, 'deps')
self.tracewriter = tr.TraceWriter(trace_fn, break_if_there=False)
self.tracereader = tr.TraceReader(trace_fn)
self.requirements = {}
for key in ("build-requires", "requires", "conflicts"):
req_set = set()
for inst in self.instances:
req_set |= set(pkg["name"]
for pkg in inst.get_option(key) or [])
self.requirements[key] = req_set
def __init__(self, distro, root_dir, instances, opts=None):
super(YumDependencyHandler, self).__init__(distro, root_dir, instances, opts)
# Various paths we will use while operating
self.rpmbuild_dir = sh.joinpths(self.deps_dir, "rpmbuild")
self.deps_repo_dir = sh.joinpths(self.deps_dir, "openstack-deps")
self.deps_src_repo_dir = sh.joinpths(self.deps_dir, "openstack-deps-sources")
self.rpm_sources_dir = sh.joinpths(self.rpmbuild_dir, "SOURCES")
self.anvil_repo_dir = sh.joinpths(self.root_dir, "repo")
# Executables we require to operate
self.rpmbuild_executable = sh.which("rpmbuild")
self.specprint_executable = sh.which('specprint', ["tools/"])
# We inspect yum for packages, this helper allows us to do this.
self.helper = yum_helper.Helper(self.log_dir, self.REPOS)
# See if we are requested to run at a higher make parallelism level
self._jobs = self.JOBS
if 'jobs' in self.opts:
try:
self._jobs = int(self.opts.get('jobs', self.JOBS))
if self._jobs <= 0:
self._jobs = self.JOBS
except (TypeError, ValueError):
pass
def __init__(self, distro, root_dir, instances, opts):
super(YumDependencyHandler, self).__init__(distro, root_dir, instances,
opts)
# Various paths we will use while operating
self.rpmbuild_dir = sh.joinpths(self.deps_dir, "rpmbuild")
self.deps_repo_dir = sh.joinpths(self.deps_dir, "openstack-deps")
self.deps_src_repo_dir = sh.joinpths(self.deps_dir,
"openstack-deps-sources")
self.rpm_sources_dir = sh.joinpths(self.rpmbuild_dir, "SOURCES")
self.anvil_repo_dir = sh.joinpths(self.root_dir, "repo")
self.build_requires_filename = sh.joinpths(self.deps_dir,
"build-requires")
self.yum_satisfies_filename = sh.joinpths(self.deps_dir,
"yum-satisfiable")
# Executables we require to operate
self.rpmbuild_executable = sh.which("rpmbuild")
self.specprint_executable = sh.which('specprint', ["tools/"])
# We inspect yum for packages, this helper allows us to do this.
self.helper = yum_helper.Helper(self.log_dir, self.REPOS)
# See if we are requested to run at a higher make parallelism level
try:
self.jobs = max(self.JOBS, int(self.opts.get('jobs')))
except (TypeError, ValueError):
self.jobs = self.JOBS
def __init__(self):
self._executable = sh.which("explode_envra", ["tools/"])
def __init__(self):
self._executable = sh.which("explode_envra", ["tools/"])
class YumDependencyHandler(base.DependencyHandler):
OPENSTACK_DEPS_PACKAGE_NAME = "openstack-deps"
OPENSTACK_EPOCH = 2
py2rpm_executable = sh.which("py2rpm", ["tools/"])
REPO_FN = "anvil.repo"
YUM_REPO_DIR = "/etc/yum.repos.d/"
BANNED_PACKAGES = [
'distribute',
'setuptools',
]
def __init__(self, distro, root_dir, instances):
super(YumDependencyHandler, self).__init__(distro, root_dir, instances)
self.rpmbuild_dir = sh.joinpths(self.deps_dir, "rpmbuild")
self.deps_repo_dir = sh.joinpths(self.deps_dir, "openstack-deps")
self.deps_src_repo_dir = sh.joinpths(self.deps_dir,
"openstack-deps-sources")
self.anvil_repo_filename = sh.joinpths(self.deps_dir, self.REPO_FN)
# Track what file we create so they can be cleaned up on uninstall.
trace_fn = tr.trace_filename(root_dir, 'deps')
self.tracewriter = tr.TraceWriter(trace_fn, break_if_there=False)
self.tracereader = tr.TraceReader(trace_fn)
self.helper = yum_helper.Helper()
def py2rpm_start_cmdline(self):
cmdline = [
self.py2rpm_executable,
"--rpm-base",
self.rpmbuild_dir,
]
if self.python_names:
cmdline += [
"--epoch-map",
] + [
"%s==%s" % (name, self.OPENSTACK_EPOCH)
for name in self.python_names
]
package_map = self.distro._dependency_handler.get("package_map", {})
if package_map:
cmdline += [
"--package-map",
] + [
"%s==%s" % (key, value)
for key, value in package_map.iteritems()
]
arch_dependent = self.distro._dependency_handler.get(
"arch_dependent", [])
if arch_dependent:
cmdline += [
"--arch-dependent",
] + arch_dependent
return cmdline
def package(self):
super(YumDependencyHandler, self).package()
self._write_all_deps_package()
self._build_dependencies()
self._build_openstack()
self._create_deps_repo()
def filter_download_requires(self):
yum_map = {}
for pkg in yum_helper.Helper().get_available():
for provides in pkg.provides:
yum_map.setdefault(provides[0], set()).add(
(pkg.version, pkg.repo))
nopips = [
pkg_resources.Requirement.parse(name).key
for name in self.python_names
]
pips_to_download = []
req_to_install = [
pkg_resources.Requirement.parse(pkg)
for pkg in self.pips_to_install
]
req_to_install = [
req for req in req_to_install if req.key not in nopips
]
requested_names = [req.key for req in req_to_install]
rpm_to_install = self._convert_names_python2rpm(requested_names)
satisfied_list = []
for (req, rpm_name) in zip(req_to_install, rpm_to_install):
yum_versions = yum_map.get(rpm_name, [])
satisfied = False
for (version, repo) in yum_versions:
if version in req:
satisfied = True
satisfied_list.append((req, rpm_name, version, repo))
break
if not satisfied:
pips_to_download.append(str(req))
if satisfied_list:
# Organize by repo
repos = collections.defaultdict(list)
for (req, rpm_name, version, repo) in satisfied_list:
repos[repo].append("%s as %s-%s" % (req, rpm_name, version))
for r in sorted(repos.keys()):
header = ("%s Python packages are already available "
"as RPMs from repository %s")
header = header % (len(repos[r]), colorizer.quote(r))
utils.log_iterable(sorted(repos[r]), logger=LOG, header=header)
return pips_to_download
@staticmethod
def _get_component_name(pkg_dir):
return sh.basename(sh.dirname(pkg_dir))
def _write_all_deps_package(self):
spec_filename = sh.joinpths(
self.rpmbuild_dir, "SPECS",
"%s.spec" % self.OPENSTACK_DEPS_PACKAGE_NAME)
# Clean out previous dirs.
for dirname in (self.rpmbuild_dir, self.deps_repo_dir,
self.deps_src_repo_dir):
sh.deldir(dirname)
sh.mkdirslist(dirname, tracewriter=self.tracewriter)
def get_version_release():
right_now = datetime.now()
components = [
str(right_now.year),
str(right_now.month),
str(right_now.day),
]
return (".".join(components), right_now.strftime("%s"))
(version, release) = get_version_release()
spec_content = """Name: %s
Version: %s
Release: %s
License: Apache 2.0
Summary: OpenStack dependencies
BuildArch: noarch
""" % (self.OPENSTACK_DEPS_PACKAGE_NAME, version, release)
packages = {}
for inst in self.instances:
try:
for pack in inst.packages:
packages[pack["name"]] = pack
except AttributeError:
pass
scripts = {}
script_map = {
"pre-install": "%pre",
"post-install": "%post",
"pre-uninstall": "%preun",
"post-uninstall": "%postun",
}
for pack_name in sorted(packages.iterkeys()):
pack = packages[pack_name]
cont = [spec_content, "Requires: ", pack["name"]]
version = pack.get("version")
if version:
cont.append(" ")
cont.append(version)
cont.append("\n")
spec_content = "".join(cont)
for script_name in script_map.iterkeys():
try:
script_list = pack[script_name]
except (KeyError, ValueError):
continue
script_body = scripts.get(script_name, "")
script_body = "%s\n# %s\n" % (script_body, pack_name)
for script in script_list:
try:
line = " ".join(
sh.shellquote(word) for word in script["cmd"])
except (KeyError, ValueError):
continue
if script.get("ignore_failure"):
ignore = " 2>/dev/null || true"
else:
ignore = ""
script_body = "".join((script_body, line, ignore, "\n"))
scripts[script_name] = script_body
spec_content += "\n%description\n\n"
for script_name in sorted(script_map.iterkeys()):
try:
script_body = scripts[script_name]
except KeyError:
pass
else:
spec_content = "%s\n%s\n%s\n" % (
spec_content, script_map[script_name], script_body)
spec_content += "\n%files\n"
sh.write_file(spec_filename,
spec_content,
tracewriter=self.tracewriter)
cmdline = [
"rpmbuild",
"-ba",
"--define",
"_topdir %s" % self.rpmbuild_dir,
spec_filename,
]
LOG.info("Building %s RPM" % self.OPENSTACK_DEPS_PACKAGE_NAME)
sh.execute(cmdline)
def _build_dependencies(self):
package_files = self.download_dependencies()
def filter_files(package_files):
for p in package_files:
banned = False
for k in self.BANNED_PACKAGES:
if k in p.lower():
banned = True
if banned:
continue
yield p
package_files = [f for f in filter_files(package_files)]
if not package_files:
LOG.info("No RPM packages of OpenStack dependencies to build")
return
package_base_names = [sh.basename(f) for f in package_files]
utils.log_iterable(sorted(package_base_names),
logger=LOG,
header=("Building %s dependency RPM"
" packages") % (len(package_files)))
with utils.progress_bar(name='Building',
max_am=len(package_files)) as p_bar:
for (i, filename) in enumerate(sorted(package_files)):
cmdline = self.py2rpm_start_cmdline() + ["--", filename]
build_filename = "py2rpm-%s.out" % sh.basename(filename)
out_filename = sh.joinpths(self.log_dir, build_filename)
sh.execute_save_output(cmdline,
out_filename=out_filename,
quiet=True)
p_bar.update(i + 1)
def _build_openstack(self):
if not self.package_dirs:
LOG.warn("No RPM packages of OpenStack installs to build")
return
component_names = [
self._get_component_name(d) for d in self.package_dirs
]
utils.log_iterable(sorted(component_names),
logger=LOG,
header=("Building %s OpenStack RPM"
" packages") % (len(self.package_dirs)))
with utils.progress_bar(name='Building',
max_am=len(self.package_dirs)) as p_bar:
for (i, pkg_dir) in enumerate(sorted(self.package_dirs)):
component_name = self._get_component_name(pkg_dir)
cmdline = self.py2rpm_start_cmdline() + ["--", pkg_dir]
out_filename = sh.joinpths(self.log_dir,
"py2rpm.%s.out" % (component_name))
sh.execute_save_output(cmdline,
out_filename=out_filename,
quiet=True)
p_bar.update(i + 1)
def _create_deps_repo(self):
for filename in sh.listdir(sh.joinpths(self.rpmbuild_dir, "RPMS"),
recursive=True,
files_only=True):
sh.move(filename, self.deps_repo_dir, force=True)
for filename in sh.listdir(sh.joinpths(self.rpmbuild_dir, "SRPMS"),
recursive=True,
files_only=True):
sh.move(filename, self.deps_src_repo_dir, force=True)
for repo_dir in self.deps_repo_dir, self.deps_src_repo_dir:
cmdline = ["createrepo", repo_dir]
LOG.info("Creating repo at %s" % repo_dir)
sh.execute(cmdline)
LOG.info("Writing %s to %s", self.REPO_FN, self.anvil_repo_filename)
(_fn, content) = utils.load_template('packaging', self.REPO_FN)
params = {
"baseurl_bin": "file://%s" % self.deps_repo_dir,
"baseurl_src": "file://%s" % self.deps_src_repo_dir
}
sh.write_file(self.anvil_repo_filename,
utils.expand_template(content, params),
tracewriter=self.tracewriter)
def _convert_names_python2rpm(self, python_names):
if not self.python_names:
return []
cmdline = self.py2rpm_start_cmdline() + ["--convert"] + python_names
rpm_names = []
for name in sh.execute(cmdline)[0].splitlines():
# name is "Requires: rpm-name"
try:
rpm_names.append(name.split(":")[1].strip())
except IndexError:
pass
return rpm_names
def install(self):
super(YumDependencyHandler, self).install()
repo_filename = sh.joinpths(self.YUM_REPO_DIR, self.REPO_FN)
# Ensure we copy the local repo file name to the main repo so that
# yum will find it when installing packages.
sh.write_file(repo_filename,
sh.load_file(self.anvil_repo_filename),
tracewriter=self.tracewriter)
# Erase it if its been previously installed.
cmdline = []
if self.helper.is_installed(self.OPENSTACK_DEPS_PACKAGE_NAME):
cmdline.append(self.OPENSTACK_DEPS_PACKAGE_NAME)
for p in self.nopackages:
if self.helper.is_installed(p):
cmdline.append(p)
if cmdline:
cmdline = ["yum", "erase", "-y"] + cmdline
sh.execute(cmdline, stdout_fh=sys.stdout, stderr_fh=sys.stderr)
cmdline = ["yum", "clean", "all"]
sh.execute(cmdline)
cmdline = ["yum", "install", "-y", self.OPENSTACK_DEPS_PACKAGE_NAME]
sh.execute(cmdline, stdout_fh=sys.stdout, stderr_fh=sys.stderr)
rpm_names = self._convert_names_python2rpm(self.python_names)
if rpm_names:
cmdline = ["yum", "install", "-y"] + rpm_names
sh.execute(cmdline, stdout_fh=sys.stdout, stderr_fh=sys.stderr)
def uninstall(self):
super(YumDependencyHandler, self).uninstall()
if self.tracereader.exists():
for f in self.tracereader.files_touched():
sh.unlink(f)
for d in self.tracereader.dirs_made():
sh.deldir(d)
sh.unlink(self.tracereader.filename())
self.tracereader = None
rpm_names = []
for name in self._convert_names_python2rpm(self.python_names):
if self.helper.is_installed(name):
rpm_names.append(name)
if rpm_names:
cmdline = ["yum", "remove", "--remove-leaves", "-y"] + rpm_names
sh.execute(cmdline, stdout_fh=sys.stdout, stderr_fh=sys.stderr)
def __init__(self):
self._multipip_executable = sh.which("multipip", ["tools/"])
def __init__(self):
self._multipip_executable = sh.which("multipip", ["tools/"])
def setUp(self):
super(TestTools, self).setUp()
self.multipip = [sys.executable, sh.which("multipip", ['tools'])]
class DependencyHandler(object):
"""Basic class for handler of OpenStack dependencies.
"""
MAX_PIP_DOWNLOAD_ATTEMPTS = 4
multipip_executable = sh.which("multipip", ["tools/"])
def __init__(self, distro, root_dir, instances):
self.distro = distro
self.root_dir = root_dir
self.instances = instances
self.deps_dir = sh.joinpths(self.root_dir, "deps")
self.download_dir = sh.joinpths(self.deps_dir, "download")
self.log_dir = sh.joinpths(self.deps_dir, "output")
self.gathered_requires_filename = sh.joinpths(self.deps_dir,
"pip-requires")
self.forced_requires_filename = sh.joinpths(self.deps_dir,
"forced-requires")
self.pip_executable = str(self.distro.get_command_config('pip'))
self.pips_to_install = []
self.forced_packages = []
# these packages conflict with our deps and must be removed
self.nopackages = []
self.package_dirs = self._get_package_dirs(instances)
self.python_names = self._get_python_names(self.package_dirs)
@staticmethod
def _get_package_dirs(instances):
package_dirs = []
for inst in instances:
app_dir = inst.get_option("app_dir")
if sh.isfile(sh.joinpths(app_dir, "setup.py")):
package_dirs.append(app_dir)
return package_dirs
@staticmethod
def _get_python_names(package_dirs):
python_names = []
for pkg_dir in package_dirs:
cmdline = ["python", "setup.py", "--name"]
python_names.append(
sh.execute(cmdline, cwd=pkg_dir)[0].splitlines()[-1].strip())
return python_names
def package(self):
requires_files = []
extra_pips = []
for inst in self.instances:
try:
requires_files.extend(inst.requires_files)
except AttributeError:
pass
for pkg in inst.get_option("pips") or []:
extra_pips.append("%s%s" %
(pkg["name"], pkg.get("version", "")))
requires_files = filter(sh.isfile, requires_files)
self.gather_pips_to_install(requires_files, extra_pips)
self.clean_pip_requires(requires_files)
def install(self):
self.nopackages = []
for inst in self.instances:
for pkg in inst.get_option("nopackages") or []:
self.nopackages.append(pkg["name"])
def uninstall(self):
pass
def clean_pip_requires(self, requires_files):
# Fixup incompatible dependencies
if not (requires_files and self.forced_packages):
return
utils.log_iterable(sorted(requires_files),
logger=LOG,
header="Adjusting %s pip 'requires' files" %
(len(requires_files)))
forced_by_key = dict((pkg.key, pkg) for pkg in self.forced_packages)
for fn in requires_files:
old_lines = sh.load_file(fn).splitlines()
new_lines = []
for line in old_lines:
try:
req = pkg_resources.Requirement.parse(line)
new_lines.append(str(forced_by_key[req.key]))
except:
# we don't force the package or it has a bad format
new_lines.append(line)
contents = "# Cleaned on %s\n\n%s\n" % (utils.iso8601(),
"\n".join(new_lines))
sh.write_file_and_backup(fn, contents)
def gather_pips_to_install(self, requires_files, extra_pips=None):
"""Analyze requires_files and extra_pips.
Updates `self.forced_packages` and `self.pips_to_install`.
Writes requirements to `self.gathered_requires_filename`.
"""
extra_pips = extra_pips or []
cmdline = [
self.multipip_executable, "--skip-requirements-regex",
"python.*client", "--pip", self.pip_executable
]
cmdline = cmdline + extra_pips + ["-r"] + requires_files
output = sh.execute(cmdline, check_exit_code=False)
conflict_descr = output[1].strip()
forced_keys = set()
if conflict_descr:
for line in conflict_descr.splitlines():
LOG.warning(line)
if line.endswith(": incompatible requirements"):
forced_keys.add(line.split(":", 1)[0].lower())
self.pips_to_install = [
pkg for pkg in utils.splitlines_not_empty(output[0])
if pkg.lower() not in OPENSTACK_PACKAGES
]
sh.write_file(self.gathered_requires_filename,
"\n".join(self.pips_to_install))
if not self.pips_to_install:
LOG.error("No dependencies for OpenStack found."
"Something went wrong. Please check:")
LOG.error("'%s'" % "' '".join(cmdline))
raise RuntimeError("No dependencies for OpenStack found")
utils.log_iterable(sorted(self.pips_to_install),
logger=LOG,
header="Full known python dependency list")
self.forced_packages = []
for pip in self.pips_to_install:
req = pkg_resources.Requirement.parse(pip)
if req.key in forced_keys:
self.forced_packages.append(req)
sh.write_file(self.forced_requires_filename,
"\n".join(str(req) for req in self.forced_packages))
def filter_download_requires(self):
if not self.python_names:
return self.pips_to_install
cmdline = [
self.multipip_executable,
"--pip",
self.pip_executable,
] + self.pips_to_install + [
"--ignore-packages",
] + self.python_names
output = sh.execute(cmdline)
pips_to_download = list(utils.splitlines_not_empty(output[0]))
return pips_to_download
def _try_download_dependencies(self, attempt, pips_to_download,
pip_download_dir, pip_cache_dir,
pip_build_dir):
pips_to_download = [str(p) for p in pips_to_download]
cmdline = [
self.pip_executable,
"install",
"--download",
pip_download_dir,
"--download-cache",
pip_cache_dir,
"--build",
pip_build_dir,
]
cmdline.extend(sorted(pips_to_download))
download_filename = "pip-download-attempt-%s.out"
download_filename = download_filename % (attempt)
out_filename = sh.joinpths(self.log_dir, download_filename)
sh.execute_save_output(cmdline, out_filename=out_filename)
def download_dependencies(self, clear_cache=False):
"""Download dependencies from `$deps_dir/download-requires`.
:param clear_cache: clear `$deps_dir/cache` dir (pip can work incorrectly
when it has a cache)
"""
sh.deldir(self.download_dir)
sh.mkdir(self.download_dir, recurse=True)
download_requires_filename = sh.joinpths(self.deps_dir,
"download-requires")
raw_pips_to_download = self.filter_download_requires()
pips_to_download = [
pkg_resources.Requirement.parse(str(p.strip()))
for p in raw_pips_to_download if p.strip()
]
sh.write_file(download_requires_filename,
"\n".join(str(req) for req in pips_to_download))
if not pips_to_download:
return []
pip_dir = sh.joinpths(self.deps_dir, "pip")
pip_download_dir = sh.joinpths(pip_dir, "download")
pip_build_dir = sh.joinpths(pip_dir, "build")
pip_cache_dir = sh.joinpths(pip_dir, "cache")
if clear_cache:
sh.deldir(pip_cache_dir)
pip_failures = []
how_many = len(pips_to_download)
for attempt in xrange(self.MAX_PIP_DOWNLOAD_ATTEMPTS):
# NOTE(aababilov): pip has issues with already downloaded files
sh.deldir(pip_download_dir)
sh.mkdir(pip_download_dir, recurse=True)
sh.deldir(pip_build_dir)
utils.log_iterable(sorted(raw_pips_to_download),
logger=LOG,
header=("Downloading %s python dependencies "
"(attempt %s)" % (how_many, attempt)))
failed = False
try:
self._try_download_dependencies(attempt, pips_to_download,
pip_download_dir,
pip_cache_dir, pip_build_dir)
pip_failures = []
except exc.ProcessExecutionError as e:
LOG.exception("Failed downloading python dependencies")
pip_failures.append(e)
failed = True
if not failed:
break
if pip_failures:
raise pip_failures[-1]
for filename in sh.listdir(pip_download_dir, files_only=True):
sh.move(filename, self.download_dir)
return sh.listdir(self.download_dir, files_only=True)