def _try_download(self, pips_to_download, attempt=0):
def on_download_finish(time_taken):
LOG.info("Took %0.2f seconds to download...", time_taken)
LOG.info("Downloading %s dependencies with pip (attempt %s)...",
len(pips_to_download), attempt)
output_filename = sh.joinpths(
self.log_dir, "pip-download-attempt-%s.log" % (attempt))
LOG.info("Please wait this may take a while...")
LOG.info("Check %s for download activity details...",
colorizer.quote(output_filename))
utils.time_it(on_download_finish, pip_helper.download_dependencies,
self.download_dir, pips_to_download, output_filename)
def try_download(attempt):
LOG.info("Downloading %s dependencies with pip (attempt %s)...",
len(pips_to_download), attempt)
output_filename = sh.joinpths(self.log_dir,
"pip-download-attempt-%s.log" % (attempt))
LOG.info("Please wait this may take a while...")
LOG.info("Check %s for download activity details...",
colorizer.quote(output_filename))
utils.time_it(on_download_finish,
pip_helper.download_dependencies,
self.download_dir,
pips_to_download,
output_filename)
def package_finish(self):
super(VenvDependencyHandler, self).package_finish()
for instance in self.instances:
if not self._is_buildable(instance):
continue
venv_dir = sh.abspth(self._venv_directory_for(instance))
release = str(instance.get_option("release", default_value=1))
if release and not release.startswith('-'):
release = '-' + release
version_full = instance.egg_info['version'] + release
# Replace paths with virtualenv deployment directory.
if self.opts.get('venv_deploy_dir'):
deploy_dir = sh.joinpths(self.opts.get('venv_deploy_dir'),
instance.name)
replacer = functools.partial(
re.subn, re.escape(instance.get_option('component_dir')),
deploy_dir)
bin_dir = sh.joinpths(venv_dir, 'bin')
adjustments, files_replaced = self._replace_deployment_paths(
bin_dir, replacer)
if files_replaced:
LOG.info("Adjusted %s deployment path(s) in %s files",
adjustments, files_replaced)
tar_path = sh.joinpths(
self.opts.get('venv_deploy_dir'), '%s/%s-%s-venv/venv' %
(instance.name, instance.name, version_full))
else:
tar_path = '%s/%s-%s-venv/venv' % (instance.name,
instance.name, version_full)
# Create a tarball containing the virtualenv.
tar_filename = sh.joinpths(
venv_dir, '%s-%s-venv.tar.gz' % (instance.name, version_full))
LOG.info("Making tarball of %s built for %s with version %s at %s",
venv_dir, instance.name, version_full, tar_filename)
utils.time_it(functools.partial(_on_finish, "Tarball creation"),
self._make_tarball, venv_dir, tar_filename, tar_path)
def package_finish(self):
super(VenvDependencyHandler, self).package_finish()
for instance in self.instances:
if not self._is_buildable(instance):
continue
venv_dir = sh.abspth(self._venv_directory_for(instance))
release = str(instance.get_option("release", default_value=1))
if release and not release.startswith('-'):
release = '-' + release
version_full = instance.egg_info['version'] + release
# Replace paths with virtualenv deployment directory.
if self.opts.get('venv_deploy_dir'):
deploy_dir = sh.joinpths(self.opts.get('venv_deploy_dir'),
instance.name)
replacer = functools.partial(
re.subn, re.escape(instance.get_option('component_dir')),
deploy_dir)
bin_dir = sh.joinpths(venv_dir, 'bin')
adjustments, files_replaced = self._replace_deployment_paths(bin_dir,
replacer)
if files_replaced:
LOG.info("Adjusted %s deployment path(s) in %s files",
adjustments, files_replaced)
tar_path = sh.joinpths(self.opts.get('venv_deploy_dir'), '%s/%s-%s-venv/venv' % (
instance.name, instance.name, version_full))
else:
tar_path = '%s/%s-%s-venv/venv' % (instance.name, instance.name, version_full)
# Create a tarball containing the virtualenv.
tar_filename = sh.joinpths(venv_dir, '%s-%s-venv.tar.gz' % (instance.name,
version_full))
LOG.info("Making tarball of %s built for %s with version %s at %s", venv_dir,
instance.name, version_full, tar_filename)
utils.time_it(functools.partial(_on_finish, "Tarball creation"),
self._make_tarball, venv_dir, tar_filename, tar_path)
def _scan_pip_requires(self, requires_files):
own_eggs = self._python_eggs(False)
def replace_forced_requirements(fn, forced_by_key):
old_lines = sh.load_file(fn).splitlines()
new_lines = []
alterations = []
for line in old_lines:
try:
source_req = pip_helper.extract_requirement(line)
except (ValueError, TypeError):
pass
else:
if source_req:
validate_requirement(fn, source_req)
try:
replace_req = forced_by_key[source_req.key]
except KeyError:
pass
else:
replace_req = str(replace_req)
source_req = str(source_req)
if replace_req != source_req:
line = replace_req
alterations.append(
"%s => %s" %
(colorizer.quote(source_req),
colorizer.quote(replace_req)))
new_lines.append(line)
if alterations:
contents = "# Cleaned on %s\n\n%s\n" % (utils.iso8601(),
"\n".join(new_lines))
sh.write_file_and_backup(fn, contents)
utils.log_iterable(alterations,
logger=LOG,
header="Replaced %s requirements in %s" %
(len(alterations), fn),
color=None)
return len(alterations)
def on_replace_done(fn, time_taken):
LOG.debug(
"Replacing potential forced requirements in %s"
" took %s seconds", colorizer.quote(fn), time_taken)
def validate_requirement(filename, source_req):
install_egg = None
for egg_info in own_eggs:
if egg_info['name'] == source_req.key:
install_egg = egg_info
break
if not install_egg:
return
# Ensure what we are about to install/create will actually work
# with the desired version. If it is not compatible then we should
# abort and someone should update the tag/branch in the origin
# file (or fix it via some other mechanism).
if install_egg['version'] not in source_req:
msg = ("Can not satisfy '%s' with '%s', version"
" conflict found in %s")
raise exc.DependencyException(
msg % (source_req, install_egg['req'], filename))
if not requires_files:
return
requires_files = sorted(requires_files)
utils.log_iterable(requires_files,
logger=LOG,
header="Scanning %s pip 'requires' files" %
(len(requires_files)))
forced_by_key = {}
for pkg in self.forced_pips:
forced_by_key[pkg.key] = pkg
mutations = 0
for fn in requires_files:
LOG.debug("Replacing any potential forced requirements in %s",
colorizer.quote(fn))
mutations += utils.time_it(functools.partial(on_replace_done, fn),
replace_forced_requirements, fn,
forced_by_key)
# NOTE(imelnikov): after updating requirement lists we should re-fetch
# data from them again, so we drop pip helper caches here.
if mutations > 0:
pip_helper.drop_caches()
def _package_instance(self, instance, attempt=0):
if not self._is_buildable(instance):
# Skip things that aren't python...
LOG.warn("Skipping building %s (not python)",
colorizer.quote(instance.name, quote_color='red'))
return
def gather_extras():
extra_reqs = []
for p in instance.get_option("pips", default_value=[]):
req = pip_helper.create_requirement(p['name'], p.get('version'))
extra_reqs.append(req)
if instance.get_bool_option('use_tests_requires', default_value=True):
for p in instance.get_option("test_requires", default_value=[]):
extra_reqs.append(pip_helper.create_requirement(p))
return extra_reqs
all_requires_what = self._filter_download_requires()
LOG.info("Packaging %s (attempt %s)",
colorizer.quote(instance.name), attempt)
all_requires_mapping = {}
for req in all_requires_what:
if isinstance(req, six.string_types):
req = pip_helper.extract_requirement(req)
all_requires_mapping[req.key] = req
direct_requires_what = []
direct_requires_keys = set()
egg_info = getattr(instance, 'egg_info', None)
if egg_info is not None:
# Ensure we have gotten all the things...
test_dependencies = (egg_info.get('test_dependencies', [])
if instance.get_bool_option(
'use_tests_requires', default_value=True)
else [])
for req in itertools.chain(egg_info.get('dependencies', []),
test_dependencies):
if isinstance(req, six.string_types):
req = pip_helper.extract_requirement(req)
if req.key not in direct_requires_keys:
direct_requires_what.append(req)
direct_requires_keys.add(req.key)
requires_what = []
extra_requires_what = gather_extras()
for req in extra_requires_what:
if req.key in all_requires_mapping:
req = all_requires_mapping[req.key]
requires_what.append(req)
try:
direct_requires_keys.remove(req.key)
except KeyError:
pass
for req in direct_requires_what:
if req.key not in direct_requires_keys:
continue
if req.key in all_requires_mapping:
req = all_requires_mapping[req.key]
requires_what.append(req)
what = 'installation for %s' % colorizer.quote(instance.name)
utils.time_it(functools.partial(_on_finish, "Dependency %s" % what),
self._install_into_venv, instance,
requires_what)
extra_env_overrides = {
'PBR_VERSION': instance.egg_info['version'],
}
utils.time_it(functools.partial(_on_finish, "Instance %s" % what),
self._install_into_venv, instance,
[instance.get_option('app_dir')],
extra_env_overrides=extra_env_overrides)
def _package_instance(self, instance, attempt=0):
if not self._is_buildable(instance):
# Skip things that aren't python...
LOG.warn("Skipping building %s (not python)",
colorizer.quote(instance.name, quote_color='red'))
return
def gather_extras():
extra_reqs = []
for p in instance.get_option("pips", default_value=[]):
req = pip_helper.create_requirement(p['name'],
p.get('version'))
extra_reqs.append(req)
if instance.get_bool_option('use_tests_requires',
default_value=True):
for p in instance.get_option("test_requires",
default_value=[]):
extra_reqs.append(pip_helper.create_requirement(p))
return extra_reqs
all_requires_what = self._filter_download_requires()
LOG.info("Packaging %s (attempt %s)", colorizer.quote(instance.name),
attempt)
all_requires_mapping = {}
for req in all_requires_what:
if isinstance(req, six.string_types):
req = pip_helper.extract_requirement(req)
all_requires_mapping[req.key] = req
direct_requires_what = []
direct_requires_keys = set()
egg_info = getattr(instance, 'egg_info', None)
if egg_info is not None:
# Ensure we have gotten all the things...
test_dependencies = (egg_info.get(
'test_dependencies', []) if instance.get_bool_option(
'use_tests_requires', default_value=True) else [])
for req in itertools.chain(egg_info.get('dependencies', []),
test_dependencies):
if isinstance(req, six.string_types):
req = pip_helper.extract_requirement(req)
if req.key not in direct_requires_keys:
direct_requires_what.append(req)
direct_requires_keys.add(req.key)
requires_what = []
extra_requires_what = gather_extras()
for req in extra_requires_what:
if req.key in all_requires_mapping:
req = all_requires_mapping[req.key]
requires_what.append(req)
try:
direct_requires_keys.remove(req.key)
except KeyError:
pass
for req in direct_requires_what:
if req.key not in direct_requires_keys:
continue
if req.key in all_requires_mapping:
req = all_requires_mapping[req.key]
requires_what.append(req)
what = 'installation for %s' % colorizer.quote(instance.name)
utils.time_it(functools.partial(_on_finish, "Dependency %s" % what),
self._install_into_venv, instance, requires_what)
extra_env_overrides = {
'PBR_VERSION': instance.egg_info['version'],
}
utils.time_it(functools.partial(_on_finish, "Instance %s" % what),
self._install_into_venv,
instance, [instance.get_option('app_dir')],
extra_env_overrides=extra_env_overrides)
def _scan_pip_requires(self, requires_files):
own_eggs = self._python_eggs(False)
def replace_forced_requirements(fn, forced_by_key):
old_lines = sh.load_file(fn).splitlines()
new_lines = []
alterations = []
for line in old_lines:
try:
source_req = pip_helper.extract_requirement(line)
except (ValueError, TypeError):
pass
else:
if source_req:
validate_requirement(fn, source_req)
try:
replace_req = forced_by_key[source_req.key]
except KeyError:
pass
else:
replace_req = str(replace_req)
source_req = str(source_req)
if replace_req != source_req:
line = replace_req
alterations.append("%s => %s"
% (colorizer.quote(source_req),
colorizer.quote(replace_req)))
new_lines.append(line)
if alterations:
contents = "# Cleaned on %s\n\n%s\n" % (utils.iso8601(), "\n".join(new_lines))
sh.write_file_and_backup(fn, contents)
utils.log_iterable(alterations,
logger=LOG,
header="Replaced %s requirements in %s"
% (len(alterations), fn),
color=None)
return len(alterations)
def on_replace_done(fn, time_taken):
LOG.debug("Replacing potential forced requirements in %s"
" took %s seconds", colorizer.quote(fn), time_taken)
def validate_requirement(filename, source_req):
install_egg = None
for egg_info in own_eggs:
if egg_info['name'] == source_req.key:
install_egg = egg_info
break
if not install_egg:
return
# Ensure what we are about to install/create will actually work
# with the desired version. If it is not compatible then we should
# abort and someone should update the tag/branch in the origin
# file (or fix it via some other mechanism).
if install_egg['version'] not in source_req:
msg = ("Can not satisfy '%s' with '%s', version"
" conflict found in %s")
raise exc.DependencyException(msg % (source_req,
install_egg['req'],
filename))
if not requires_files:
return
requires_files = sorted(requires_files)
utils.log_iterable(requires_files,
logger=LOG,
header="Scanning %s pip 'requires' files" % (len(requires_files)))
forced_by_key = {}
for pkg in self.forced_pips:
forced_by_key[pkg.key] = pkg
mutations = 0
for fn in requires_files:
LOG.debug("Replacing any potential forced requirements in %s",
colorizer.quote(fn))
mutations += utils.time_it(functools.partial(on_replace_done, fn),
replace_forced_requirements,
fn, forced_by_key)
# NOTE(imelnikov): after updating requirement lists we should re-fetch
# data from them again, so we drop pip helper caches here.
if mutations > 0:
pip_helper.drop_caches()
