def download_dependencies(self):
"""Download dependencies from `$deps_dir/download-requires`."""
# NOTE(aababilov): do not drop download_dir - it can be reused
sh.mkdirslist(self.download_dir, tracewriter=self.tracewriter)
pips_to_download = self._filter_download_requires()
sh.write_file(self.download_requires_filename,
"\n".join([str(req) for req in pips_to_download]))
if not pips_to_download:
return ([], [])
# NOTE(aababilov): user could have changed persona, so,
# check that all requirements are downloaded
if (sh.isfile(self.downloaded_flag_file) and
self._requirements_satisfied(pips_to_download, self.download_dir)):
LOG.info("All python dependencies have been already downloaded")
else:
def try_download(attempt):
LOG.info("Downloading %s dependencies with pip (attempt %s)...",
len(pips_to_download), attempt)
output_filename = sh.joinpths(self.log_dir,
"pip-download-attempt-%s.log" % (attempt))
pip_helper.download_dependencies(self.download_dir,
pips_to_download,
output_filename)
utils.retry(self.MAX_PIP_DOWNLOAD_ATTEMPTS,
self.PIP_DOWNLOAD_DELAY, try_download)
# NOTE(harlowja): Mark that we completed downloading successfully
sh.touch_file(self.downloaded_flag_file, die_if_there=False,
quiet=True, tracewriter=self.tracewriter)
pips_downloaded = [pip_helper.extract_requirement(p) for p in pips_to_download]
what_downloaded = self._examine_download_dir(pips_downloaded, self.download_dir)
return (pips_downloaded, what_downloaded)
def _install_into_venv(self, instance, requirements):
venv_dir = self._venv_directory_for(instance)
base_pip = [sh.joinpths(venv_dir, 'bin', 'pip')]
env_overrides = {
'PATH': os.pathsep.join([sh.joinpths(venv_dir, "bin"),
env.get_key('PATH', default_value='')]),
'VIRTUAL_ENV': venv_dir,
}
sh.mkdirslist(self.cache_dir, tracewriter=self.tracewriter)
def try_install(attempt, requirements):
cmd = list(base_pip) + ['install']
cmd.extend([
'--download-cache',
self.cache_dir,
])
if isinstance(requirements, six.string_types):
cmd.extend([
'--requirement',
requirements
])
else:
for req in requirements:
cmd.append(str(req))
sh.execute(cmd, env_overrides=env_overrides)
# Sometimes pip fails downloading things, retry it when this happens...
utils.retry(3, 5, try_install, requirements=requirements)
def download_dependencies(self):
"""Download dependencies from `$deps_dir/download-requires`."""
# NOTE(aababilov): do not drop download_dir - it can be reused
sh.mkdirslist(self.download_dir, tracewriter=self.tracewriter)
pips_to_download = self._filter_download_requires()
sh.write_file(self.download_requires_filename,
"\n".join([str(req) for req in pips_to_download]))
if not pips_to_download:
return ([], [])
# NOTE(aababilov): user could have changed persona, so,
# check that all requirements are downloaded....
if self._requirements_satisfied(pips_to_download, self.download_dir):
LOG.info("All python dependencies have been already downloaded")
else:
def on_download_finish(time_taken):
LOG.info("Took %0.2f seconds to download...", time_taken)
def try_download(attempt):
LOG.info("Downloading %s dependencies with pip (attempt %s)...",
len(pips_to_download), attempt)
output_filename = sh.joinpths(self.log_dir,
"pip-download-attempt-%s.log" % (attempt))
LOG.info("Please wait this may take a while...")
LOG.info("Check %s for download activity details...",
colorizer.quote(output_filename))
utils.time_it(on_download_finish,
pip_helper.download_dependencies,
self.download_dir,
pips_to_download,
output_filename)
utils.retry(self.MAX_PIP_DOWNLOAD_ATTEMPTS,
self.PIP_DOWNLOAD_DELAY, try_download)
pips_downloaded = [pip_helper.extract_requirement(p) for p in pips_to_download]
what_downloaded = self._examine_download_dir(pips_downloaded, self.download_dir)
return (pips_downloaded, what_downloaded)
def download_dependencies(self):
"""Download dependencies from `$deps_dir/download-requires`."""
# NOTE(aababilov): do not drop download_dir - it can be reused
sh.mkdirslist(self.download_dir, tracewriter=self.tracewriter)
pips_to_download = self._filter_download_requires()
sh.write_file(self.download_requires_filename,
"\n".join([str(req) for req in pips_to_download]))
if not pips_to_download:
return ([], [])
# NOTE(aababilov): user could have changed persona, so,
# check that all requirements are downloaded....
if self._requirements_satisfied(pips_to_download, self.download_dir):
LOG.info("All python dependencies have been already downloaded")
else:
utils.retry(self.retries, self.retry_delay,
self._try_download, pips_to_download)
pips_downloaded = [pip_helper.extract_requirement(p) for p in pips_to_download]
what_downloaded = self._examine_download_dir(pips_downloaded, self.download_dir)
return (pips_downloaded, what_downloaded)
def download_dependencies(self):
"""Download dependencies from `$deps_dir/download-requires`."""
# NOTE(aababilov): do not drop download_dir - it can be reused
sh.mkdirslist(self.download_dir, tracewriter=self.tracewriter)
pips_to_download = self._filter_download_requires()
sh.write_file(self.download_requires_filename,
"\n".join([str(req) for req in pips_to_download]))
if not pips_to_download:
return ([], [])
# NOTE(aababilov): user could have changed persona, so,
# check that all requirements are downloaded
if (sh.isfile(self.downloaded_flag_file)
and self._requirements_satisfied(pips_to_download,
self.download_dir)):
LOG.info("All python dependencies have been already downloaded")
else:
def try_download(attempt):
LOG.info(
"Downloading %s dependencies with pip (attempt %s)...",
len(pips_to_download), attempt)
output_filename = sh.joinpths(
self.log_dir, "pip-download-attempt-%s.log" % (attempt))
pip_helper.download_dependencies(self.download_dir,
pips_to_download,
output_filename)
utils.retry(self.MAX_PIP_DOWNLOAD_ATTEMPTS,
self.PIP_DOWNLOAD_DELAY, try_download)
# NOTE(harlowja): Mark that we completed downloading successfully
sh.touch_file(self.downloaded_flag_file,
die_if_there=False,
quiet=True,
tracewriter=self.tracewriter)
pips_downloaded = [
pip_helper.extract_requirement(p) for p in pips_to_download
]
what_downloaded = self._examine_download_dir(pips_downloaded,
self.download_dir)
return (pips_downloaded, what_downloaded)
def download_dependencies(self):
"""Download dependencies from `$deps_dir/download-requires`."""
# NOTE(aababilov): do not drop download_dir - it can be reused
sh.mkdirslist(self.download_dir, tracewriter=self.tracewriter)
pips_to_download = self._filter_download_requires()
sh.write_file(self.download_requires_filename,
"\n".join([str(req) for req in pips_to_download]))
if not pips_to_download:
return ([], [])
# NOTE(aababilov): user could have changed persona, so,
# check that all requirements are downloaded....
if self._requirements_satisfied(pips_to_download, self.download_dir):
LOG.info("All python dependencies have been already downloaded")
else:
utils.retry(self.retries, self.retry_delay, self._try_download,
pips_to_download)
pips_downloaded = [
pip_helper.extract_requirement(p) for p in pips_to_download
]
what_downloaded = self._examine_download_dir(pips_downloaded,
self.download_dir)
return (pips_downloaded, what_downloaded)
