def _extract(self, src, dest):
"""extract source distribution into vendor directory"""
finder = FileFinder(src)
for path, _ in finder.find('*'):
base, ext = os.path.splitext(path)
if ext == '.whl':
# Wheels would extract into a directory with the name of the package, but
# we want the platform signifiers, minus the version number.
# Wheel filenames look like:
# {distribution}-{version}(-{build tag})?-{python tag}-{abi tag}-{platform tag}
bits = base.split('-')
# Remove the version number.
bits.pop(1)
target = os.path.join(dest, '-'.join(bits))
mozfile.remove(
target) # remove existing version of vendored package
os.mkdir(target)
mozfile.extract(os.path.join(finder.base, path), target)
else:
# packages extract into package-version directory name and we strip the version
tld = mozfile.extract(os.path.join(finder.base, path), dest)[0]
target = os.path.join(dest, tld.rpartition('-')[0])
mozfile.remove(
target) # remove existing version of vendored package
mozfile.move(tld, target)
# If any files inside the vendored package were symlinks, turn them into normal files
# because hg.mozilla.org forbids symlinks in the repository.
link_finder = FileFinder(target)
for _, f in link_finder.find('**'):
if os.path.islink(f.path):
link_target = os.path.realpath(f.path)
os.unlink(f.path)
shutil.copyfile(link_target, f.path)
def _extract(self, src, dest, keep_extra_files=False):
"""extract source distribution into vendor directory"""
ignore = ()
if not keep_extra_files:
ignore = (
"*/doc",
"*/docs",
"*/test",
"*/tests",
)
finder = FileFinder(src)
for path, _ in finder.find("*"):
base, ext = os.path.splitext(path)
# packages extract into package-version directory name and we strip the version
tld = mozfile.extract(os.path.join(finder.base, path),
dest,
ignore=ignore)[0]
target = os.path.join(dest, tld.rpartition("-")[0])
mozfile.remove(
target) # remove existing version of vendored package
mozfile.move(tld, target)
# If any files inside the vendored package were symlinks, turn them into normal files
# because hg.mozilla.org forbids symlinks in the repository.
link_finder = FileFinder(target)
for _, f in link_finder.find("**"):
if os.path.islink(f.path):
link_target = os.path.realpath(f.path)
os.unlink(f.path)
shutil.copyfile(link_target, f.path)
def install_pip_package(self, package, vendored=False):
"""Install a package via pip.
The supplied package is specified using a pip requirement specifier.
e.g. 'foo' or 'foo==1.0'.
If the package is already installed, this is a no-op.
If vendored is True, no package index will be used and no dependencies
will be installed.
"""
import mozfile
from mozfile import TemporaryDirectory
if sys.executable.startswith(self.bin_path):
# If we're already running in this interpreter, we can optimize in
# the case that the package requirement is already satisfied.
from pip._internal.req.constructors import install_req_from_line
req = install_req_from_line(package)
req.check_if_exists(use_user_site=False)
if req.satisfied_by is not None:
return
args = ["install"]
vendored_dist_info_dir = None
if vendored:
args.extend([
"--no-deps",
"--no-index",
# The setup will by default be performed in an isolated build
# environment, and since we're running with --no-index, this
# means that pip will be unable to install in the isolated build
# environment any dependencies that might be specified in a
# setup_requires directive for the package. Since we're manually
# controlling our build environment, build isolation isn't a
# concern and we can disable that feature. Note that this is
# safe and doesn't risk trampling any other packages that may be
# installed due to passing `--no-deps --no-index` as well.
"--no-build-isolation",
])
vendored_dist_info_dir = next(
(d for d in os.listdir(package) if d.endswith(".dist-info")),
None)
with TemporaryDirectory() as tmp:
if vendored_dist_info_dir:
# This is a vendored wheel. We have to re-pack it in order for pip
# to install it.
wheel_file = os.path.join(
tmp, "{}-1.0-py3-none-any.whl".format(
os.path.basename(package)))
shutil.make_archive(wheel_file, "zip", package)
mozfile.move("{}.zip".format(wheel_file), wheel_file)
package = wheel_file
args.append(package)
return self._run_pip(args, stderr=subprocess.STDOUT)
def test_move_file(self):
file_path = os.path.join(self.tempdir, *stubs.files[1])
moved_path = file_path + '.moved'
self.assertTrue(os.path.isfile(file_path))
self.assertFalse(os.path.exists(moved_path))
mozfile.move(file_path, moved_path)
self.assertFalse(os.path.exists(file_path))
self.assertTrue(os.path.isfile(moved_path))
def test_move_file(self):
file_path = os.path.join(self.tempdir, *stubs.files[1])
moved_path = file_path + '.moved'
self.assertTrue(os.path.isfile(file_path))
self.assertFalse(os.path.exists(moved_path))
mozfile.move(file_path, moved_path)
self.assertFalse(os.path.exists(file_path))
self.assertTrue(os.path.isfile(moved_path))
def test_move_file_with_retry(self):
file_path = os.path.join(self.tempdir, *stubs.files[1])
moved_path = file_path + '.moved'
with wait_file_opened_in_thread(file_path, 0.2):
# first move attempt should fail on windows and be retried
mozfile.move(file_path, moved_path)
self.assertFalse(os.path.exists(file_path))
self.assertTrue(os.path.isfile(moved_path))
def test_move_file_with_retry(self):
file_path = os.path.join(self.tempdir, *stubs.files[1])
moved_path = file_path + '.moved'
with wait_file_opened_in_thread(file_path, 0.2):
# first move attempt should fail on windows and be retried
mozfile.move(file_path, moved_path)
self.assertFalse(os.path.exists(file_path))
self.assertTrue(os.path.isfile(moved_path))
def _extract(self, src, dest, keep_extra_files=False):
"""extract source distribution into vendor directory"""
ignore = ()
if not keep_extra_files:
ignore = ("*/doc", "*/docs", "*/test", "*/tests")
finder = FileFinder(src)
for archive, _ in finder.find("*"):
_, ext = os.path.splitext(archive)
archive_path = os.path.join(finder.base, archive)
if ext == ".whl":
# Archive is named like "$package-name-1.0-py2.py3-none-any.whl", and should
# have four dashes that aren't part of the package name.
package_name, version, spec, abi, platform_and_suffix = archive.rsplit(
"-", 4
)
target_package_dir = os.path.join(dest, package_name)
mozfile.remove(target_package_dir)
os.mkdir(target_package_dir)
# Extract all the contents of the wheel into the package subdirectory.
# We're expecting at least a code directory and a ".dist-info" directory,
# though there may be a ".data" directory as well.
mozfile.extract(archive_path, target_package_dir, ignore=ignore)
_denormalize_symlinks(target_package_dir)
else:
# Archive is named like "$package-name-1.0.tar.gz", and the rightmost
# dash should separate the package name from the rest of the archive
# specifier.
package_name, archive_postfix = archive.rsplit("-", 1)
package_dir = os.path.join(dest, package_name)
mozfile.remove(package_dir)
# The archive should only contain one top-level directory, which has
# the source files. We extract this directory directly to
# the vendor directory.
extracted_files = mozfile.extract(archive_path, dest, ignore=ignore)
assert len(extracted_files) == 1
extracted_package_dir = extracted_files[0]
# The extracted package dir includes the version in the name,
# which we don't we don't want.
mozfile.move(extracted_package_dir, package_dir)
_denormalize_symlinks(package_dir)
def _download(self, url, dest, finished_callback, chunk_size, session):
# save the file under a temporary name
# this allow to not use a broken file in case things went really bad
# while downloading the file (ie the python interpreter is killed
# abruptly)
temp = None
bytes_so_far = 0
try:
with closing(session.get(url, stream=True)) as response:
total_size = int(response.headers['Content-length'].strip())
self._update_progress(bytes_so_far, total_size)
# we use NamedTemporaryFile as raw open() call was causing
# issues on windows - see:
# https://bugzilla.mozilla.org/show_bug.cgi?id=1185756
with tempfile.NamedTemporaryFile(
delete=False,
mode='wb',
suffix='.tmp',
dir=os.path.dirname(dest)) as temp:
for chunk in response.iter_content(chunk_size):
if self.is_canceled():
break
if chunk:
temp.write(chunk)
bytes_so_far += len(chunk)
self._update_progress(bytes_so_far, total_size)
response.raise_for_status()
except Exception:
self.__error = sys.exc_info()
try:
if temp is None:
pass # not even opened the temp file, nothing to do
elif self.is_canceled() or self.__error:
mozfile.remove(temp.name)
else:
# if all goes well, then rename the file to the real dest
mozfile.remove(dest) # just in case it already existed
mozfile.move(temp.name, dest)
finally:
if finished_callback:
finished_callback(self)
def _download(self, url, dest, finished_callback, chunk_size, session):
# save the file under a temporary name
# this allow to not use a broken file in case things went really bad
# while downloading the file (ie the python interpreter is killed
# abruptly)
temp = None
bytes_so_far = 0
try:
with closing(session.get(url, stream=True)) as response:
total_size = int(response.headers['Content-length'].strip())
self._update_progress(bytes_so_far, total_size)
# we use NamedTemporaryFile as raw open() call was causing
# issues on windows - see:
# https://bugzilla.mozilla.org/show_bug.cgi?id=1185756
with tempfile.NamedTemporaryFile(
delete=False,
suffix='.tmp',
dir=os.path.dirname(dest)) as temp:
for chunk in response.iter_content(chunk_size):
if self.is_canceled():
break
if chunk:
temp.write(chunk)
bytes_so_far += len(chunk)
self._update_progress(bytes_so_far, total_size)
response.raise_for_status()
except:
self.__error = sys.exc_info()
try:
if temp is None:
pass # not even opened the temp file, nothing to do
elif self.is_canceled() or self.__error:
mozfile.remove(temp.name)
else:
# if all goes well, then rename the file to the real dest
mozfile.remove(dest) # just in case it already existed
mozfile.move(temp.name, dest)
finally:
if finished_callback:
finished_callback(self)
