def _extract(self, src, dest): """extract source distribution into vendor directory""" finder = FileFinder(src) for path, _ in finder.find('*'): base, ext = os.path.splitext(path) if ext == '.whl': # Wheels would extract into a directory with the name of the package, but # we want the platform signifiers, minus the version number. # Wheel filenames look like: # {distribution}-{version}(-{build tag})?-{python tag}-{abi tag}-{platform tag} bits = base.split('-') # Remove the version number. bits.pop(1) target = os.path.join(dest, '-'.join(bits)) mozfile.remove( target) # remove existing version of vendored package os.mkdir(target) mozfile.extract(os.path.join(finder.base, path), target) else: # packages extract into package-version directory name and we strip the version tld = mozfile.extract(os.path.join(finder.base, path), dest)[0] target = os.path.join(dest, tld.rpartition('-')[0]) mozfile.remove( target) # remove existing version of vendored package mozfile.move(tld, target) # If any files inside the vendored package were symlinks, turn them into normal files # because hg.mozilla.org forbids symlinks in the repository. link_finder = FileFinder(target) for _, f in link_finder.find('**'): if os.path.islink(f.path): link_target = os.path.realpath(f.path) os.unlink(f.path) shutil.copyfile(link_target, f.path)
def _extract(self, src, dest, keep_extra_files=False): """extract source distribution into vendor directory""" ignore = () if not keep_extra_files: ignore = ( "*/doc", "*/docs", "*/test", "*/tests", ) finder = FileFinder(src) for path, _ in finder.find("*"): base, ext = os.path.splitext(path) # packages extract into package-version directory name and we strip the version tld = mozfile.extract(os.path.join(finder.base, path), dest, ignore=ignore)[0] target = os.path.join(dest, tld.rpartition("-")[0]) mozfile.remove( target) # remove existing version of vendored package mozfile.move(tld, target) # If any files inside the vendored package were symlinks, turn them into normal files # because hg.mozilla.org forbids symlinks in the repository. link_finder = FileFinder(target) for _, f in link_finder.find("**"): if os.path.islink(f.path): link_target = os.path.realpath(f.path) os.unlink(f.path) shutil.copyfile(link_target, f.path)
def install_pip_package(self, package, vendored=False): """Install a package via pip. The supplied package is specified using a pip requirement specifier. e.g. 'foo' or 'foo==1.0'. If the package is already installed, this is a no-op. If vendored is True, no package index will be used and no dependencies will be installed. """ import mozfile from mozfile import TemporaryDirectory if sys.executable.startswith(self.bin_path): # If we're already running in this interpreter, we can optimize in # the case that the package requirement is already satisfied. from pip._internal.req.constructors import install_req_from_line req = install_req_from_line(package) req.check_if_exists(use_user_site=False) if req.satisfied_by is not None: return args = ["install"] vendored_dist_info_dir = None if vendored: args.extend([ "--no-deps", "--no-index", # The setup will by default be performed in an isolated build # environment, and since we're running with --no-index, this # means that pip will be unable to install in the isolated build # environment any dependencies that might be specified in a # setup_requires directive for the package. Since we're manually # controlling our build environment, build isolation isn't a # concern and we can disable that feature. Note that this is # safe and doesn't risk trampling any other packages that may be # installed due to passing `--no-deps --no-index` as well. "--no-build-isolation", ]) vendored_dist_info_dir = next( (d for d in os.listdir(package) if d.endswith(".dist-info")), None) with TemporaryDirectory() as tmp: if vendored_dist_info_dir: # This is a vendored wheel. We have to re-pack it in order for pip # to install it. wheel_file = os.path.join( tmp, "{}-1.0-py3-none-any.whl".format( os.path.basename(package))) shutil.make_archive(wheel_file, "zip", package) mozfile.move("{}.zip".format(wheel_file), wheel_file) package = wheel_file args.append(package) return self._run_pip(args, stderr=subprocess.STDOUT)
def test_move_file(self): file_path = os.path.join(self.tempdir, *stubs.files[1]) moved_path = file_path + '.moved' self.assertTrue(os.path.isfile(file_path)) self.assertFalse(os.path.exists(moved_path)) mozfile.move(file_path, moved_path) self.assertFalse(os.path.exists(file_path)) self.assertTrue(os.path.isfile(moved_path))
def test_move_file_with_retry(self): file_path = os.path.join(self.tempdir, *stubs.files[1]) moved_path = file_path + '.moved' with wait_file_opened_in_thread(file_path, 0.2): # first move attempt should fail on windows and be retried mozfile.move(file_path, moved_path) self.assertFalse(os.path.exists(file_path)) self.assertTrue(os.path.isfile(moved_path))
def _extract(self, src, dest, keep_extra_files=False): """extract source distribution into vendor directory""" ignore = () if not keep_extra_files: ignore = ("*/doc", "*/docs", "*/test", "*/tests") finder = FileFinder(src) for archive, _ in finder.find("*"): _, ext = os.path.splitext(archive) archive_path = os.path.join(finder.base, archive) if ext == ".whl": # Archive is named like "$package-name-1.0-py2.py3-none-any.whl", and should # have four dashes that aren't part of the package name. package_name, version, spec, abi, platform_and_suffix = archive.rsplit( "-", 4 ) target_package_dir = os.path.join(dest, package_name) mozfile.remove(target_package_dir) os.mkdir(target_package_dir) # Extract all the contents of the wheel into the package subdirectory. # We're expecting at least a code directory and a ".dist-info" directory, # though there may be a ".data" directory as well. mozfile.extract(archive_path, target_package_dir, ignore=ignore) _denormalize_symlinks(target_package_dir) else: # Archive is named like "$package-name-1.0.tar.gz", and the rightmost # dash should separate the package name from the rest of the archive # specifier. package_name, archive_postfix = archive.rsplit("-", 1) package_dir = os.path.join(dest, package_name) mozfile.remove(package_dir) # The archive should only contain one top-level directory, which has # the source files. We extract this directory directly to # the vendor directory. extracted_files = mozfile.extract(archive_path, dest, ignore=ignore) assert len(extracted_files) == 1 extracted_package_dir = extracted_files[0] # The extracted package dir includes the version in the name, # which we don't we don't want. mozfile.move(extracted_package_dir, package_dir) _denormalize_symlinks(package_dir)
def _download(self, url, dest, finished_callback, chunk_size, session): # save the file under a temporary name # this allow to not use a broken file in case things went really bad # while downloading the file (ie the python interpreter is killed # abruptly) temp = None bytes_so_far = 0 try: with closing(session.get(url, stream=True)) as response: total_size = int(response.headers['Content-length'].strip()) self._update_progress(bytes_so_far, total_size) # we use NamedTemporaryFile as raw open() call was causing # issues on windows - see: # https://bugzilla.mozilla.org/show_bug.cgi?id=1185756 with tempfile.NamedTemporaryFile( delete=False, mode='wb', suffix='.tmp', dir=os.path.dirname(dest)) as temp: for chunk in response.iter_content(chunk_size): if self.is_canceled(): break if chunk: temp.write(chunk) bytes_so_far += len(chunk) self._update_progress(bytes_so_far, total_size) response.raise_for_status() except Exception: self.__error = sys.exc_info() try: if temp is None: pass # not even opened the temp file, nothing to do elif self.is_canceled() or self.__error: mozfile.remove(temp.name) else: # if all goes well, then rename the file to the real dest mozfile.remove(dest) # just in case it already existed mozfile.move(temp.name, dest) finally: if finished_callback: finished_callback(self)
def _download(self, url, dest, finished_callback, chunk_size, session): # save the file under a temporary name # this allow to not use a broken file in case things went really bad # while downloading the file (ie the python interpreter is killed # abruptly) temp = None bytes_so_far = 0 try: with closing(session.get(url, stream=True)) as response: total_size = int(response.headers['Content-length'].strip()) self._update_progress(bytes_so_far, total_size) # we use NamedTemporaryFile as raw open() call was causing # issues on windows - see: # https://bugzilla.mozilla.org/show_bug.cgi?id=1185756 with tempfile.NamedTemporaryFile( delete=False, suffix='.tmp', dir=os.path.dirname(dest)) as temp: for chunk in response.iter_content(chunk_size): if self.is_canceled(): break if chunk: temp.write(chunk) bytes_so_far += len(chunk) self._update_progress(bytes_so_far, total_size) response.raise_for_status() except: self.__error = sys.exc_info() try: if temp is None: pass # not even opened the temp file, nothing to do elif self.is_canceled() or self.__error: mozfile.remove(temp.name) else: # if all goes well, then rename the file to the real dest mozfile.remove(dest) # just in case it already existed mozfile.move(temp.name, dest) finally: if finished_callback: finished_callback(self)