def print_results(hits, name_column_width=None, terminal_width=None): if not hits: return if name_column_width is None: name_column_width = max((len(hit['name']) for hit in hits)) + 4 installed_packages = [p.project_name for p in pkg_resources.working_set] for hit in hits: name = hit['name'] summary = hit['summary'] or '' if terminal_width is not None: # wrap and indent summary to fit terminal summary = textwrap.wrap( summary, terminal_width - name_column_width - 5, ) summary = ('\n' + ' ' * (name_column_width + 3)).join(summary) line = '%s - %s' % (name.ljust(name_column_width), summary) try: logger.info(line) with indent_log(): latest = highest_version(hit['versions']) logger.info('LATEST: %s', latest) if name in installed_packages: dist = pkg_resources.get_distribution(name) with indent_log(): logger.info('INSTALLED: %s' % dist.version) except UnicodeEncodeError: pass
def install(self, install_options, global_options=(), *args, **kwargs): """ Install everything in this set (after having downloaded and unpacked the packages) """ to_install = self._to_install() if to_install: logger.info("Installing collected packages: %s", ", ".join([req.name for req in to_install])) with indent_log(): for requirement in to_install: if requirement.conflicts_with: logger.info("Found existing installation: %s", requirement.conflicts_with) with indent_log(): requirement.uninstall(auto_confirm=True) try: requirement.install(install_options, global_options, *args, **kwargs) except: # if install did not succeed, rollback previous uninstall if requirement.conflicts_with and not requirement.install_succeeded: requirement.rollback_uninstall() raise else: if requirement.conflicts_with and requirement.install_succeeded: requirement.commit_uninstall() requirement.remove_temporary_source() self.successfully_installed = to_install
def list(self, options, args): if args: raise InstallationError( 'You cannot give an argument with --list') for path in sorted(self.paths()): if not os.path.exists(path): continue basename = os.path.basename(path.rstrip(os.path.sep)) if os.path.isfile(path) and zipfile.is_zipfile(path): if os.path.dirname(path) not in self.paths(): logger.info('Zipped egg: %s', display_path(path)) continue if (basename != 'site-packages' and basename != 'dist-packages' and not path.replace('\\', '/').endswith('lib/python')): continue logger.info('In %s:', display_path(path)) with indent_log(): zipped = [] unzipped = [] for filename in sorted(os.listdir(path)): ext = os.path.splitext(filename)[1].lower() if ext in ('.pth', '.egg-info', '.egg-link'): continue if ext == '.py': logger.debug( 'Not displaying %s: not a package', display_path(filename) ) continue full = os.path.join(path, filename) if os.path.isdir(full): unzipped.append((filename, self.count_package(full))) elif zipfile.is_zipfile(full): zipped.append(filename) else: logger.debug( 'Unknown file: %s', display_path(filename), ) if zipped: logger.info('Zipped packages:') with indent_log(): for filename in zipped: logger.info(filename) else: logger.info('No zipped packages.') if unzipped: if options.sort_files: unzipped.sort(key=lambda x: -x[1]) logger.info('Unzipped packages:') with indent_log(): for filename, count in unzipped: logger.info('%s (%i files)', filename, count) else: logger.info('No unzipped packages.')
def _copy_dist_from_dir(link_path, location): """Copy distribution files in `link_path` to `location`. Invoked when user requests to install a local directory. E.g.: pip install . pip install ~/dev/git-repos/python-prompt-toolkit """ # Note: This is currently VERY SLOW if you have a lot of data in the # directory, because it copies everything with `shutil.copytree`. # What it should really do is build an sdist and install that. # See https://github.com/pypa/pip/issues/2195 if os.path.isdir(location): rmtree(location) # build an sdist setup_py = 'setup.py' sdist_args = [sys.executable] sdist_args.append('-c') sdist_args.append(SETUPTOOLS_SHIM % setup_py) sdist_args.append('sdist') sdist_args += ['--dist-dir', location] logger.info('Running setup.py sdist for %s', link_path) with indent_log(): call_subprocess(sdist_args, cwd=link_path, show_stdout=False) # unpack sdist into `location` sdist = os.path.join(location, os.listdir(location)[0]) logger.info('Unpacking sdist %s into %s', sdist, location) unpack_file(sdist, location, content_type=None, link=None)
def _find_all_versions(self, project_name): """Find all available versions for project_name This checks index_urls, find_links and dependency_links All versions found are returned See _link_package_versions for details on which files are accepted """ index_locations = self._get_index_urls_locations(project_name) file_locations, url_locations = self._sort_locations(index_locations) fl_file_loc, fl_url_loc = self._sort_locations(self.find_links) file_locations.extend(fl_file_loc) url_locations.extend(fl_url_loc) _flocations, _ulocations = self._sort_locations(self.dependency_links) file_locations.extend(_flocations) # We trust every url that the user has given us whether it was given # via --index-url or --find-links locations = [Link(url, trusted=True) for url in url_locations] # We explicitly do not trust links that came from dependency_links locations.extend([Link(url) for url in _ulocations]) logger.debug("%d location(s) to search for versions of %s:", len(locations), project_name) for location in locations: logger.debug("* %s", location) self._validate_secure_origin(logger, location) find_links_versions = list( self._package_versions( # We trust every directly linked archive in find_links (Link(url, "-f", trusted=True) for url in self.find_links), project_name.lower(), ) ) page_versions = [] for page in self._get_pages(locations, project_name): logger.debug("Analyzing links from page %s", page.url) with indent_log(): page_versions.extend(self._package_versions(page.links, project_name.lower())) dependency_versions = list( self._package_versions((Link(url) for url in self.dependency_links), project_name.lower()) ) if dependency_versions: logger.debug( "dependency_links found: %s", ", ".join([version.location.url for version in dependency_versions]) ) file_versions = list(self._package_versions((Link(url) for url in file_locations), project_name.lower())) if file_versions: file_versions.sort(reverse=True) logger.debug( "Local files found: %s", ", ".join([url_to_path(candidate.location.url) for candidate in file_versions]) ) # This is an intentional priority ordering return file_versions + find_links_versions + page_versions + dependency_versions
def run_egg_info(self): assert self.source_dir if self.name: logger.debug( 'Running setup.py (path:%s) egg_info for package %s', self.setup_py, self.name, ) else: logger.debug( 'Running setup.py (path:%s) egg_info for package from %s', self.setup_py, self.link, ) with indent_log(): script = SETUPTOOLS_SHIM % self.setup_py base_cmd = [sys.executable, '-c', script] if self.isolated: base_cmd += ["--no-user-cfg"] egg_info_cmd = base_cmd + ['egg_info'] # We can't put the .egg-info files at the root, because then the # source code will be mistaken for an installed egg, causing # problems if self.editable: egg_base_option = [] else: egg_info_dir = os.path.join(self.setup_py_dir, 'pip-egg-info') ensure_dir(egg_info_dir) egg_base_option = ['--egg-base', 'pip-egg-info'] call_subprocess( egg_info_cmd + egg_base_option, cwd=self.setup_py_dir, show_stdout=False, command_level=logging.DEBUG, command_desc='python setup.py egg_info') if not self.req: if isinstance( pkg_resources.parse_version(self.pkg_info()["Version"]), Version): op = "==" else: op = "===" self.req = Requirement( "".join([ self.pkg_info()["Name"], op, self.pkg_info()["Version"], ]) ) self._correct_build_location() else: metadata_name = canonicalize_name(self.pkg_info()["Name"]) if canonicalize_name(self.req.name) != metadata_name: logger.warning( 'Running setup.py (path:%s) egg_info for package %s ' 'produced metadata for project name %s. Fix your ' '#egg=%s fragments.', self.setup_py, self.name, metadata_name, self.name ) self.req = Requirement(metadata_name)
def install_editable(self, install_options, global_options=()): logger.info('Running setup.py develop for %s', self.name) if self.isolated: global_options = list(global_options) + ["--no-user-cfg"] with indent_log(): # FIXME: should we do --install-headers here too? cwd = self.source_dir if self.editable_options and \ 'subdirectory' in self.editable_options: cwd = os.path.join(cwd, self.editable_options['subdirectory']) call_subprocess( [ sys.executable, '-c', "import setuptools, tokenize; __file__=%r; exec(compile(" "getattr(tokenize, 'open', open)(__file__).read().replace" "('\\r\\n', '\\n'), __file__, 'exec'))" % self.setup_py ] + list(global_options) + ['develop', '--no-deps'] + list(install_options), cwd=cwd, filter_stdout=self._filter_install, show_stdout=False) self.install_succeeded = True
def remove(self, auto_confirm=False, verbose=False): """Remove paths in ``self.paths`` with confirmation (unless ``auto_confirm`` is True).""" if not self.paths: logger.info( "Can't uninstall '%s'. No files were found to uninstall.", self.dist.project_name, ) return dist_name_version = ( self.dist.project_name + "-" + self.dist.version ) logger.info('Uninstalling %s:', dist_name_version) with indent_log(): if auto_confirm or self._allowed_to_proceed(verbose): self.save_dir.create() for path in sorted(self.compact(self.paths)): new_path = self._stash(path) logger.debug('Removing file or directory %s', path) self._moved_paths.append(path) renames(path, new_path) for pth in self.pth.values(): pth.remove() logger.info('Successfully uninstalled %s', dist_name_version)
def install_editable(self, install_options, global_options=(), prefix=None): logger.info('Running setup.py develop for %s', self.name) if self.isolated: global_options = list(global_options) + ["--no-user-cfg"] if prefix: prefix_param = ['--prefix={0}'.format(prefix)] install_options = list(install_options) + prefix_param with indent_log(): # FIXME: should we do --install-headers here too? call_subprocess( [ sys.executable, '-c', SETUPTOOLS_SHIM % self.setup_py ] + list(global_options) + ['develop', '--no-deps'] + list(install_options), cwd=self.setup_py_dir, show_stdout=False) self.install_succeeded = True
def print_results(hits, name_column_width=None, terminal_width=None): if not hits: return if name_column_width is None: name_column_width = max([ len(hit['name']) + len(highest_version(hit.get('versions', ['-']))) for hit in hits ]) + 4 installed_packages = [p.project_name for p in pkg_resources.working_set] for hit in hits: name = hit['name'] summary = hit['summary'] or '' latest = highest_version(hit.get('versions', ['-'])) if terminal_width is not None: target_width = terminal_width - name_column_width - 5 if target_width > 10: # wrap and indent summary to fit terminal summary = textwrap.wrap(summary, target_width) summary = ('\n' + ' ' * (name_column_width + 3)).join(summary) line = '%-*s - %s' % (name_column_width, '%s (%s)' % (name, latest), summary) try: logger.info(line) if name in installed_packages: dist = pkg_resources.get_distribution(name) with indent_log(): if dist.version == latest: logger.info('INSTALLED: %s (latest)', dist.version) else: logger.info('INSTALLED: %s', dist.version) logger.info('LATEST: %s', latest) except UnicodeEncodeError: pass
def remove(self, auto_confirm=False): """Remove paths in ``self.paths`` with confirmation (unless ``auto_confirm`` is True).""" if not self._can_uninstall(): return if not self.paths: logger.info("Can't uninstall '%s'. No files were found to uninstall.", self.dist.project_name) return logger.info("Uninstalling %s-%s:", self.dist.project_name, self.dist.version) with indent_log(): paths = sorted(self.compact(self.paths)) if auto_confirm: response = "y" else: for path in paths: logger.info(path) response = ask("Proceed (y/n)? ", ("y", "n")) if self._refuse: logger.info("Not removing or modifying (outside of prefix):") for path in self.compact(self._refuse): logger.info(path) if response == "y": self.save_dir = tempfile.mkdtemp(suffix="-uninstall", prefix="pip-") for path in paths: new_path = self._stash(path) logger.debug("Removing file or directory %s", path) self._moved_paths.append(path) renames(path, new_path) for pth in self.pth.values(): pth.remove() logger.info("Successfully uninstalled %s-%s", self.dist.project_name, self.dist.version)
def run(self): if getattr(self.distribution, 'salt_download_windows_dlls', None) is None: print('This command is not meant to be called on it\'s own') exit(1) import platform from pip.utils.logging import indent_log platform_bits, _ = platform.architecture() url = 'https://repo.saltstack.com/windows/dependencies/{bits}/{fname}.dll' dest = os.path.join(os.path.dirname(sys.executable), '{fname}.dll') with indent_log(): for fname in ('libeay32', 'ssleay32', 'libsodium', 'msvcr120'): # See if the library is already on the system if find_library(fname): continue furl = url.format(bits=platform_bits[:2], fname=fname) fdest = dest.format(fname=fname) if not os.path.exists(fdest): log.info('Downloading {0}.dll to {1} from {2}'.format(fname, fdest, furl)) try: import requests from contextlib import closing with closing(requests.get(furl, stream=True)) as req: if req.status_code == 200: with open(fdest, 'wb') as wfh: for chunk in req.iter_content(chunk_size=4096): if chunk: # filter out keep-alive new chunks wfh.write(chunk) wfh.flush() else: log.error( 'Failed to download {0}.dll to {1} from {2}'.format( fname, fdest, furl ) ) except ImportError: req = urlopen(furl) if req.getcode() == 200: with open(fdest, 'wb') as wfh: if IS_PY3: while True: chunk = req.read(4096) if len(chunk) == 0: break; wfh.write(chunk) wfh.flush() else: while True: for chunk in req.read(4096): if not chunk: break wfh.write(chunk) wfh.flush() else: log.error( 'Failed to download {0}.dll to {1} from {2}'.format( fname, fdest, furl ) )
def _display(msg, paths): if not paths: return logger.info(msg) with indent_log(): for path in sorted(self.compact(paths)): logger.info(path)
def cleanup_files(self): """Clean up files, remove builds.""" logger.debug('Cleaning up...') with indent_log(): for req in self.reqs_to_cleanup: req.remove_temporary_source() if self._pip_has_created_build_dir(): logger.debug('Removing temporary dir %s...', self.build_dir) rmtree(self.build_dir)
def run_egg_info(self): assert self.source_dir if self.name: logger.debug( 'Running setup.py (path:%s) egg_info for package %s', self.setup_py, self.name, ) else: logger.debug( 'Running setup.py (path:%s) egg_info for package from %s', self.setup_py, self.link, ) with indent_log(): script = self._run_setup_py script = script.replace('__SETUP_PY__', repr(self.setup_py)) script = script.replace('__PKG_NAME__', repr(self.name)) base_cmd = [sys.executable, '-c', script] if self.isolated: base_cmd += ["--no-user-cfg"] egg_info_cmd = base_cmd + ['egg_info'] # We can't put the .egg-info files at the root, because then the # source code will be mistaken for an installed egg, causing # problems if self.editable: egg_base_option = [] else: egg_info_dir = os.path.join(self.source_dir, 'pip-egg-info') ensure_dir(egg_info_dir) egg_base_option = ['--egg-base', 'pip-egg-info'] cwd = self.source_dir if self.editable_options and \ 'subdirectory' in self.editable_options: cwd = os.path.join(cwd, self.editable_options['subdirectory']) call_subprocess( egg_info_cmd + egg_base_option, cwd=cwd, show_stdout=False, command_level=logging.DEBUG, command_desc='python setup.py egg_info') if not self.req: if isinstance( pkg_resources.parse_version(self.pkg_info()["Version"]), Version): op = "==" else: op = "===" self.req = pkg_resources.Requirement.parse( "".join([ self.pkg_info()["Name"], op, self.pkg_info()["Version"], ])) self._correct_build_location()
def zip_package(self, module_name, filename, no_pyc): logger.info('Zip %s (in %s)', module_name, display_path(filename)) orig_filename = filename if filename.endswith('.egg'): dest_filename = filename else: dest_filename = filename + '.zip' with indent_log(): # FIXME: I think this needs to be undoable: if filename == dest_filename: filename = backup_dir(orig_filename) logger.info( 'Moving %s aside to %s', orig_filename, filename, ) if not self.simulate: shutil.move(orig_filename, filename) try: logger.debug( 'Creating zip file in %s', display_path(dest_filename), ) if not self.simulate: zip = zipfile.ZipFile(dest_filename, 'w') zip.writestr(module_name + '/', '') for dirpath, dirnames, filenames in os.walk(filename): if no_pyc: filenames = [f for f in filenames if not f.lower().endswith('.pyc')] for fns, is_dir in [ (dirnames, True), (filenames, False)]: for fn in fns: full = os.path.join(dirpath, fn) dest = os.path.join( module_name, dirpath[len(filename):].lstrip( os.path.sep ), fn, ) if is_dir: zip.writestr(dest + '/', '') else: zip.write(full, dest) zip.close() logger.debug( 'Removing old directory %s', display_path(filename), ) if not self.simulate: rmtree(filename) except: # FIXME: need to do an undo here raise # FIXME: should also be undone: self.add_filename_to_pth(dest_filename)
def export(self, location): """Export the svn repository at the url to the destination location""" url, rev = self.get_url_rev() rev_options = get_rev_options(url, rev) logger.info("Exporting svn repository %s to %s", url, location) with indent_log(): if os.path.exists(location): # Subversion doesn't like to check out over an existing # directory --force fixes this, but was only added in svn 1.5 rmtree(location) self.run_command(["export"] + rev_options + [url, location], show_stdout=False)
def run(self): if getattr(self.distribution, 'salt_installing_m2crypto_windows', None) is None: print('This command is not meant to be called on it\'s own') exit(1) import platform from pip.utils import call_subprocess from pip.utils.logging import indent_log platform_bits, _ = platform.architecture() with indent_log(): call_subprocess( ['pip', 'install', '--egg', 'M2CryptoWin{0}'.format(platform_bits[:2])] )
def export(self, location): """Export the svn repository at the url to the destination location""" url, rev = self.get_url_rev() rev_options = get_rev_options(url, rev) logger.info('Exporting svn repository %s to %s', url, location) with indent_log(): if os.path.exists(location): # Subversion doesn't like to check out over an existing # directory --force fixes this, but was only added in svn 1.5 rmtree(location) self.run_command(['export'] + rev_options + [url, location], show_stdout=False)
def install(self, install_options, global_options=(), *args, **kwargs): """ Install everything in this set (after having downloaded and unpacked the packages) """ to_install = self._to_install() if to_install: logger.info( 'Installing collected packages: %s', ', '.join([req.name for req in to_install]), ) with indent_log(): for requirement in to_install: if requirement.conflicts_with: logger.info( 'Found existing installation: %s', requirement.conflicts_with, ) with indent_log(): requirement.uninstall(auto_confirm=True) try: requirement.install(install_options, global_options, *args, **kwargs) except: should_rollback = (requirement.conflicts_with and not requirement.install_succeeded) # if install did not succeed, rollback previous uninstall if should_rollback: requirement.uninstalled_pathset.rollback() raise else: should_commit = (requirement.conflicts_with and requirement.install_succeeded) if should_commit: requirement.uninstalled_pathset.commit() requirement.remove_temporary_source() self.successfully_installed = to_install
def build(self): """Build wheels.""" # unpack and constructs req set self.requirement_set.prepare_files(self.finder) reqset = self.requirement_set.requirements.values() buildset = [] for req in reqset: if req.is_wheel: logger.info( 'Skipping %s, due to already being wheel.', req.name, ) elif req.editable: logger.info( 'Skipping %s, due to being editable', req.name, ) else: buildset.append(req) if not buildset: return True # Build the wheels. logger.info( 'Building wheels for collected packages: %s', ', '.join([req.name for req in buildset]), ) with indent_log(): build_success, build_failure = [], [] for req in buildset: if self._build_one(req): build_success.append(req) else: build_failure.append(req) # notify success/failure if build_success: logger.info( 'Successfully built %s', ' '.join([req.name for req in build_success]), ) if build_failure: logger.info( 'Failed to build %s', ' '.join([req.name for req in build_failure]), ) # Return True if all builds were successful return len(build_failure) == 0
def export(self, location): """Export the svn repository at the url to the destination location""" url, rev = self.get_url_rev() rev_options = get_rev_options(url, rev) logger.info('Exporting svn repository %s to %s', url, location) with indent_log(): if os.path.exists(location): # Subversion doesn't like to check out over an existing # directory --force fixes this, but was only added in svn 1.5 rmtree(location) call_subprocess( [self.cmd, 'export'] + rev_options + [url, location], filter_stdout=self._filter, show_stdout=False)
def run(self): if getattr(self.distribution, 'salt_download_windows_dlls', None) is None: print('This command is not meant to be called on it\'s own') exit(1) import platform from pip.utils.logging import indent_log platform_bits, _ = platform.architecture() url = 'https://repo.saltstack.com/windows/dependencies/{bits}/{fname}.dll' dest = os.path.join(os.path.dirname(sys.executable), '{fname}.dll') with indent_log(): for fname in ('libeay32', 'ssleay32', 'libsodium', 'msvcr120'): # See if the library is already on the system if find_library(fname): continue furl = url.format(bits=platform_bits[:2], fname=fname) fdest = dest.format(fname=fname) if not os.path.exists(fdest): log.info('Downloading {0}.dll to {1} from {2}'.format( fname, fdest, furl)) try: import requests from contextlib import closing with closing(requests.get(furl, stream=True)) as req: if req.status_code == 200: with open(fdest, 'wb') as wfh: for chunk in req.iter_content( chunk_size=4096): if chunk: # filter out keep-alive new chunks wfh.write(chunk) wfh.flush() else: log.error( 'Failed to download {0}.dll to {1} from {2}' .format(fname, fdest, furl)) except ImportError: req = urlopen(furl) if req.getcode() == 200: with open(fdest, 'wb') as wfh: while True: chunk = req.read(4096) if not chunk: break wfh.write(chunk) wfh.flush() else: log.error( 'Failed to download {0}.dll to {1} from {2}'. format(fname, fdest, furl))
def run(self): if getattr(self.distribution, 'salt_installing_pyyaml_windows', None) is None: print('This command is not meant to be called on it\'s own') exit(1) import platform from pip.utils import call_subprocess from pip.utils.logging import indent_log platform_bits, _ = platform.architecture() call_arguments = ['easy_install', '-Z'] if platform_bits == '64bit': call_arguments.append( uri_to_resource('64/PyYAML-3.11.win-amd64-py2.7.exe')) else: call_arguments.append( uri_to_resource('32/PyYAML-3.11.win32-py2.7.exe')) with indent_log(): call_subprocess(call_arguments)
def remove(self, auto_confirm=False): """Remove paths in ``self.paths`` with confirmation (unless ``auto_confirm`` is True).""" if not self._can_uninstall(): return if not self.paths: logger.info( "Can't uninstall '%s'. No files were found to uninstall.", self.dist.project_name, ) return logger.info( 'Uninstalling %s-%s:', self.dist.project_name, self.dist.version ) with indent_log(): paths = sorted(self.compact(self.paths)) if auto_confirm: response = 'y' else: for path in paths: logger.info(path) response = ask('Proceed (y/n)? ', ('y', 'n')) if self._refuse: logger.info('Not removing or modifying (outside of prefix):') for path in self.compact(self._refuse): logger.info(path) if response == 'y': self.save_dir = tempfile.mkdtemp(suffix='-uninstall', prefix='pip-') for path in paths: new_path = self._stash(path) logger.debug('Removing file or directory %s', path) self._moved_paths.append(path) renames(path, new_path) for pth in self.pth.values(): pth.remove() logger.info( 'Successfully uninstalled %s-%s', self.dist.project_name, self.dist.version )
def run(self): if getattr(self.distribution, 'salt_installing_pycrypto_windows', None) is None: print('This command is not meant to be called on it\'s own') exit(1) import platform from pip.utils import call_subprocess from pip.utils.logging import indent_log platform_bits, _ = platform.architecture() call_arguments = ['pip', 'install', 'wheel'] if platform_bits == '64bit': call_arguments.append( 'http://repo.saltstack.com/windows/dependencies/64/pycrypto-2.6.1-cp27-none-win_amd64.whl' ) else: call_arguments.append( 'http://repo.saltstack.com/windows/dependencies/32/pycrypto-2.6.1-cp27-none-win32.whl' ) with indent_log(): call_subprocess(call_arguments)
def run(self): if getattr(self.distribution, 'salt_installing_pyyaml_windows', None) is None: print('This command is not meant to be called on it\'s own') exit(1) import platform from pip.utils import call_subprocess from pip.utils.logging import indent_log platform_bits, _ = platform.architecture() call_arguments = ['easy_install', '-Z'] if platform_bits == '64bit': call_arguments.append( 'https://repo.saltstack.com/windows/dependencies/64/PyYAML-3.11.win-amd64-py2.7.exe' ) else: call_arguments.append( 'https://repo.saltstack.com/windows/dependencies/32/PyYAML-3.11.win32-py2.7.exe' ) with indent_log(): call_subprocess(call_arguments)
def _prepare_installed_requirement(self, req, resolver, skip_reason): """Prepare an already-installed requirement """ assert req.satisfied_by, "req should have been satisfied but isn't" assert skip_reason is not None, ( "did not get skip reason skipped but req.satisfied_by " "is set to %r" % (req.satisfied_by, )) logger.info('Requirement %s: %s (%s)', skip_reason, req, req.satisfied_by.version) with indent_log(): if resolver.require_hashes: logger.debug( 'Since it is already installed, we are trusting this ' 'package without checking its hash. To ensure a ' 'completely repeatable environment, install into an ' 'empty virtualenv.') abstract_dist = Installed(req) return abstract_dist
def run(self): if getattr(self.distribution, 'salt_installing_pywin32_windows', None) is None: print('This command is not meant to be called on it\'s own') exit(1) import platform # Detect if already installed, any version will work from pip.utils import get_installed_version if get_installed_version('pywin32') is not None: print('PyWin32 already installed') return if get_installed_version('pypiwin32') is not None: print('PyWin32 already installed') return # Install PyWin32 from Salt repo from pip.utils import call_subprocess from pip.utils.logging import indent_log platform_bits, _ = platform.architecture() call_arguments = ['pip', 'install', 'wheel'] if platform_bits == '64bit': if IS_PY3: call_arguments.append( 'https://repo.saltstack.com/windows/dependencies/64/pywin32-221-cp35-cp35m-win_amd64.whl' ) else: call_arguments.append( 'https://repo.saltstack.com/windows/dependencies/64/pywin32-221-cp27-cp27m-win_amd64.whl' ) else: if IS_PY3: call_arguments.append( 'https://repo.saltstack.com/windows/dependencies/32/pywin32-221-cp35-cp35m-win32.whl' ) else: call_arguments.append( 'https://repo.saltstack.com/windows/dependencies/32/pywin32-221-cp27-cp27m-win32.whl' ) with indent_log(): call_subprocess(call_arguments)
def install_editable(self, install_options, global_options=(), prefix=None): logger.info('Running setup.py develop for %s', self.name) if self.isolated: global_options = list(global_options) + ["--no-user-cfg"] if prefix: prefix_param = ['--prefix={0}'.format(prefix)] install_options = list(install_options) + prefix_param with indent_log(): # FIXME: should we do --install-headers here too? call_subprocess( [sys.executable, '-c', SETUPTOOLS_SHIM % self.setup_py] + list(global_options) + ['develop', '--no-deps'] + list(install_options), cwd=self.setup_py_dir, show_stdout=False) self.install_succeeded = True
def install_editable(self, install_options, global_options=()): logger.info('Running setup.py develop for %s', self.name) if self.isolated: global_options = list(global_options) + ["--no-user-cfg"] with indent_log(): # FIXME: should we do --install-headers here too? cwd = self.source_dir if self.editable_options and \ 'subdirectory' in self.editable_options: cwd = os.path.join(cwd, self.editable_options['subdirectory']) call_subprocess([ sys.executable, '-c', "import setuptools, tokenize; __file__=%r; exec(compile(" "getattr(tokenize, 'open', open)(__file__).read().replace" "('\\r\\n', '\\n'), __file__, 'exec'))" % self.setup_py ] + list(global_options) + ['develop', '--no-deps'] + list(install_options), cwd=cwd, show_stdout=False) self.install_succeeded = True
def _copy_dist_from_dir(link_path, location): """Copy distribution files in `link_path` to `location`. Invoked when user requests to install a local directory. E.g.: pip install . pip install ~/dev/git-repos/python-prompt-toolkit """ # Note: This is currently VERY SLOW if you have a lot of data in the # directory, because it copies everything with `shutil.copytree`. # What it should really do is build an sdist and install that. # See https://github.com/pypa/pip/issues/2195 if os.path.isdir(location): rmtree(location) # build an sdist setup_py = 'setup.py' sdist_args = [sys.executable] sdist_args.append('-c') sdist_args.append( "import setuptools, tokenize;__file__=%r;" "exec(compile(getattr(tokenize, 'open', open)(__file__).read()" ".replace('\\r\\n', '\\n'), __file__, 'exec'))" % setup_py) sdist_args.append('sdist') sdist_args += ['--dist-dir', location] logger.info('Running setup.py sdist for %s', link_path) with indent_log(): call_subprocess(sdist_args, cwd=link_path, show_stdout=False) # unpack sdist into `location` sdist = os.path.join(location, os.listdir(location)[0]) logger.info('Unpacking sdist %s into %s', sdist, location) unpack_file(sdist, location, content_type=None, link=None)
def print_results(hits, name_column_width=None, terminal_width=None): if not hits: return if name_column_width is None: name_column_width = max([ len(hit['name']) + len(hit.get('versions', ['-'])[-1]) for hit in hits ]) + 4 installed_packages = [p.project_name for p in pkg_resources.working_set] for hit in hits: name = hit['name'] summary = hit['summary'] or '' version = hit.get('versions', ['-'])[-1] if terminal_width is not None: # wrap and indent summary to fit terminal summary = textwrap.wrap( summary, terminal_width - name_column_width - 5, ) summary = ('\n' + ' ' * (name_column_width + 3)).join(summary) line = '%-*s - %s' % (name_column_width, '%s (%s)' % (name, version), summary) try: logger.info(line) if name in installed_packages: dist = pkg_resources.get_distribution(name) with indent_log(): latest = highest_version(hit['versions']) if dist.version == latest: logger.info('INSTALLED: %s (latest)', dist.version) else: logger.info('INSTALLED: %s', dist.version) logger.info('LATEST: %s', latest) except UnicodeEncodeError: pass
def _prepare_editable_requirement(self, req, resolver): """Prepare an editable requirement """ assert req.editable, "cannot prepare a non-editable req as editable" logger.info('Obtaining %s', req) with indent_log(): if resolver.require_hashes: raise InstallationError( 'The editable requirement %s cannot be installed when ' 'requiring hashes, because there is no single file to ' 'hash.' % req) req.ensure_has_source_dir(self.src_dir) req.update_editable(not self._download_should_save) abstract_dist = make_abstract_dist(req) abstract_dist.prep_for_dist() if self._download_should_save: req.archive(self.download_dir) req.check_if_exists() return abstract_dist
def cleanup_files(self): """Clean up files, remove builds.""" logger.debug('Cleaning up...') with indent_log(): for req in self.reqs_to_cleanup: req.remove_temporary_source()
def _prepare_file(self, finder, req_to_install, require_hashes=False, ignore_dependencies=False): """Prepare a single requirements file. :return: A list of additional InstallRequirements to also install. """ # Tell user what we are doing for this requirement: # obtain (editable), skipping, processing (local url), collecting # (remote url or package name) if req_to_install.constraint or req_to_install.prepared: return [] req_to_install.prepared = True # ###################### # # # print log messages # # # ###################### # if req_to_install.editable: logger.info('Obtaining %s', req_to_install) else: # satisfied_by is only evaluated by calling _check_skip_installed, # so it must be None here. assert req_to_install.satisfied_by is None if not self.ignore_installed: skip_reason = self._check_skip_installed( req_to_install, finder) if req_to_install.satisfied_by: assert skip_reason is not None, ( '_check_skip_installed returned None but ' 'req_to_install.satisfied_by is set to %r' % (req_to_install.satisfied_by,)) logger.info( 'Requirement %s: %s (%s)', skip_reason, req_to_install, req_to_install.satisfied_by.version) else: if (req_to_install.link and req_to_install.link.scheme == 'file'): path = url_to_path(req_to_install.link.url) logger.info('Processing %s', display_path(path)) else: logger.info('Collecting %s', req_to_install) with indent_log(): # ################################ # # # vcs update or unpack archive # # # ################################ # if req_to_install.editable: if require_hashes: raise InstallationError( 'The editable requirement %s cannot be installed when ' 'requiring hashes, because there is no single file to ' 'hash.' % req_to_install) req_to_install.ensure_has_source_dir(self.src_dir) req_to_install.update_editable(not self.is_download) abstract_dist = make_abstract_dist(req_to_install) abstract_dist.prep_for_dist() if self.is_download: req_to_install.archive(self.download_dir) req_to_install.check_if_exists() in_toto_verify_wrapper(req_to_install.source_dir, toto_verify=self.toto_verify, toto_default=self.toto_default) elif req_to_install.satisfied_by: if require_hashes: logger.debug( 'Since it is already installed, we are trusting this ' 'package without checking its hash. To ensure a ' 'completely repeatable environment, install into an ' 'empty virtualenv.') abstract_dist = Installed(req_to_install) else: # @@ if filesystem packages are not marked # editable in a req, a non deterministic error # occurs when the script attempts to unpack the # build directory req_to_install.ensure_has_source_dir(self.build_dir) # If a checkout exists, it's unwise to keep going. version # inconsistencies are logged later, but do not fail the # installation. # FIXME: this won't upgrade when there's an existing # package unpacked in `req_to_install.source_dir` if os.path.exists( os.path.join(req_to_install.source_dir, 'setup.py')): raise PreviousBuildDirError( "pip can't proceed with requirements '%s' due to a" " pre-existing build directory (%s). This is " "likely due to a previous installation that failed" ". pip is being responsible and not assuming it " "can delete this. Please delete it and try again." % (req_to_install, req_to_install.source_dir) ) req_to_install.populate_link( finder, self._is_upgrade_allowed(req_to_install), require_hashes ) # We can't hit this spot and have populate_link return None. # req_to_install.satisfied_by is None here (because we're # guarded) and upgrade has no impact except when satisfied_by # is not None. # Then inside find_requirement existing_applicable -> False # If no new versions are found, DistributionNotFound is raised, # otherwise a result is guaranteed. assert req_to_install.link link = req_to_install.link # Now that we have the real link, we can tell what kind of # requirements we have and raise some more informative errors # than otherwise. (For example, we can raise VcsHashUnsupported # for a VCS URL rather than HashMissing.) if require_hashes: # We could check these first 2 conditions inside # unpack_url and save repetition of conditions, but then # we would report less-useful error messages for # unhashable requirements, complaining that there's no # hash provided. if is_vcs_url(link): raise VcsHashUnsupported() elif is_file_url(link) and is_dir_url(link): raise DirectoryUrlHashUnsupported() if (not req_to_install.original_link and not req_to_install.is_pinned): # Unpinned packages are asking for trouble when a new # version is uploaded. This isn't a security check, but # it saves users a surprising hash mismatch in the # future. # # file:/// URLs aren't pinnable, so don't complain # about them not being pinned. raise HashUnpinned() hashes = req_to_install.hashes( trust_internet=not require_hashes) if require_hashes and not hashes: # Known-good hashes are missing for this requirement, so # shim it with a facade object that will provoke hash # computation and then raise a HashMissing exception # showing the user what the hash should be. hashes = MissingHashes() try: download_dir = self.download_dir # We always delete unpacked sdists after pip ran. autodelete_unpacked = True if req_to_install.link.is_wheel \ and self.wheel_download_dir: # when doing 'pip wheel` we download wheels to a # dedicated dir. download_dir = self.wheel_download_dir if req_to_install.link.is_wheel: if download_dir: # When downloading, we only unpack wheels to get # metadata. autodelete_unpacked = True else: # When installing a wheel, we use the unpacked # wheel. autodelete_unpacked = False #print "req.source_dir: %s, req_to_install.link: %s, download_dir: %s" % (req_to_install.source_dir, req_to_install.link, download_dir) unpack_url( req_to_install.link, req_to_install.source_dir, download_dir, autodelete_unpacked, session=self.session, hashes=hashes, toto_verify=self.toto_verify, toto_default=self.toto_default) except requests.HTTPError as exc: logger.critical( 'Could not install requirement %s because ' 'of error %s', req_to_install, exc, ) raise InstallationError( 'Could not install requirement %s because ' 'of HTTP error %s for URL %s' % (req_to_install, exc, req_to_install.link) ) abstract_dist = make_abstract_dist(req_to_install) abstract_dist.prep_for_dist() if self.is_download: # Make a .zip of the source_dir we already created. if req_to_install.link.scheme in vcs.all_schemes: req_to_install.archive(self.download_dir) # req_to_install.req is only avail after unpack for URL # pkgs repeat check_if_exists to uninstall-on-upgrade # (#14) if not self.ignore_installed: req_to_install.check_if_exists() if req_to_install.satisfied_by: if self.upgrade or self.ignore_installed: # don't uninstall conflict if user install and # conflict is not user install if not (self.use_user_site and not dist_in_usersite( req_to_install.satisfied_by)): req_to_install.conflicts_with = \ req_to_install.satisfied_by req_to_install.satisfied_by = None else: logger.info( 'Requirement already satisfied (use ' '--upgrade to upgrade): %s', req_to_install, ) # ###################### # # # parse dependencies # # # ###################### # dist = abstract_dist.dist(finder) try: check_dist_requires_python(dist) except UnsupportedPythonVersion as e: if self.ignore_requires_python: logger.warning(e.args[0]) else: req_to_install.remove_temporary_source() raise more_reqs = [] def add_req(subreq, extras_requested): sub_install_req = InstallRequirement( str(subreq), req_to_install, isolated=self.isolated, wheel_cache=self._wheel_cache, ) more_reqs.extend(self.add_requirement( sub_install_req, req_to_install.name, extras_requested=extras_requested)) # We add req_to_install before its dependencies, so that we # can refer to it when adding dependencies. if not self.has_requirement(req_to_install.name): # 'unnamed' requirements will get added here self.add_requirement(req_to_install, None) if not ignore_dependencies: if (req_to_install.extras): logger.debug( "Installing extra requirements: %r", ','.join(req_to_install.extras), ) missing_requested = sorted( set(req_to_install.extras) - set(dist.extras) ) for missing in missing_requested: logger.warning( '%s does not provide the extra \'%s\'', dist, missing ) available_requested = sorted( set(dist.extras) & set(req_to_install.extras) ) for subreq in dist.requires(available_requested): add_req(subreq, extras_requested=available_requested) # cleanup tmp src self.reqs_to_cleanup.append(req_to_install) if not req_to_install.editable and not req_to_install.satisfied_by: # XXX: --no-install leads this to report 'Successfully # downloaded' for only non-editable reqs, even though we took # action on them. self.successfully_downloaded.append(req_to_install) return more_reqs
def install(self, install_options, global_options=[], root=None, prefix=None): if self.editable: self.install_editable( install_options, global_options, prefix=prefix) return if self.is_wheel: version = pip.wheel.wheel_version(self.source_dir) pip.wheel.check_compatibility(version, self.name) self.move_wheel_files(self.source_dir, root=root, prefix=prefix) self.install_succeeded = True return # Extend the list of global and install options passed on to # the setup.py call with the ones from the requirements file. # Options specified in requirements file override those # specified on the command line, since the last option given # to setup.py is the one that is used. global_options += self.options.get('global_options', []) install_options += self.options.get('install_options', []) if self.isolated: global_options = list(global_options) + ["--no-user-cfg"] temp_location = tempfile.mkdtemp('-record', 'pip-') record_filename = os.path.join(temp_location, 'install-record.txt') try: install_args = self.get_install_args( global_options, record_filename, root, prefix) msg = 'Running setup.py install for %s' % (self.name,) with open_spinner(msg) as spinner: with indent_log(): call_subprocess( install_args + install_options, cwd=self.setup_py_dir, show_stdout=False, spinner=spinner, ) if not os.path.exists(record_filename): logger.debug('Record file %s not found', record_filename) return self.install_succeeded = True if self.as_egg: # there's no --always-unzip option we can pass to install # command so we unable to save the installed-files.txt return def prepend_root(path): if root is None or not os.path.isabs(path): return path else: return change_root(root, path) with open(record_filename) as f: for line in f: directory = os.path.dirname(line) if directory.endswith('.egg-info'): egg_info_dir = prepend_root(directory) break else: logger.warning( 'Could not find .egg-info directory in install record' ' for %s', self, ) # FIXME: put the record somewhere # FIXME: should this be an error? return new_lines = [] with open(record_filename) as f: for line in f: filename = line.strip() if os.path.isdir(filename): filename += os.path.sep new_lines.append( os.path.relpath( prepend_root(filename), egg_info_dir) ) inst_files_path = os.path.join(egg_info_dir, 'installed-files.txt') with open(inst_files_path, 'w') as f: f.write('\n'.join(new_lines) + '\n') finally: if os.path.exists(record_filename): os.remove(record_filename) rmtree(temp_location)
def _prepare_file(self, finder, req_to_install): """Prepare a single requirements files. :return: A list of addition InstallRequirements to also install. """ # Tell user what we are doing for this requirement: # obtain (editable), skipping, processing (local url), collecting # (remote url or package name) if req_to_install.constraint or req_to_install.prepared: return [] req_to_install.prepared = True if req_to_install.editable: logger.info('Obtaining %s', req_to_install) else: # satisfied_by is only evaluated by calling _check_skip_installed, # so it must be None here. assert req_to_install.satisfied_by is None if not self.ignore_installed: skip_reason = self._check_skip_installed( req_to_install, finder) if req_to_install.satisfied_by: assert skip_reason is not None, ( '_check_skip_installed returned None but ' 'req_to_install.satisfied_by is set to %r' % (req_to_install.satisfied_by,)) logger.info( 'Requirement already %s: %s', skip_reason, req_to_install) else: if (req_to_install.link and req_to_install.link.scheme == 'file'): path = url_to_path(req_to_install.link.url) logger.info('Processing %s', display_path(path)) else: logger.info('Collecting %s', req_to_install) with indent_log(): # ################################ # # # vcs update or unpack archive # # # ################################ # if req_to_install.editable: req_to_install.ensure_has_source_dir(self.src_dir) req_to_install.update_editable(not self.is_download) abstract_dist = make_abstract_dist(req_to_install) abstract_dist.prep_for_dist() if self.is_download: req_to_install.archive(self.download_dir) elif req_to_install.satisfied_by: abstract_dist = Installed(req_to_install) else: # @@ if filesystem packages are not marked # editable in a req, a non deterministic error # occurs when the script attempts to unpack the # build directory req_to_install.ensure_has_source_dir(self.build_dir) # If a checkout exists, it's unwise to keep going. version # inconsistencies are logged later, but do not fail the # installation. # FIXME: this won't upgrade when there's an existing # package unpacked in `req_to_install.source_dir` if os.path.exists( os.path.join(req_to_install.source_dir, 'setup.py')): raise PreviousBuildDirError( "pip can't proceed with requirements '%s' due to a" " pre-existing build directory (%s). This is " "likely due to a previous installation that failed" ". pip is being responsible and not assuming it " "can delete this. Please delete it and try again." % (req_to_install, req_to_install.source_dir) ) req_to_install.populate_link(finder, self.upgrade) # We can't hit this spot and have populate_link return None. # req_to_install.satisfied_by is None here (because we're # guarded) and upgrade has no impact except when satisfied_by # is not None. # Then inside find_requirement existing_applicable -> False # If no new versions are found, DistributionNotFound is raised, # otherwise a result is guaranteed. assert req_to_install.link try: download_dir = self.download_dir # We always delete unpacked sdists after pip ran. autodelete_unpacked = True if req_to_install.link.is_wheel \ and self.wheel_download_dir: # when doing 'pip wheel` we download wheels to a # dedicated dir. download_dir = self.wheel_download_dir if req_to_install.link.is_wheel: if download_dir: # When downloading, we only unpack wheels to get # metadata. autodelete_unpacked = True else: # When installing a wheel, we use the unpacked # wheel. autodelete_unpacked = False unpack_url( req_to_install.link, req_to_install.source_dir, download_dir, autodelete_unpacked, session=self.session) except requests.HTTPError as exc: logger.critical( 'Could not install requirement %s because ' 'of error %s', req_to_install, exc, ) raise InstallationError( 'Could not install requirement %s because ' 'of HTTP error %s for URL %s' % (req_to_install, exc, req_to_install.link) ) abstract_dist = make_abstract_dist(req_to_install) abstract_dist.prep_for_dist() if self.is_download: # Make a .zip of the source_dir we already created. if req_to_install.link.scheme in vcs.all_schemes: req_to_install.archive(self.download_dir) # req_to_install.req is only avail after unpack for URL # pkgs repeat check_if_exists to uninstall-on-upgrade # (#14) if not self.ignore_installed: req_to_install.check_if_exists() if req_to_install.satisfied_by: if self.upgrade or self.ignore_installed: # don't uninstall conflict if user install and # conflict is not user install if not (self.use_user_site and not dist_in_usersite( req_to_install.satisfied_by)): req_to_install.conflicts_with = \ req_to_install.satisfied_by req_to_install.satisfied_by = None else: logger.info( 'Requirement already satisfied (use ' '--upgrade to upgrade): %s', req_to_install, ) # ###################### # # # parse dependencies # # # ###################### # dist = abstract_dist.dist(finder) more_reqs = [] def add_req(subreq): sub_install_req = InstallRequirement( str(subreq), req_to_install, isolated=self.isolated, wheel_cache=self._wheel_cache, ) more_reqs.extend(self.add_requirement( sub_install_req, req_to_install.name)) # We add req_to_install before its dependencies, so that we # can refer to it when adding dependencies. if not self.has_requirement(req_to_install.name): # 'unnamed' requirements will get added here self.add_requirement(req_to_install, None) if not self.ignore_dependencies: if (req_to_install.extras): logger.debug( "Installing extra requirements: %r", ','.join(req_to_install.extras), ) missing_requested = sorted( set(req_to_install.extras) - set(dist.extras) ) for missing in missing_requested: logger.warning( '%s does not provide the extra \'%s\'', dist, missing ) available_requested = sorted( set(dist.extras) & set(req_to_install.extras) ) for subreq in dist.requires(available_requested): add_req(subreq) # cleanup tmp src self.reqs_to_cleanup.append(req_to_install) if not req_to_install.editable and not req_to_install.satisfied_by: # XXX: --no-install leads this to report 'Successfully # downloaded' for only non-editable reqs, even though we took # action on them. self.successfully_downloaded.append(req_to_install) return more_reqs
def find_all_candidates(self, project_name): """Find all available InstallationCandidate for project_name This checks index_urls, find_links and dependency_links. All versions found are returned as an InstallationCandidate list. See _link_package_versions for details on which files are accepted """ index_locations = self._get_index_urls_locations(project_name) index_file_loc, index_url_loc = self._sort_locations(index_locations) fl_file_loc, fl_url_loc = self._sort_locations( self.find_links, expand_dir=True) dep_file_loc, dep_url_loc = self._sort_locations(self.dependency_links) file_locations = ( Link(url) for url in itertools.chain( index_file_loc, fl_file_loc, dep_file_loc) ) # We trust every url that the user has given us whether it was given # via --index-url or --find-links # We explicitly do not trust links that came from dependency_links # We want to filter out any thing which does not have a secure origin. url_locations = [ link for link in itertools.chain( (Link(url) for url in index_url_loc), (Link(url) for url in fl_url_loc), (Link(url) for url in dep_url_loc), ) if self._validate_secure_origin(logger, link) ] logger.debug('%d location(s) to search for versions of %s:', len(url_locations), project_name) for location in url_locations: logger.debug('* %s', location) canonical_name = canonicalize_name(project_name) formats = fmt_ctl_formats(self.format_control, canonical_name) search = Search(project_name, canonical_name, formats) find_links_versions = self._package_versions( # We trust every directly linked archive in find_links (Link(url, '-f') for url in self.find_links), search ) page_versions = [] for page in self._get_pages(url_locations, project_name): logger.debug('Analyzing links from page %s', page.url) with indent_log(): page_versions.extend( self._package_versions(page.links, search) ) dependency_versions = self._package_versions( (Link(url) for url in self.dependency_links), search ) if dependency_versions: logger.debug( 'dependency_links found: %s', ', '.join([ version.location.url for version in dependency_versions ]) ) file_versions = self._package_versions(file_locations, search) if file_versions: file_versions.sort(reverse=True) logger.debug( 'Local files found: %s', ', '.join([ url_to_path(candidate.location.url) for candidate in file_versions ]) ) # This is an intentional priority ordering return ( file_versions + find_links_versions + page_versions + dependency_versions )
def _resolve_one(self, requirement_set, req_to_install): """Prepare a single requirements file. :return: A list of additional InstallRequirements to also install. """ # Tell user what we are doing for this requirement: # obtain (editable), skipping, processing (local url), collecting # (remote url or package name) if req_to_install.constraint or req_to_install.prepared: return [] req_to_install.prepared = True abstract_dist = self.preparer.prepare_requirement(req_to_install, self) # register tmp src for cleanup in case something goes wrong requirement_set.reqs_to_cleanup.append(req_to_install) # Parse and return dependencies dist = abstract_dist.dist(self.finder) try: check_dist_requires_python(dist) except UnsupportedPythonVersion as err: if self.ignore_requires_python: logger.warning(err.args[0]) else: raise more_reqs = [] def add_req(subreq, extras_requested): sub_install_req = InstallRequirement.from_req( str(subreq), req_to_install, isolated=self.isolated, wheel_cache=self.wheel_cache, ) more_reqs.extend( requirement_set.add_requirement( sub_install_req, req_to_install.name, extras_requested=extras_requested)) with indent_log(): # We add req_to_install before its dependencies, so that we # can refer to it when adding dependencies. if not requirement_set.has_requirement(req_to_install.name): # 'unnamed' requirements will get added here requirement_set.add_requirement(req_to_install, None) if not self.ignore_dependencies: if req_to_install.extras: logger.debug( "Installing extra requirements: %r", ','.join(req_to_install.extras), ) missing_requested = sorted( set(req_to_install.extras) - set(dist.extras)) for missing in missing_requested: logger.warning('%s does not provide the extra \'%s\'', dist, missing) available_requested = sorted( set(dist.extras) & set(req_to_install.extras)) for subreq in dist.requires(available_requested): add_req(subreq, extras_requested=available_requested) if not req_to_install.editable and not req_to_install.satisfied_by: # XXX: --no-install leads this to report 'Successfully # downloaded' for only non-editable reqs, even though we took # action on them. requirement_set.successfully_downloaded.append(req_to_install) return more_reqs
def install(self, install_options, global_options=[], root=None, prefix=None): if self.editable: self.install_editable(install_options, global_options, prefix=prefix) return if self.is_wheel: version = pip.wheel.wheel_version(self.source_dir) pip.wheel.check_compatibility(version, self.name) self.move_wheel_files(self.source_dir, root=root, prefix=prefix) self.install_succeeded = True return # Extend the list of global and install options passed on to # the setup.py call with the ones from the requirements file. # Options specified in requirements file override those # specified on the command line, since the last option given # to setup.py is the one that is used. global_options += self.options.get('global_options', []) install_options += self.options.get('install_options', []) if self.isolated: global_options = list(global_options) + ["--no-user-cfg"] with TempDirectory(kind="record") as temp_dir: record_filename = os.path.join(temp_dir.path, 'install-record.txt') install_args = self.get_install_args(global_options, record_filename, root, prefix) msg = 'Running setup.py install for %s' % (self.name, ) with open_spinner(msg) as spinner: with indent_log(): call_subprocess( install_args + install_options, cwd=self.setup_py_dir, show_stdout=False, spinner=spinner, ) if not os.path.exists(record_filename): logger.debug('Record file %s not found', record_filename) return self.install_succeeded = True def prepend_root(path): if root is None or not os.path.isabs(path): return path else: return change_root(root, path) with open(record_filename) as f: for line in f: directory = os.path.dirname(line) if directory.endswith('.egg-info'): egg_info_dir = prepend_root(directory) break else: logger.warning( 'Could not find .egg-info directory in install record' ' for %s', self, ) # FIXME: put the record somewhere # FIXME: should this be an error? return new_lines = [] with open(record_filename) as f: for line in f: filename = line.strip() if os.path.isdir(filename): filename += os.path.sep new_lines.append( os.path.relpath(prepend_root(filename), egg_info_dir)) ensure_dir(egg_info_dir) inst_files_path = os.path.join(egg_info_dir, 'installed-files.txt') with open(inst_files_path, 'w') as f: f.write('\n'.join(new_lines) + '\n')
def _prepare_file(self, finder, req_to_install): """Prepare a single requirements files. :return: A list of addition InstallRequirements to also install. """ # Tell user what we are doing for this requirement: # obtain (editable), skipping, processing (local url), collecting # (remote url or package name) if req_to_install.constraint or req_to_install.prepared: return [] req_to_install.prepared = True if req_to_install.editable: logger.info('Obtaining %s', req_to_install) else: # satisfied_by is only evaluated by calling _check_skip_installed, # so it must be None here. assert req_to_install.satisfied_by is None if not self.ignore_installed: skip_reason = self._check_skip_installed( req_to_install, finder) if req_to_install.satisfied_by: assert skip_reason is not None, ( '_check_skip_installed returned None but ' 'req_to_install.satisfied_by is set to %r' % (req_to_install.satisfied_by, )) logger.info('Requirement already %s: %s', skip_reason, req_to_install) else: if (req_to_install.link and req_to_install.link.scheme == 'file'): path = url_to_path(req_to_install.link.url) logger.info('Processing %s', display_path(path)) else: logger.info('Collecting %s', req_to_install) with indent_log(): # ################################ # # # vcs update or unpack archive # # # ################################ # if req_to_install.editable: req_to_install.ensure_has_source_dir(self.src_dir) req_to_install.update_editable(not self.is_download) abstract_dist = make_abstract_dist(req_to_install) abstract_dist.prep_for_dist() if self.is_download: req_to_install.archive(self.download_dir) elif req_to_install.satisfied_by: abstract_dist = Installed(req_to_install) else: # @@ if filesystem packages are not marked # editable in a req, a non deterministic error # occurs when the script attempts to unpack the # build directory req_to_install.ensure_has_source_dir(self.build_dir) # If a checkout exists, it's unwise to keep going. version # inconsistencies are logged later, but do not fail the # installation. # FIXME: this won't upgrade when there's an existing # package unpacked in `req_to_install.source_dir` if os.path.exists( os.path.join(req_to_install.source_dir, 'setup.py')): raise PreviousBuildDirError( "pip can't proceed with requirements '%s' due to a" " pre-existing build directory (%s). This is " "likely due to a previous installation that failed" ". pip is being responsible and not assuming it " "can delete this. Please delete it and try again." % (req_to_install, req_to_install.source_dir)) req_to_install.populate_link(finder, self.upgrade) # We can't hit this spot and have populate_link return None. # req_to_install.satisfied_by is None here (because we're # guarded) and upgrade has no impact except when satisfied_by # is not None. # Then inside find_requirement existing_applicable -> False # If no new versions are found, DistributionNotFound is raised, # otherwise a result is guaranteed. assert req_to_install.link try: download_dir = self.download_dir # We always delete unpacked sdists after pip ran. autodelete_unpacked = True if req_to_install.link.is_wheel \ and self.wheel_download_dir: # when doing 'pip wheel` we download wheels to a # dedicated dir. download_dir = self.wheel_download_dir if req_to_install.link.is_wheel: if download_dir: # When downloading, we only unpack wheels to get # metadata. autodelete_unpacked = True else: # When installing a wheel, we use the unpacked # wheel. autodelete_unpacked = False unpack_url(req_to_install.link, req_to_install.source_dir, download_dir, autodelete_unpacked, session=self.session) except requests.HTTPError as exc: logger.critical( 'Could not install requirement %s because ' 'of error %s', req_to_install, exc, ) raise InstallationError( 'Could not install requirement %s because ' 'of HTTP error %s for URL %s' % (req_to_install, exc, req_to_install.link)) abstract_dist = make_abstract_dist(req_to_install) abstract_dist.prep_for_dist() if self.is_download: # Make a .zip of the source_dir we already created. if req_to_install.link.scheme in vcs.all_schemes: req_to_install.archive(self.download_dir) # req_to_install.req is only avail after unpack for URL # pkgs repeat check_if_exists to uninstall-on-upgrade # (#14) if not self.ignore_installed: req_to_install.check_if_exists() if req_to_install.satisfied_by: if self.upgrade or self.ignore_installed: # don't uninstall conflict if user install and # conflict is not user install if not (self.use_user_site and not dist_in_usersite( req_to_install.satisfied_by)): req_to_install.conflicts_with = \ req_to_install.satisfied_by req_to_install.satisfied_by = None else: logger.info( 'Requirement already satisfied (use ' '--upgrade to upgrade): %s', req_to_install, ) # ###################### # # # parse dependencies # # # ###################### # dist = abstract_dist.dist(finder) more_reqs = [] def add_req(subreq): sub_install_req = InstallRequirement( str(subreq), req_to_install, isolated=self.isolated, wheel_cache=self._wheel_cache, ) more_reqs.extend( self.add_requirement(sub_install_req, req_to_install.name)) # We add req_to_install before its dependencies, so that we # can refer to it when adding dependencies. if not self.has_requirement(req_to_install.name): # 'unnamed' requirements will get added here self.add_requirement(req_to_install, None) if not self.ignore_dependencies: if (req_to_install.extras): logger.debug( "Installing extra requirements: %r", ','.join(req_to_install.extras), ) missing_requested = sorted( set(req_to_install.extras) - set(dist.extras)) for missing in missing_requested: logger.warning('%s does not provide the extra \'%s\'', dist, missing) available_requested = sorted( set(dist.extras) & set(req_to_install.extras)) for subreq in dist.requires(available_requested): add_req(subreq) # cleanup tmp src self.reqs_to_cleanup.append(req_to_install) if not req_to_install.editable and not req_to_install.satisfied_by: # XXX: --no-install leads this to report 'Successfully # downloaded' for only non-editable reqs, even though we took # action on them. self.successfully_downloaded.append(req_to_install) return more_reqs
def install(self, install_options, global_options=[], root=None, prefix=None): if self.editable: self.install_editable(install_options, global_options, prefix=prefix) return if self.is_wheel: version = pip.wheel.wheel_version(self.source_dir) pip.wheel.check_compatibility(version, self.name) self.move_wheel_files(self.source_dir, root=root, prefix=prefix) self.install_succeeded = True return # Extend the list of global and install options passed on to # the setup.py call with the ones from the requirements file. # Options specified in requirements file override those # specified on the command line, since the last option given # to setup.py is the one that is used. global_options += self.options.get('global_options', []) install_options += self.options.get('install_options', []) if self.isolated: global_options = list(global_options) + ["--no-user-cfg"] temp_location = tempfile.mkdtemp('-record', 'pip-') record_filename = os.path.join(temp_location, 'install-record.txt') try: install_args = [sys.executable, "-u"] install_args.append('-c') install_args.append(SETUPTOOLS_SHIM % self.setup_py) install_args += list(global_options) + \ ['install', '--record', record_filename] if not self.as_egg: install_args += ['--single-version-externally-managed'] if root is not None: install_args += ['--root', root] if prefix is not None: install_args += ['--prefix', prefix] if self.pycompile: install_args += ["--compile"] else: install_args += ["--no-compile"] if running_under_virtualenv(): py_ver_str = 'python' + sysconfig.get_python_version() install_args += [ '--install-headers', os.path.join(sys.prefix, 'include', 'site', py_ver_str, self.name) ] msg = 'Running setup.py install for %s' % (self.name, ) with open_spinner(msg) as spinner: with indent_log(): call_subprocess( install_args + install_options, cwd=self.source_dir, show_stdout=False, spinner=spinner, ) if not os.path.exists(record_filename): logger.debug('Record file %s not found', record_filename) return self.install_succeeded = True if self.as_egg: # there's no --always-unzip option we can pass to install # command so we unable to save the installed-files.txt return def prepend_root(path): if root is None or not os.path.isabs(path): return path else: return change_root(root, path) with open(record_filename) as f: for line in f: directory = os.path.dirname(line) if directory.endswith('.egg-info'): egg_info_dir = prepend_root(directory) break else: logger.warning( 'Could not find .egg-info directory in install record' ' for %s', self, ) # FIXME: put the record somewhere # FIXME: should this be an error? return new_lines = [] with open(record_filename) as f: for line in f: filename = line.strip() if os.path.isdir(filename): filename += os.path.sep new_lines.append( os.path.relpath(prepend_root(filename), egg_info_dir)) inst_files_path = os.path.join(egg_info_dir, 'installed-files.txt') with open(inst_files_path, 'w') as f: f.write('\n'.join(new_lines) + '\n') finally: if os.path.exists(record_filename): os.remove(record_filename) rmtree(temp_location)
def build(self, autobuilding=False): """Build wheels. :param unpack: If True, replace the sdist we built from the with the newly built wheel, in preparation for installation. :return: True if all the wheels built correctly. """ assert self._wheel_dir or (autobuilding and self._cache_root) # unpack sdists and constructs req set self.requirement_set.prepare_files(self.finder) reqset = self.requirement_set.requirements.values() buildset = [] for req in reqset: if req.constraint: continue if req.is_wheel: if not autobuilding: logger.info('Skipping %s, due to already being wheel.', req.name) elif req.editable: if not autobuilding: logger.info( 'Skipping bdist_wheel for %s, due to being editable', req.name) elif autobuilding and req.link and not req.link.is_artifact: pass elif autobuilding and not req.source_dir: pass else: if autobuilding: link = req.link base, ext = link.splitext() if pip.index.egg_info_matches(base, None, link) is None: # Doesn't look like a package - don't autobuild a wheel # because we'll have no way to lookup the result sanely continue if "binary" not in pip.index.fmt_ctl_formats( self.finder.format_control, pkg_resources.safe_name(req.name).lower()): logger.info( "Skipping bdist_wheel for %s, due to binaries " "being disabled for it.", req.name) continue buildset.append(req) if not buildset: return True # Build the wheels. logger.info( 'Building wheels for collected packages: %s', ', '.join([req.name for req in buildset]), ) with indent_log(): build_success, build_failure = [], [] for req in buildset: if autobuilding: output_dir = _cache_for_link(self._cache_root, req.link) ensure_dir(output_dir) else: output_dir = self._wheel_dir wheel_file = self._build_one(req, output_dir) if wheel_file: build_success.append(req) if autobuilding: # XXX: This is mildly duplicative with prepare_files, # but not close enough to pull out to a single common # method. # The code below assumes temporary source dirs - # prevent it doing bad things. if req.source_dir and not os.path.exists( os.path.join(req.source_dir, PIP_DELETE_MARKER_FILENAME)): raise AssertionError( "bad source dir - missing marker") # Delete the source we built the wheel from req.remove_temporary_source() # set the build directory again - name is known from # the work prepare_files did. req.source_dir = req.build_location( self.requirement_set.build_dir) # Update the link for this. req.link = pip.index.Link(path_to_url(wheel_file), trusted=True) assert req.link.is_wheel # extract the wheel into the dir unpack_url(req.link, req.source_dir, None, False, session=self.requirement_set.session) else: build_failure.append(req) # notify success/failure if build_success: logger.info( 'Successfully built %s', ' '.join([req.name for req in build_success]), ) if build_failure: logger.info( 'Failed to build %s', ' '.join([req.name for req in build_failure]), ) # Return True if all builds were successful return len(build_failure) == 0
def _prepare_linked_requirement(self, req, resolver): """Prepare a requirement that would be obtained from req.link """ # TODO: Breakup into smaller functions if req.link and req.link.scheme == 'file': path = url_to_path(req.link.url) logger.info('Processing %s', display_path(path)) else: logger.info('Collecting %s', req) with indent_log(): # @@ if filesystem packages are not marked # editable in a req, a non deterministic error # occurs when the script attempts to unpack the # build directory req.ensure_has_source_dir(self.build_dir) # If a checkout exists, it's unwise to keep going. version # inconsistencies are logged later, but do not fail the # installation. # FIXME: this won't upgrade when there's an existing # package unpacked in `req.source_dir` # package unpacked in `req.source_dir` if os.path.exists(os.path.join(req.source_dir, 'setup.py')): raise PreviousBuildDirError( "pip can't proceed with requirements '%s' due to a" " pre-existing build directory (%s). This is " "likely due to a previous installation that failed" ". pip is being responsible and not assuming it " "can delete this. Please delete it and try again." % (req, req.source_dir)) req.populate_link(resolver.finder, resolver._is_upgrade_allowed(req), resolver.require_hashes) # We can't hit this spot and have populate_link return None. # req.satisfied_by is None here (because we're # guarded) and upgrade has no impact except when satisfied_by # is not None. # Then inside find_requirement existing_applicable -> False # If no new versions are found, DistributionNotFound is raised, # otherwise a result is guaranteed. assert req.link link = req.link # Now that we have the real link, we can tell what kind of # requirements we have and raise some more informative errors # than otherwise. (For example, we can raise VcsHashUnsupported # for a VCS URL rather than HashMissing.) if resolver.require_hashes: # We could check these first 2 conditions inside # unpack_url and save repetition of conditions, but then # we would report less-useful error messages for # unhashable requirements, complaining that there's no # hash provided. if is_vcs_url(link): raise VcsHashUnsupported() elif is_file_url(link) and is_dir_url(link): raise DirectoryUrlHashUnsupported() if not req.original_link and not req.is_pinned: # Unpinned packages are asking for trouble when a new # version is uploaded. This isn't a security check, but # it saves users a surprising hash mismatch in the # future. # # file:/// URLs aren't pinnable, so don't complain # about them not being pinned. raise HashUnpinned() hashes = req.hashes(trust_internet=not resolver.require_hashes) if resolver.require_hashes and not hashes: # Known-good hashes are missing for this requirement, so # shim it with a facade object that will provoke hash # computation and then raise a HashMissing exception # showing the user what the hash should be. hashes = MissingHashes() try: download_dir = self.download_dir # We always delete unpacked sdists after pip ran. autodelete_unpacked = True if req.link.is_wheel and self.wheel_download_dir: # when doing 'pip wheel` we download wheels to a # dedicated dir. download_dir = self.wheel_download_dir if req.link.is_wheel: if download_dir: # When downloading, we only unpack wheels to get # metadata. autodelete_unpacked = True else: # When installing a wheel, we use the unpacked # wheel. autodelete_unpacked = False unpack_url(req.link, req.source_dir, download_dir, autodelete_unpacked, session=resolver.session, hashes=hashes, progress_bar=self.progress_bar) except requests.HTTPError as exc: logger.critical( 'Could not install requirement %s because of error %s', req, exc, ) raise InstallationError( 'Could not install requirement %s because of HTTP ' 'error %s for URL %s' % (req, exc, req.link)) abstract_dist = make_abstract_dist(req) abstract_dist.prep_for_dist() if self._download_should_save: # Make a .zip of the source_dir we already created. if req.link.scheme in vcs.all_schemes: req.archive(self.download_dir) # req.req is only avail after unpack for URL # pkgs repeat check_if_exists to uninstall-on-upgrade # (#14) if not resolver.ignore_installed: req.check_if_exists() if req.satisfied_by: should_modify = (resolver.upgrade_strategy != "to-satisfy-only" or resolver.ignore_installed) if should_modify: resolver._set_req_to_reinstall(req) else: logger.info( 'Requirement already satisfied (use ' '--upgrade to upgrade): %s', req, ) return abstract_dist
match = _svn_revision_re.search(output) if not match: logger.warning( 'Cannot determine revision of svn checkout %s', display_path(location), ) logger.debug('Output that cannot be parsed: \n%s', output) return url, None return url, match.group(1) def export(self, location): """Export the svn repository at the url to the destination location""" url, rev = self.get_url_rev() rev_options = get_rev_options(url, rev) logger.info('Exporting svn repository %s to %s', url, location) with indent_log(): if os.path.exists(location): # Subversion doesn't like to check out over an existing # directory --force fixes this, but was only added in svn 1.5 rmtree(location) <<<<<<< HEAD self.run_command( ['export'] + rev_options + [url, location], filter_stdout=self._filter, show_stdout=False) def switch(self, dest, url, rev_options): self.run_command(['switch'] + rev_options + [url, dest]) def update(self, dest, rev_options): self.run_command(['update'] + rev_options + [dest]) =======
def build(self, autobuilding=False): """Build wheels. :param unpack: If True, replace the sdist we built from the with the newly built wheel, in preparation for installation. :return: True if all the wheels built correctly. """ assert self._wheel_dir or (autobuilding and self._cache_root) # unpack sdists and constructs req set self.requirement_set.prepare_files(self.finder) reqset = self.requirement_set.requirements.values() buildset = [] for req in reqset: if req.constraint: continue if req.is_wheel: if not autobuilding: logger.info( 'Skipping %s, due to already being wheel.', req.name) elif req.editable: if not autobuilding: logger.info( 'Skipping bdist_wheel for %s, due to being editable', req.name) elif autobuilding and req.link and not req.link.is_artifact: pass elif autobuilding and not req.source_dir: pass else: if autobuilding: link = req.link base, ext = link.splitext() if pip.index.egg_info_matches(base, None, link) is None: # Doesn't look like a package - don't autobuild a wheel # because we'll have no way to lookup the result sanely continue if "binary" not in pip.index.fmt_ctl_formats( self.finder.format_control, canonicalize_name(req.name)): logger.info( "Skipping bdist_wheel for %s, due to binaries " "being disabled for it.", req.name) continue buildset.append(req) if not buildset: return True # Build the wheels. logger.info( 'Building wheels for collected packages: %s', ', '.join([req.name for req in buildset]), ) with indent_log(): build_success, build_failure = [], [] for req in buildset: if autobuilding: output_dir = _cache_for_link(self._cache_root, req.link) try: ensure_dir(output_dir) except OSError as e: logger.warn("Building wheel for %s failed: %s", req.name, e) build_failure.append(req) continue else: output_dir = self._wheel_dir wheel_file = self._build_one(req, output_dir) if wheel_file: build_success.append(req) if autobuilding: # XXX: This is mildly duplicative with prepare_files, # but not close enough to pull out to a single common # method. # The code below assumes temporary source dirs - # prevent it doing bad things. if req.source_dir and not os.path.exists(os.path.join( req.source_dir, PIP_DELETE_MARKER_FILENAME)): raise AssertionError( "bad source dir - missing marker") # Delete the source we built the wheel from req.remove_temporary_source() # set the build directory again - name is known from # the work prepare_files did. req.source_dir = req.build_location( self.requirement_set.build_dir) # Update the link for this. req.link = pip.index.Link( path_to_url(wheel_file)) assert req.link.is_wheel # extract the wheel into the dir unpack_url( req.link, req.source_dir, None, False, session=self.requirement_set.session) else: build_failure.append(req) # notify success/failure if build_success: logger.info( 'Successfully built %s', ' '.join([req.name for req in build_success]), ) if build_failure: logger.info( 'Failed to build %s', ' '.join([req.name for req in build_failure]), ) # Return True if all builds were successful return len(build_failure) == 0
def run_egg_info(self): assert self.source_dir if self.name: logger.debug( 'Running setup.py (path:%s) egg_info for package %s', self.setup_py, self.name, ) else: logger.debug( 'Running setup.py (path:%s) egg_info for package from %s', self.setup_py, self.link, ) with indent_log(): script = SETUPTOOLS_SHIM % self.setup_py base_cmd = [sys.executable, '-c', script] if self.isolated: base_cmd += ["--no-user-cfg"] egg_info_cmd = base_cmd + ['egg_info'] # We can't put the .egg-info files at the root, because then the # source code will be mistaken for an installed egg, causing # problems if self.editable: egg_base_option = [] else: egg_info_dir = os.path.join(self.source_dir, 'pip-egg-info') ensure_dir(egg_info_dir) egg_base_option = ['--egg-base', 'pip-egg-info'] cwd = self.source_dir if self.editable_options and \ 'subdirectory' in self.editable_options: cwd = os.path.join(cwd, self.editable_options['subdirectory']) call_subprocess(egg_info_cmd + egg_base_option, cwd=cwd, show_stdout=False, command_level=logging.DEBUG, command_desc='python setup.py egg_info') if not self.req: if isinstance( pkg_resources.parse_version(self.pkg_info()["Version"]), Version): op = "==" else: op = "===" self.req = pkg_resources.Requirement.parse("".join([ self.pkg_info()["Name"], op, self.pkg_info()["Version"], ])) self._correct_build_location() else: metadata_name = canonicalize_name(self.pkg_info()["Name"]) if canonicalize_name(self.req.project_name) != metadata_name: logger.warning( 'Running setup.py (path:%s) egg_info for package %s ' 'produced metadata for project name %s. Fix your ' '#egg=%s fragments.', self.setup_py, self.name, metadata_name, self.name) self.req = pkg_resources.Requirement.parse(metadata_name)
def prepare_files(self, finder): """ Prepare process. Create temp directories, download and/or unpack files. """ from pip.index import Link unnamed = list(self.unnamed_requirements) reqs = list(self.requirements.values()) while reqs or unnamed: if unnamed: req_to_install = unnamed.pop(0) else: req_to_install = reqs.pop(0) install = True best_installed = False not_found = None # ############################################# # # # Search for archive to fulfill requirement # # # ############################################# # if not self.ignore_installed and not req_to_install.editable: req_to_install.check_if_exists() if req_to_install.satisfied_by: if self.upgrade: if not self.force_reinstall and not req_to_install.url: try: url = finder.find_requirement( req_to_install, self.upgrade) except BestVersionAlreadyInstalled: best_installed = True install = False except DistributionNotFound as exc: not_found = exc else: # Avoid the need to call find_requirement again req_to_install.url = url.url if not best_installed: # don't uninstall conflict if user install and # conflict is not user install if not (self.use_user_site and not dist_in_usersite( req_to_install.satisfied_by )): req_to_install.conflicts_with = \ req_to_install.satisfied_by req_to_install.satisfied_by = None else: install = False if req_to_install.satisfied_by: if best_installed: logger.info( 'Requirement already up-to-date: %s', req_to_install, ) else: logger.info( 'Requirement already satisfied (use --upgrade to ' 'upgrade): %s', req_to_install, ) if req_to_install.editable: logger.info('Obtaining %s', req_to_install) elif install: if (req_to_install.url and req_to_install.url.lower().startswith('file:')): path = url_to_path(req_to_install.url) logger.info('Processing %s', display_path(path)) else: logger.info('Collecting %s', req_to_install) with indent_log(): # ################################ # # # vcs update or unpack archive # # # ################################ # is_wheel = False if req_to_install.editable: if req_to_install.source_dir is None: location = req_to_install.build_location(self.src_dir) req_to_install.source_dir = location else: location = req_to_install.source_dir if not os.path.exists(self.build_dir): _make_build_dir(self.build_dir) req_to_install.update_editable(not self.is_download) if self.is_download: req_to_install.run_egg_info() req_to_install.archive(self.download_dir) else: req_to_install.run_egg_info() elif install: # @@ if filesystem packages are not marked # editable in a req, a non deterministic error # occurs when the script attempts to unpack the # build directory # NB: This call can result in the creation of a temporary # build directory location = req_to_install.build_location( self.build_dir, ) unpack = True url = None # If a checkout exists, it's unwise to keep going. version # inconsistencies are logged later, but do not fail the # installation. if os.path.exists(os.path.join(location, 'setup.py')): raise PreviousBuildDirError( "pip can't proceed with requirements '%s' due to a" " pre-existing build directory (%s). This is " "likely due to a previous installation that failed" ". pip is being responsible and not assuming it " "can delete this. Please delete it and try again." % (req_to_install, location) ) else: # FIXME: this won't upgrade when there's an existing # package unpacked in `location` if req_to_install.url is None: if not_found: raise not_found url = finder.find_requirement( req_to_install, upgrade=self.upgrade, ) else: # FIXME: should req_to_install.url already be a # link? url = Link(req_to_install.url) assert url if url: try: if ( url.filename.endswith(wheel_ext) and self.wheel_download_dir ): # when doing 'pip wheel` download_dir = self.wheel_download_dir do_download = True else: download_dir = self.download_dir do_download = self.is_download unpack_url( url, location, download_dir, do_download, session=self.session, ) except requests.HTTPError as exc: logger.critical( 'Could not install requirement %s because ' 'of error %s', req_to_install, exc, ) raise InstallationError( 'Could not install requirement %s because ' 'of HTTP error %s for URL %s' % (req_to_install, exc, url) ) else: unpack = False if unpack: is_wheel = url and url.filename.endswith(wheel_ext) if self.is_download: req_to_install.source_dir = location if not is_wheel: # FIXME:https://github.com/pypa/pip/issues/1112 req_to_install.run_egg_info() if url and url.scheme in vcs.all_schemes: req_to_install.archive(self.download_dir) elif is_wheel: req_to_install.source_dir = location req_to_install.url = url.url else: req_to_install.source_dir = location req_to_install.run_egg_info() req_to_install.assert_source_matches_version() # req_to_install.req is only avail after unpack for URL # pkgs repeat check_if_exists to uninstall-on-upgrade # (#14) if not self.ignore_installed: req_to_install.check_if_exists() if req_to_install.satisfied_by: if self.upgrade or self.ignore_installed: # don't uninstall conflict if user install and # conflict is not user install if not (self.use_user_site and not dist_in_usersite( req_to_install.satisfied_by)): req_to_install.conflicts_with = \ req_to_install.satisfied_by req_to_install.satisfied_by = None else: logger.info( 'Requirement already satisfied (use ' '--upgrade to upgrade): %s', req_to_install, ) install = False # ###################### # # # parse dependencies # # # ###################### # if (req_to_install.extras): logger.debug( "Installing extra requirements: %r", ','.join(req_to_install.extras), ) if is_wheel: dist = list( pkg_resources.find_distributions(location) )[0] else: # sdists if req_to_install.satisfied_by: dist = req_to_install.satisfied_by else: dist = req_to_install.get_dist() # FIXME: shouldn't be globally added: if dist.has_metadata('dependency_links.txt'): finder.add_dependency_links( dist.get_metadata_lines('dependency_links.txt') ) if not self.ignore_dependencies: for subreq in dist.requires( req_to_install.extras): if self.has_requirement( subreq.project_name): # FIXME: check for conflict continue subreq = InstallRequirement( str(subreq), req_to_install, isolated=self.isolated, ) reqs.append(subreq) self.add_requirement(subreq) if not self.has_requirement(req_to_install.name): # 'unnamed' requirements will get added here self.add_requirement(req_to_install) # cleanup tmp src if (self.is_download or req_to_install._temp_build_dir is not None): self.reqs_to_cleanup.append(req_to_install) if install: self.successfully_downloaded.append(req_to_install)
def install(self, install_options, global_options=(), *args, **kwargs): """ Install everything in this set (after having downloaded and unpacked the packages) """ to_install = [r for r in self.requirements.values()[::-1] if not r.satisfied_by] # DISTRIBUTE TO SETUPTOOLS UPGRADE HACK (1 of 3 parts) # move the distribute-0.7.X wrapper to the end because it does not # install a setuptools package. by moving it to the end, we ensure it's # setuptools dependency is handled first, which will provide the # setuptools package # TODO: take this out later distribute_req = pkg_resources.Requirement.parse("distribute>=0.7") for req in to_install: if (req.name == 'distribute' and req.installed_version in distribute_req): to_install.remove(req) to_install.append(req) if to_install: logger.info( 'Installing collected packages: %s', ', '.join([req.name for req in to_install]), ) with indent_log(): for requirement in to_install: # DISTRIBUTE TO SETUPTOOLS UPGRADE HACK (1 of 3 parts) # when upgrading from distribute-0.6.X to the new merged # setuptools in py2, we need to force setuptools to uninstall # distribute. In py3, which is always using distribute, this # conversion is already happening in distribute's # pkg_resources. It's ok *not* to check if setuptools>=0.7 # because if someone were actually trying to ugrade from # distribute to setuptools 0.6.X, then all this could do is # actually help, although that upgade path was certainly never # "supported" # TODO: remove this later if requirement.name == 'setuptools': try: # only uninstall distribute<0.7. For >=0.7, setuptools # will also be present, and that's what we need to # uninstall distribute_requirement = \ pkg_resources.Requirement.parse("distribute<0.7") existing_distribute = \ pkg_resources.get_distribution("distribute") if existing_distribute in distribute_requirement: requirement.conflicts_with = existing_distribute except pkg_resources.DistributionNotFound: # distribute wasn't installed, so nothing to do pass if requirement.conflicts_with: logger.info( 'Found existing installation: %s', requirement.conflicts_with, ) with indent_log(): requirement.uninstall(auto_confirm=True) try: requirement.install( install_options, global_options, *args, **kwargs ) except: # if install did not succeed, rollback previous uninstall if (requirement.conflicts_with and not requirement.install_succeeded): requirement.rollback_uninstall() raise else: if (requirement.conflicts_with and requirement.install_succeeded): requirement.commit_uninstall() requirement.remove_temporary_source() self.successfully_installed = to_install
def run_egg_info(self): assert self.source_dir if self.name: logger.debug( 'Running setup.py (path:%s) egg_info for package %s', self.setup_py, self.name, ) else: logger.debug( 'Running setup.py (path:%s) egg_info for package from %s', self.setup_py, self.url, ) with indent_log(): # if it's distribute>=0.7, it won't contain an importable # setuptools, and having an egg-info dir blocks the ability of # setup.py to find setuptools plugins, so delete the egg-info dir # if no setuptools. it will get recreated by the run of egg_info # NOTE: this self.name check only works when installing from a # specifier (not archive path/urls) # TODO: take this out later if (self.name == 'distribute' and not os.path.isdir( os.path.join(self.source_dir, 'setuptools'))): rmtree(os.path.join(self.source_dir, 'distribute.egg-info')) script = self._run_setup_py script = script.replace('__SETUP_PY__', repr(self.setup_py)) script = script.replace('__PKG_NAME__', repr(self.name)) base_cmd = [sys.executable, '-c', script] if self.isolated: base_cmd += ["--no-user-cfg"] egg_info_cmd = base_cmd + ['egg_info'] # We can't put the .egg-info files at the root, because then the # source code will be mistaken for an installed egg, causing # problems if self.editable: egg_base_option = [] else: egg_info_dir = os.path.join(self.source_dir, 'pip-egg-info') if not os.path.exists(egg_info_dir): os.makedirs(egg_info_dir) egg_base_option = ['--egg-base', 'pip-egg-info'] cwd = self.source_dir if self.editable_options and \ 'subdirectory' in self.editable_options: cwd = os.path.join(cwd, self.editable_options['subdirectory']) call_subprocess( egg_info_cmd + egg_base_option, cwd=cwd, filter_stdout=self._filter_install, show_stdout=False, command_level=logging.DEBUG, command_desc='python setup.py egg_info') if not self.req: if isinstance( pkg_resources.parse_version(self.pkg_info()["Version"]), Version): op = "==" else: op = "===" self.req = pkg_resources.Requirement.parse( "".join([ self.pkg_info()["Name"], op, self.pkg_info()["Version"], ])) self.correct_build_location()
def install(self, install_options, global_options=(), root=None, strip_file_prefix=None): if self.editable: self.install_editable(install_options, global_options) return if self.is_wheel: version = pip.wheel.wheel_version(self.source_dir) pip.wheel.check_compatibility(version, self.name) self.move_wheel_files( self.source_dir, root=root, strip_file_prefix=strip_file_prefix ) self.install_succeeded = True return if self.isolated: global_options = list(global_options) + ["--no-user-cfg"] temp_location = tempfile.mkdtemp('-record', 'pip-') record_filename = os.path.join(temp_location, 'install-record.txt') try: install_args = [sys.executable] install_args.append('-c') install_args.append( "import setuptools, tokenize;__file__=%r;" "exec(compile(getattr(tokenize, 'open', open)(__file__).read()" ".replace('\\r\\n', '\\n'), __file__, 'exec'))" % self.setup_py ) install_args += list(global_options) + \ ['install', '--record', record_filename] if not self.as_egg: install_args += ['--single-version-externally-managed'] if root is not None: install_args += ['--root', root] if self.pycompile: install_args += ["--compile"] else: install_args += ["--no-compile"] if running_under_virtualenv(): # FIXME: I'm not sure if this is a reasonable location; # probably not but we can't put it in the default location, as # that is a virtualenv symlink that isn't writable py_ver_str = 'python' + sysconfig.get_python_version() install_args += ['--install-headers', os.path.join(sys.prefix, 'include', 'site', py_ver_str)] logger.info('Running setup.py install for %s', self.name) with indent_log(): call_subprocess( install_args + install_options, cwd=self.source_dir, filter_stdout=self._filter_install, show_stdout=False, ) if not os.path.exists(record_filename): logger.debug('Record file %s not found', record_filename) return self.install_succeeded = True if self.as_egg: # there's no --always-unzip option we can pass to install # command so we unable to save the installed-files.txt return def prepend_root(path): if root is None or not os.path.isabs(path): return path else: return change_root(root, path) with open(record_filename) as f: for line in f: directory = os.path.dirname(line) if directory.endswith('.egg-info'): egg_info_dir = prepend_root(directory) break else: logger.warning( 'Could not find .egg-info directory in install record' ' for %s', self, ) # FIXME: put the record somewhere # FIXME: should this be an error? return new_lines = [] with open(record_filename) as f: for line in f: filename = line.strip() if os.path.isdir(filename): filename += os.path.sep new_lines.append( make_path_relative( prepend_root(filename), egg_info_dir) ) inst_files_path = os.path.join(egg_info_dir, 'installed-files.txt') with open(inst_files_path, 'w') as f: f.write('\n'.join(new_lines) + '\n') finally: if os.path.exists(record_filename): os.remove(record_filename) rmtree(temp_location)
def run_egg_info(self): assert self.source_dir if self.name: logger.debug( 'Running setup.py (path:%s) egg_info for package %s', self.setup_py, self.name, ) else: logger.debug( 'Running setup.py (path:%s) egg_info for package from %s', self.setup_py, self.link, ) with indent_log(): # if it's distribute>=0.7, it won't contain an importable # setuptools, and having an egg-info dir blocks the ability of # setup.py to find setuptools plugins, so delete the egg-info dir # if no setuptools. it will get recreated by the run of egg_info # NOTE: this self.name check only works when installing from a # specifier (not archive path/urls) # TODO: take this out later if (self.name == 'distribute' and not os.path.isdir( os.path.join(self.source_dir, 'setuptools'))): rmtree(os.path.join(self.source_dir, 'distribute.egg-info')) script = self._run_setup_py script = script.replace('__SETUP_PY__', repr(self.setup_py)) script = script.replace('__PKG_NAME__', repr(self.name)) base_cmd = [sys.executable, '-c', script] if self.isolated: base_cmd += ["--no-user-cfg"] egg_info_cmd = base_cmd + ['egg_info'] # We can't put the .egg-info files at the root, because then the # source code will be mistaken for an installed egg, causing # problems if self.editable: egg_base_option = [] else: egg_info_dir = os.path.join(self.source_dir, 'pip-egg-info') if not os.path.exists(egg_info_dir): os.makedirs(egg_info_dir) egg_base_option = ['--egg-base', 'pip-egg-info'] cwd = self.source_dir if self.editable_options and \ 'subdirectory' in self.editable_options: cwd = os.path.join(cwd, self.editable_options['subdirectory']) call_subprocess( egg_info_cmd + egg_base_option, cwd=cwd, filter_stdout=_filter_install, show_stdout=False, command_level=logging.DEBUG, command_desc='python setup.py egg_info') if not self.req: if isinstance( pkg_resources.parse_version(self.pkg_info()["Version"]), Version): op = "==" else: op = "===" self.req = pkg_resources.Requirement.parse( "".join([ self.pkg_info()["Name"], op, self.pkg_info()["Version"], ])) self._correct_build_location()
def find_all_candidates(self, project_name): """Find all available InstallationCandidate for project_name This checks index_urls, find_links and dependency_links. All versions found are returned as an InstallationCandidate list. See _link_package_versions for details on which files are accepted """ index_locations = self._get_index_urls_locations(project_name) index_file_loc, index_url_loc = self._sort_locations(index_locations) fl_file_loc, fl_url_loc = self._sort_locations(self.find_links, expand_dir=True) dep_file_loc, dep_url_loc = self._sort_locations(self.dependency_links) file_locations = (Link(url) for url in itertools.chain( index_file_loc, fl_file_loc, dep_file_loc)) # We trust every url that the user has given us whether it was given # via --index-url or --find-links # We explicitly do not trust links that came from dependency_links # We want to filter out any thing which does not have a secure origin. url_locations = [ link for link in itertools.chain( (Link(url) for url in index_url_loc), (Link(url) for url in fl_url_loc), (Link(url) for url in dep_url_loc), ) if self._validate_secure_origin(logger, link) ] logger.debug('%d location(s) to search for versions of %s:', len(url_locations), project_name) for location in url_locations: logger.debug('* %s', location) canonical_name = canonicalize_name(project_name) formats = fmt_ctl_formats(self.format_control, canonical_name) search = Search(project_name, canonical_name, formats) find_links_versions = self._package_versions( # We trust every directly linked archive in find_links (Link(url, '-f') for url in self.find_links), search) page_versions = [] for page in self._get_pages(url_locations, project_name): logger.debug('Analyzing links from page %s', page.url) with indent_log(): page_versions.extend(self._package_versions( page.links, search)) dependency_versions = self._package_versions( (Link(url) for url in self.dependency_links), search) if dependency_versions: logger.debug( 'dependency_links found: %s', ', '.join( [version.location.url for version in dependency_versions])) file_versions = self._package_versions(file_locations, search) if file_versions: file_versions.sort(reverse=True) logger.debug( 'Local files found: %s', ', '.join([ url_to_path(candidate.location.url) for candidate in file_versions ])) # This is an intentional priority ordering return (file_versions + find_links_versions + page_versions + dependency_versions)
def install(self, install_options, global_options=(), root=None): if self.editable: self.install_editable(install_options, global_options) return if self.is_wheel: version = pip.wheel.wheel_version(self.source_dir) pip.wheel.check_compatibility(version, self.name) self.move_wheel_files(self.source_dir, root=root) self.install_succeeded = True return if self.isolated: global_options = list(global_options) + ["--no-user-cfg"] temp_location = tempfile.mkdtemp('-record', 'pip-') record_filename = os.path.join(temp_location, 'install-record.txt') try: install_args = [sys.executable] install_args.append('-c') install_args.append( "import setuptools, tokenize;__file__=%r;" "exec(compile(getattr(tokenize, 'open', open)(__file__).read()" ".replace('\\r\\n', '\\n'), __file__, 'exec'))" % self.setup_py ) install_args += list(global_options) + \ ['install', '--record', record_filename] if not self.as_egg: install_args += ['--single-version-externally-managed'] if root is not None: install_args += ['--root', root] if self.pycompile: install_args += ["--compile"] else: install_args += ["--no-compile"] if running_under_virtualenv(): py_ver_str = 'python' + sysconfig.get_python_version() install_args += ['--install-headers', os.path.join(sys.prefix, 'include', 'site', py_ver_str, self.name)] logger.info('Running setup.py install for %s', self.name) with indent_log(): call_subprocess( install_args + install_options, cwd=self.source_dir, filter_stdout=_filter_install, show_stdout=False, ) if not os.path.exists(record_filename): logger.debug('Record file %s not found', record_filename) return self.install_succeeded = True if self.as_egg: # there's no --always-unzip option we can pass to install # command so we unable to save the installed-files.txt return def prepend_root(path): if root is None or not os.path.isabs(path): return path else: return change_root(root, path) with open(record_filename) as f: for line in f: directory = os.path.dirname(line) if directory.endswith('.egg-info'): egg_info_dir = prepend_root(directory) break else: logger.warning( 'Could not find .egg-info directory in install record' ' for %s', self, ) # FIXME: put the record somewhere # FIXME: should this be an error? return new_lines = [] with open(record_filename) as f: for line in f: filename = line.strip() if os.path.isdir(filename): filename += os.path.sep new_lines.append( make_path_relative( prepend_root(filename), egg_info_dir) ) inst_files_path = os.path.join(egg_info_dir, 'installed-files.txt') with open(inst_files_path, 'w') as f: f.write('\n'.join(new_lines) + '\n') finally: if os.path.exists(record_filename): os.remove(record_filename) rmtree(temp_location)