def _check_md5(download_hash, link): download_hash = download_hash.hexdigest() if download_hash != link.md5_hash: logger.fatal( "MD5 hash of the package %s (%s) doesn't match the expected hash %s!" % (link, download_hash, link.md5_hash) ) raise InstallationError("Bad MD5 hash for package %s" % link)
def call_subprocess(cmd, show_stdout=True, filter_stdout=None, cwd=None, raise_on_returncode=True, command_level=logger.DEBUG, command_desc=None, extra_environ=None): if command_desc is None: cmd_parts = [] for part in cmd: if ' ' in part or '\n' in part or '"' in part or "'" in part: part = '"%s"' % part.replace('"', '\\"') cmd_parts.append(part) command_desc = ' '.join(cmd_parts) if show_stdout: stdout = None else: stdout = subprocess.PIPE logger.log(command_level, "Running command %s" % command_desc) env = os.environ.copy() if extra_environ: env.update(extra_environ) try: proc = subprocess.Popen( cmd, stderr=subprocess.STDOUT, stdin=None, stdout=stdout, cwd=cwd, env=env) except Exception, e: logger.fatal( "Error %s while executing command %s" % (e, command_desc)) raise
def unpack_file(filename, location, content_type, link): filename = os.path.realpath(filename) if ( content_type == "application/zip" or filename.endswith(".zip") or filename.endswith(".pybundle") or zipfile.is_zipfile(filename) ): unzip_file(filename, location, flatten=not filename.endswith(".pybundle")) elif ( content_type == "application/x-gzip" or tarfile.is_tarfile(filename) or splitext(filename)[1].lower() in (".tar", ".tar.gz", ".tar.bz2", ".tgz", ".tbz") ): untar_file(filename, location) elif content_type and content_type.startswith("text/html") and is_svn_page(file_contents(filename)): # We don't really care about this from pip.vcs.subversion import Subversion Subversion("svn+" + link.url).unpack(location) else: ## FIXME: handle? ## FIXME: magic signatures? logger.fatal( "Cannot unpack file %s (downloaded from %s, content-type: %s); cannot detect archive format" % (filename, location, content_type) ) raise InstallationError("Cannot determine archive format of %s" % location)
def unpack_file(filename, location, content_type, link): filename = os.path.realpath(filename) if (content_type == 'application/zip' or filename.endswith('.zip') or filename.endswith('.whl') or zipfile.is_zipfile(filename)): unzip_file( filename, location, flatten=not filename.endswith('.whl') ) elif (content_type == 'application/x-gzip' or tarfile.is_tarfile(filename) or splitext(filename)[1].lower() in ( '.tar', '.tar.gz', '.tar.bz2', '.tgz', '.tbz')): untar_file(filename, location) elif (content_type and content_type.startswith('text/html') and is_svn_page(file_contents(filename))): # We don't really care about this from pip.vcs.subversion import Subversion Subversion('svn+' + link.url).unpack(location) else: # FIXME: handle? # FIXME: magic signatures? logger.fatal( 'Cannot unpack file %s (downloaded from %s, content-type: %s); ' 'cannot detect archive format' % (filename, location, content_type) ) raise InstallationError( 'Cannot determine archive format of %s' % location )
def _check_md5(download_hash, link): download_hash = download_hash.hexdigest() if download_hash != link.md5_hash: logger.fatal( "MD5 hash of the package %s (%s) doesn't match the expected hash %s!" % (link, download_hash, link.md5_hash)) raise InstallationError('Bad MD5 hash for package %s' % link)
def _check_hash(download_hash, link): if download_hash.digest_size != hashlib.new(link.hash_name).digest_size: logger.fatal("Hash digest size of the package %d (%s) doesn't match the expected hash name %s!" % (download_hash.digest_size, link, link.hash_name)) raise HashMismatch('Hash name mismatch for package %s' % link) if download_hash.hexdigest() != link.hash: logger.fatal("Hash of the package %s (%s) doesn't match the expected hash %s!" % (link, download_hash.hexdigest(), link.hash)) raise HashMismatch('Bad %s hash for package %s' % (link.hash_name, link))
def _check_hash(download_hash, link): if download_hash.digest_size != hashlib.new(link.hash_name).digest_size: logger.fatal("Hash digest size of the package %d (%s) doesn't match the expected hash name %s!" % (download_hash.digest_size, link, link.hash_name)) raise InstallationError('Hash name mismatch for package %s' % link) if download_hash.hexdigest() != link.hash: logger.fatal("Hash of the package %s (%s) doesn't match the expected hash %s!" % (link, download_hash, link.hash)) raise InstallationError('Bad %s hash for package %s' % (link.hash_name, link))
def is_download(self): if self.download_dir: self.download_dir = os.path.expanduser(self.download_dir) if os.path.exists(self.download_dir): return True else: logger.fatal('Could not find download directory') raise InstallationError( "Could not find or access download directory '%s'" % display_path(self.download_dir)) return False
def _get_response_from_url(target_url, link): try: resp = urlopen(target_url) except urllib2.HTTPError: e = sys.exc_info()[1] logger.fatal("HTTP error %s while getting %s" % (e.code, link)) raise except IOError: e = sys.exc_info()[1] # Typically an FTP error logger.fatal("Error %s while getting %s" % (e, link)) raise return resp
def _find_url_name(self, index_url, url_name, req): """Finds the true URL name of a package, when the given name isn't quite correct. This is usually used to implement case-insensitivity.""" if not index_url.url.endswith('/'): # Vaguely part of the PyPI API... weird but true. ## FIXME: bad to modify this? index_url.url += '/' page = self._get_page(index_url, req) if page is None: logger.fatal('Cannot fetch index base URL %s' % index_url) return norm_name = normalize_name(req.url_name) for link in page.links: base = posixpath.basename(link.path.rstrip('/')) if norm_name == normalize_name(base): logger.notify('Real name of requirement %s is %s' % (url_name, base)) return base return None
def call_subprocess(cmd, show_stdout=True, filter_stdout=None, cwd=None, raise_on_returncode=True, command_level=logger.DEBUG, command_desc=None, extra_environ=None): if command_desc is None: cmd_parts = [] for part in cmd: if ' ' in part or '\n' in part or '"' in part or "'" in part: part = '"%s"' % part.replace('"', '\\"') cmd_parts.append(part) command_desc = ' '.join(cmd_parts) if show_stdout: stdout = None else: stdout = subprocess.PIPE logger.log(command_level, "Running command %s" % command_desc) env = os.environ.copy() if extra_environ: env.update(extra_environ) try: proc = subprocess.Popen(cmd, stderr=subprocess.STDOUT, stdin=None, stdout=stdout, cwd=cwd, env=env) except Exception: e = sys.exc_info()[1] logger.fatal("Error %s while executing command %s" % (e, command_desc)) raise all_output = [] if stdout is not None: stdout = proc.stdout while 1: line = console_to_str(stdout.readline()) if not line: break line = line.rstrip() all_output.append(line + '\n') if filter_stdout: level = filter_stdout(line) if isinstance(level, tuple): level, line = level logger.log(level, line) if not logger.stdout_level_matches(level): logger.show_progress() else: logger.info(line) else: returned_stdout, returned_stderr = proc.communicate() all_output = [returned_stdout or ''] proc.wait() if proc.returncode: if raise_on_returncode: if all_output: logger.notify('Complete output from command %s:' % command_desc) logger.notify('\n'.join(all_output) + '\n----------------------------------------') raise InstallationError( "Command %s failed with error code %s in %s" % (command_desc, proc.returncode, cwd)) else: logger.warn("Command %s had error code %s in %s" % (command_desc, proc.returncode, cwd)) if stdout is not None: return ''.join(all_output)
def main(self, complete_args, args, initial_options): options, args = self.parser.parse_args(args) self.merge_options(initial_options, options) if options.require_venv and not options.venv: # If a venv is required check if it can really be found if not os.environ.get('VIRTUAL_ENV'): print 'Could not find an activated virtualenv (required).' sys.exit(3) # Automatically install in currently activated venv if required options.respect_venv = True if args and args[-1] == '___VENV_RESTART___': ## FIXME: We don't do anything this this value yet: venv_location = args[-2] args = args[:-2] options.venv = None else: # If given the option to respect the activated environment # check if no venv is given as a command line parameter if options.respect_venv and os.environ.get('VIRTUAL_ENV'): if options.venv and os.path.exists(options.venv): # Make sure command line venv and environmental are the same if (os.path.realpath(os.path.expanduser(options.venv)) != os.path.realpath(os.environ.get('VIRTUAL_ENV'))): print ("Given virtualenv (%s) doesn't match " "currently activated virtualenv (%s)." % (options.venv, os.environ.get('VIRTUAL_ENV'))) sys.exit(3) else: options.venv = os.environ.get('VIRTUAL_ENV') print 'Using already activated environment %s' % options.venv level = 1 # Notify level += options.verbose level -= options.quiet level = logger.level_for_integer(4-level) complete_log = [] logger.consumers.extend( [(level, sys.stdout), (logger.DEBUG, complete_log.append)]) if options.log_explicit_levels: logger.explicit_levels = True if options.venv: if options.verbose > 0: # The logger isn't setup yet print 'Running in environment %s' % options.venv site_packages=False if options.site_packages: site_packages=True restart_in_venv(options.venv, options.venv_base, site_packages, complete_args) # restart_in_venv should actually never return, but for clarity... return ## FIXME: not sure if this sure come before or after venv restart if options.log: log_fp = open_logfile_append(options.log) logger.consumers.append((logger.DEBUG, log_fp)) else: log_fp = None socket.setdefaulttimeout(options.timeout or None) setup_proxy_handler(options.proxy) exit = 0 try: self.run(options, args) except (InstallationError, UninstallationError), e: logger.fatal(str(e)) logger.info('Exception information:\n%s' % format_exc()) exit = 1
def unpack_http_url(link, location, download_dir=None, session=None): if session is None: raise TypeError( "unpack_http_url() missing 1 required keyword argument: 'session'") temp_dir = tempfile.mkdtemp('-unpack', 'pip-') temp_location = None target_url = link.url.split('#', 1)[0] download_hash = None # If a download dir is specified, is the file already downloaded there? already_downloaded = None if download_dir: already_downloaded = os.path.join(download_dir, link.filename) if not os.path.exists(already_downloaded): already_downloaded = None # If already downloaded, does its hash match? if already_downloaded: temp_location = already_downloaded content_type = mimetypes.guess_type(already_downloaded)[0] logger.notify('File was already downloaded %s' % already_downloaded) if link.hash: download_hash = _get_hash_from_file(temp_location, link) try: _check_hash(download_hash, link) except HashMismatch: logger.warn('Previously-downloaded file %s has bad hash, ' 're-downloading.' % temp_location) temp_location = None os.unlink(already_downloaded) already_downloaded = None # let's download to a tmp dir if not temp_location: try: resp = session.get( target_url, # We use Accept-Encoding: identity here because requests # defaults to accepting compressed responses. This breaks in # a variety of ways depending on how the server is configured. # - Some servers will notice that the file isn't a compressible # file and will leave the file alone and with an empty # Content-Encoding # - Some servers will notice that the file is already # compressed and will leave the file alone and will add a # Content-Encoding: gzip header # - Some servers won't notice anything at all and will take # a file that's already been compressed and compress it again # and set the Content-Encoding: gzip header # By setting this to request only the identity encoding We're # hoping to eliminate the third case. Hopefully there does not # exist a server which when given a file will notice it is # already compressed and that you're not asking for a # compressed file and will then decompress it before sending # because if that's the case I don't think it'll ever be # possible to make this work. headers={"Accept-Encoding": "identity"}, stream=True, ) resp.raise_for_status() except requests.HTTPError as exc: logger.fatal("HTTP error %s while getting %s" % (exc.response.status_code, link)) raise content_type = resp.headers.get('content-type', '') filename = link.filename # fallback # Have a look at the Content-Disposition header for a better guess content_disposition = resp.headers.get('content-disposition') if content_disposition: type, params = cgi.parse_header(content_disposition) # We use ``or`` here because we don't want to use an "empty" value # from the filename param. filename = params.get('filename') or filename ext = splitext(filename)[1] if not ext: ext = mimetypes.guess_extension(content_type) if ext: filename += ext if not ext and link.url != resp.url: ext = os.path.splitext(resp.url)[1] if ext: filename += ext temp_location = os.path.join(temp_dir, filename) download_hash = _download_url(resp, link, temp_location) if link.hash and link.hash_name: _check_hash(download_hash, link) # a download dir is specified; let's copy the archive there if download_dir and not already_downloaded: _copy_file(temp_location, download_dir, content_type, link) # unpack the archive to the build dir location. even when only downloading # archives, they have to be unpacked to parse dependencies unpack_file(temp_location, location, content_type, link) if not already_downloaded: os.unlink(temp_location) os.rmdir(temp_dir)
def prepare_files(self, finder, force_root_egg_info=False, bundle=False): """Prepare process. Create temp directories, download and/or unpack files.""" unnamed = list(self.unnamed_requirements) reqs = list(self.requirements.values()) while reqs or unnamed: if unnamed: req_to_install = unnamed.pop(0) else: req_to_install = reqs.pop(0) install = True best_installed = False not_found = None if not self.ignore_installed and not req_to_install.editable: req_to_install.check_if_exists() if req_to_install.satisfied_by: substitute = self.install_req_checker.get_available_substitute(req_to_install) # if the req_to_install is identified as the best available substitue # AND # ( no version with req_to_install.name has been installed # OR a different version of req_to_install.name has been installed # ) # then set the self.upgrade flag to True to install req_to_install if req_to_install == substitute.requirement and ( req_to_install.name not in self.install_req_checker.pre_installed or self.install_req_checker.pre_installed[req_to_install.name].requirement is not req_to_install ): self.upgrade = True if self.upgrade: if not self.force_reinstall and not req_to_install.url: try: url = finder.find_requirement(req_to_install, self.upgrade) except BestVersionAlreadyInstalled: best_installed = True install = False except DistributionNotFound: not_found = sys.exc_info()[1] else: # Avoid the need to call find_requirement again req_to_install.url = url.url if not best_installed: # don't uninstall conflict if user install and conflict is not user install if not (self.use_user_site and not dist_in_usersite(req_to_install.satisfied_by)): req_to_install.conflicts_with = req_to_install.satisfied_by req_to_install.satisfied_by = None else: install = False if req_to_install.satisfied_by: if best_installed: logger.notify("Requirement already up-to-date: %s" % req_to_install) else: logger.notify( "Requirement already satisfied " "(use --upgrade to upgrade): %s" % req_to_install ) if req_to_install.editable: logger.notify("Obtaining %s" % req_to_install) elif install: logger.notify("Downloading/unpacking %s" % req_to_install) logger.indent += 2 try: is_bundle = False if req_to_install.editable: if req_to_install.source_dir is None: location = req_to_install.build_location(self.src_dir) req_to_install.source_dir = location else: location = req_to_install.source_dir if not os.path.exists(self.build_dir): _make_build_dir(self.build_dir) req_to_install.update_editable(not self.is_download) if self.is_download: req_to_install.run_egg_info() req_to_install.archive(self.download_dir) else: req_to_install.run_egg_info() elif install: ##@@ if filesystem packages are not marked ##editable in a req, a non deterministic error ##occurs when the script attempts to unpack the ##build directory # NB: This call can result in the creation of a temporary build directory location = req_to_install.build_location(self.build_dir, not self.is_download) ## FIXME: is the existance of the checkout good enough to use it? I don't think so. unpack = True url = None if not os.path.exists(os.path.join(location, "setup.py")): ## FIXME: this won't upgrade when there's an existing package unpacked in `location` if req_to_install.url is None: if not_found: raise not_found url = finder.find_requirement(req_to_install, upgrade=self.upgrade) else: ## FIXME: should req_to_install.url already be a link? url = Link(req_to_install.url) assert url if url: try: self.unpack_url(url, location, self.is_download) except HTTPError: e = sys.exc_info()[1] logger.fatal( "Could not install requirement %s because of error %s" % (req_to_install, e) ) raise InstallationError( "Could not install requirement %s because of HTTP error %s for URL %s" % (req_to_install, e, url) ) else: unpack = False if unpack: is_bundle = req_to_install.is_bundle if is_bundle: req_to_install.move_bundle_files(self.build_dir, self.src_dir) for subreq in req_to_install.bundle_requirements(): reqs.append(subreq) self.add_requirement(subreq) elif self.is_download: req_to_install.source_dir = location req_to_install.run_egg_info() if url and url.scheme in vcs.all_schemes: req_to_install.archive(self.download_dir) else: req_to_install.source_dir = location req_to_install.run_egg_info() if force_root_egg_info: # We need to run this to make sure that the .egg-info/ # directory is created for packing in the bundle req_to_install.run_egg_info(force_root_egg_info=True) req_to_install.assert_source_matches_version() # @@ sketchy way of identifying packages not grabbed from an index if bundle and req_to_install.url: self.copy_to_build_dir(req_to_install) install = False # req_to_install.req is only avail after unpack for URL pkgs # repeat check_if_exists to uninstall-on-upgrade (#14) req_to_install.check_if_exists() if req_to_install.satisfied_by: if self.upgrade or self.ignore_installed: # don't uninstall conflict if user install and and conflict is not user install if not (self.use_user_site and not dist_in_usersite(req_to_install.satisfied_by)): req_to_install.conflicts_with = req_to_install.satisfied_by req_to_install.satisfied_by = None else: install = False if not is_bundle: ## FIXME: shouldn't be globally added: finder.add_dependency_links(req_to_install.dependency_links) if req_to_install.extras: logger.notify("Installing extra requirements: %r" % ",".join(req_to_install.extras)) if not self.ignore_dependencies: for req in req_to_install.requirements(req_to_install.extras): try: name = pkg_resources.Requirement.parse(req).project_name except ValueError: e = sys.exc_info()[1] ## FIXME: proper warning logger.error( "Invalid requirement: %r (%s) in requirement %s" % (req, e, req_to_install) ) continue if self.has_requirement(name): ## FIXME: check for conflict continue subreq = InstallRequirement(req, req_to_install) reqs.append(subreq) self.add_requirement(subreq) if req_to_install.editable and req_to_install.source_dir: for subreq in self.install_requirements_txt(req_to_install): if self.add_requirement(subreq): reqs.append(subreq) if not self.has_requirement(req_to_install.name): #'unnamed' requirements will get added here self.add_requirement(req_to_install) if self.is_download or req_to_install._temp_build_dir is not None: self.reqs_to_cleanup.append(req_to_install) else: self.reqs_to_cleanup.append(req_to_install) if install: self.successfully_downloaded.append(req_to_install) if bundle and (req_to_install.url and req_to_install.url.startswith("file:///")): self.copy_to_build_dir(req_to_install) finally: logger.indent -= 2
else: log_fp = None socket.setdefaulttimeout(options.timeout or None) setup_proxy_handler(options.proxy) exit = 0 try: self.run(options, args) except (InstallationError, UninstallationError), e: logger.fatal(str(e)) logger.info('Exception information:\n%s' % format_exc()) exit = 1 except: logger.fatal('Exception:\n%s' % format_exc()) exit = 2 if log_fp is not None: log_fp.close() if exit: log_fn = options.log_file text = '\n'.join(complete_log) logger.fatal('Storing complete log in %s' % log_fn) log_fp = open_logfile_append(log_fn) log_fp.write(text) log_fp.close() return exit ## FIXME: should get moved somewhere else: def setup_proxy_handler(proxystr=''):
def find_requirement(self, req, upgrade): def mkurl_pypi_url(url): loc = posixpath.join(url, url_name) # For maximum compatibility with easy_install, ensure the path # ends in a trailing slash. Although this isn't in the spec # (and PyPI can handle it without the slash) some other index # implementations might break if they relied on easy_install's # behavior. if not loc.endswith('/'): loc = loc + '/' return loc url_name = req.url_name # Only check main index if index URL is given: main_index_url = None if self.index_urls: # Check that we have the url_name correctly spelled: main_index_url = Link( mkurl_pypi_url(self.index_urls[0]), trusted=True, ) # This will also cache the page, so it's okay that we get it again # later: page = self._get_page(main_index_url, req) if page is None: url_name = self._find_url_name( Link(self.index_urls[0], trusted=True), url_name, req ) or req.url_name if url_name is not None: locations = [ mkurl_pypi_url(url) for url in self.index_urls] + self.find_links else: locations = list(self.find_links) for version in req.absolute_versions: if url_name is not None and main_index_url is not None: locations = [ posixpath.join(main_index_url.url, version)] + locations file_locations, url_locations = self._sort_locations(locations) # We trust every url that the user has given us whether it was given # via --index-url or --find-links locations = [Link(url, trusted=True) for url in url_locations] logger.debug('URLs to search for versions for %s:' % req) for location in locations: logger.debug('* %s' % location) # Determine if this url used a secure transport mechanism parsed = urlparse.urlparse(str(location)) if parsed.scheme in INSECURE_SCHEMES: secure_schemes = INSECURE_SCHEMES[parsed.scheme] if len(secure_schemes) == 1: ctx = (location, parsed.scheme, secure_schemes[0], parsed.netloc) logger.warn("%s uses an insecure transport scheme (%s). " "Consider using %s if %s has it available" % ctx) elif len(secure_schemes) > 1: ctx = ( location, parsed.scheme, ", ".join(secure_schemes), parsed.netloc, ) logger.warn("%s uses an insecure transport scheme (%s). " "Consider using one of %s if %s has any of " "them available" % ctx) else: ctx = (location, parsed.scheme) logger.warn("%s uses an insecure transport scheme (%s)." % ctx) found_versions = [] found_versions.extend( self._package_versions( # We trust every directly linked archive in find_links [Link(url, '-f', trusted=True) for url in self.find_links], req.name.lower() ) ) page_versions = [] for page in self._get_pages(locations, req): logger.debug('Analyzing links from page %s' % page.url) logger.indent += 2 try: page_versions.extend( self._package_versions(page.links, req.name.lower()) ) finally: logger.indent -= 2 file_versions = list( self._package_versions( [Link(url) for url in file_locations], req.name.lower() ) ) if (not found_versions and not page_versions and not file_versions): logger.fatal( 'Could not find any downloads that satisfy the requirement' ' %s' % req ) if self.need_warn_external: logger.warn("Some externally hosted files were ignored (use " "--allow-external %s to allow)." % req.name) if self.need_warn_unverified: logger.warn("Some insecure and unverifiable files were ignored" " (use --allow-unverified %s to allow)." % req.name) raise DistributionNotFound( 'No distributions at all found for %s' % req ) installed_version = [] if req.satisfied_by is not None: installed_version = [( req.satisfied_by.parsed_version, INSTALLED_VERSION, req.satisfied_by.version, )] if file_versions: file_versions.sort(reverse=True) logger.info( 'Local files found: %s' % ', '.join([ url_to_path(link.url) for parsed, link, version in file_versions ]) ) # this is an intentional priority ordering all_versions = installed_version + file_versions + found_versions \ + page_versions applicable_versions = [] for (parsed_version, link, version) in all_versions: if version not in req.req: logger.info( "Ignoring link %s, version %s doesn't match %s" % ( link, version, ','.join([''.join(s) for s in req.req.specs]) ) ) continue elif (is_prerelease(version) and not (self.allow_all_prereleases or req.prereleases)): # If this version isn't the already installed one, then # ignore it if it's a pre-release. if link is not INSTALLED_VERSION: logger.info( "Ignoring link %s, version %s is a pre-release (use " "--pre to allow)." % (link, version) ) continue applicable_versions.append((parsed_version, link, version)) applicable_versions = self._sort_versions(applicable_versions) existing_applicable = bool([ link for parsed_version, link, version in applicable_versions if link is INSTALLED_VERSION ]) if not upgrade and existing_applicable: if applicable_versions[0][1] is INSTALLED_VERSION: logger.info( 'Existing installed version (%s) is most up-to-date and ' 'satisfies requirement' % req.satisfied_by.version ) else: logger.info( 'Existing installed version (%s) satisfies requirement ' '(most up-to-date version is %s)' % (req.satisfied_by.version, applicable_versions[0][2]) ) return None if not applicable_versions: logger.fatal( 'Could not find a version that satisfies the requirement %s ' '(from versions: %s)' % ( req, ', '.join([ version for parsed_version, link, version in all_versions ]) ) ) if self.need_warn_external: logger.warn("Some externally hosted files were ignored (use " "--allow-external to allow).") if self.need_warn_unverified: logger.warn("Some insecure and unverifiable files were ignored" " (use --allow-unverified %s to allow)." % req.name) raise DistributionNotFound( 'No distributions matching the version for %s' % req ) if applicable_versions[0][1] is INSTALLED_VERSION: # We have an existing version, and its the best version logger.info( 'Installed version (%s) is most up-to-date (past versions: ' '%s)' % ( req.satisfied_by.version, ', '.join([ version for parsed_version, link, version in applicable_versions[1:] ]) or 'none')) raise BestVersionAlreadyInstalled if len(applicable_versions) > 1: logger.info( 'Using version %s (newest of versions: %s)' % ( applicable_versions[0][2], ', '.join([ version for parsed_version, link, version in applicable_versions ]) ) ) selected_version = applicable_versions[0][1] if (selected_version.internal is not None and not selected_version.internal): logger.warn("%s an externally hosted file and may be " "unreliable" % req.name) if (selected_version.verifiable is not None and not selected_version.verifiable): logger.warn("%s is potentially insecure and " "unverifiable." % req.name) if selected_version._deprecated_regex: logger.deprecated( "1.7", "%s discovered using a deprecated method of parsing, " "in the future it will no longer be discovered" % req.name ) return selected_version
def _check_md5(download_hash, link): download_hash = download_hash.hexdigest() if download_hash != link.md5_hash: logger.fatal("MD5 hash of the package {0!s} ({1!s}) doesn't match the expected hash {2!s}!".format(link, download_hash, link.md5_hash)) raise InstallationError('Bad MD5 hash for package {0!s}'.format(link))
def find_requirement(self, req, upgrade): def mkurl_pypi_url(url): loc = posixpath.join(url, url_name) # For maximum compatibility with easy_install, ensure the path # ends in a trailing slash. Although this isn't in the spec # (and PyPI can handle it without the slash) some other index # implementations might break if they relied on easy_install's behavior. if not loc.endswith('/'): loc = loc + '/' return loc url_name = req.url_name # Only check main index if index URL is given: main_index_url = None if self.index_urls: # Check that we have the url_name correctly spelled: main_index_url = Link(mkurl_pypi_url(self.index_urls[0]), trusted=True) # This will also cache the page, so it's okay that we get it again later: page = self._get_page(main_index_url, req) if page is None: url_name = self._find_url_name( Link(self.index_urls[0], trusted=True), url_name, req) or req.url_name if url_name is not None: locations = [mkurl_pypi_url(url) for url in self.index_urls] + self.find_links else: locations = list(self.find_links) for version in req.absolute_versions: if url_name is not None and main_index_url is not None: locations = [posixpath.join(main_index_url.url, version) ] + locations file_locations, url_locations = self._sort_locations(locations) _flocations, _ulocations = self._sort_locations(self.dependency_links) file_locations.extend(_flocations) # We trust every url that the user has given us whether it was given # via --index-url or --find-links locations = [Link(url, trusted=True) for url in url_locations] # We explicitly do not trust links that came from dependency_links locations.extend([Link(url) for url in _ulocations]) logger.debug('URLs to search for versions for %s:' % req) for location in locations: logger.debug('* %s' % location) # Determine if this url used a secure transport mechanism parsed = urlparse.urlparse(str(location)) if parsed.scheme in INSECURE_SCHEMES: secure_schemes = INSECURE_SCHEMES[parsed.scheme] if len(secure_schemes) == 1: ctx = (location, parsed.scheme, secure_schemes[0], parsed.netloc) logger.warn("%s uses an insecure transport scheme (%s). " "Consider using %s if %s has it available" % ctx) elif len(secure_schemes) > 1: ctx = (location, parsed.scheme, ", ".join(secure_schemes), parsed.netloc) logger.warn("%s uses an insecure transport scheme (%s). " "Consider using one of %s if %s has any of " "them available" % ctx) else: ctx = (location, parsed.scheme) logger.warn("%s uses an insecure transport scheme (%s)." % ctx) found_versions = [] found_versions.extend( self._package_versions( # We trust every directly linked archive in find_links [Link(url, '-f', trusted=True) for url in self.find_links], req.name.lower())) page_versions = [] for page in self._get_pages(locations, req): logger.debug('Analyzing links from page %s' % page.url) logger.indent += 2 try: page_versions.extend( self._package_versions(page.links, req.name.lower())) finally: logger.indent -= 2 dependency_versions = list( self._package_versions( [Link(url) for url in self.dependency_links], req.name.lower())) if dependency_versions: logger.info('dependency_links found: %s' % ', '.join( [link.url for parsed, link, version in dependency_versions])) file_versions = list( self._package_versions([Link(url) for url in file_locations], req.name.lower())) if not found_versions and not page_versions and not dependency_versions and not file_versions: logger.fatal( 'Could not find any downloads that satisfy the requirement %s' % req) if self.need_warn_external: logger.warn("Some externally hosted files were ignored (use " "--allow-external %s to allow)." % req.name) if self.need_warn_unverified: logger.warn("Some insecure and unverifiable files were ignored" " (use --allow-unverified %s to allow)." % req.name) raise DistributionNotFound('No distributions at all found for %s' % req) installed_version = [] if req.satisfied_by is not None: installed_version = [(req.satisfied_by.parsed_version, INSTALLED_VERSION, req.satisfied_by.version)] if file_versions: file_versions.sort(reverse=True) logger.info('Local files found: %s' % ', '.join([ url_to_path(link.url) for parsed, link, version in file_versions ])) #this is an intentional priority ordering all_versions = installed_version + file_versions + found_versions + page_versions + dependency_versions applicable_versions = [] for (parsed_version, link, version) in all_versions: if version not in req.req: logger.info("Ignoring link %s, version %s doesn't match %s" % (link, version, ','.join( [''.join(s) for s in req.req.specs]))) continue elif is_prerelease(version) and not (self.allow_all_prereleases or req.prereleases): # If this version isn't the already installed one, then # ignore it if it's a pre-release. if link is not INSTALLED_VERSION: logger.info( "Ignoring link %s, version %s is a pre-release (use --pre to allow)." % (link, version)) continue applicable_versions.append((parsed_version, link, version)) applicable_versions = self._sort_versions(applicable_versions) existing_applicable = bool([ link for parsed_version, link, version in applicable_versions if link is INSTALLED_VERSION ]) if not upgrade and existing_applicable: if applicable_versions[0][1] is INSTALLED_VERSION: logger.info( 'Existing installed version (%s) is most up-to-date and satisfies requirement' % req.satisfied_by.version) else: logger.info( 'Existing installed version (%s) satisfies requirement (most up-to-date version is %s)' % (req.satisfied_by.version, applicable_versions[0][2])) return None if not applicable_versions: logger.fatal( 'Could not find a version that satisfies the requirement %s (from versions: %s)' % (req, ', '.join([ version for parsed_version, link, version in all_versions ]))) if self.need_warn_external: logger.warn("Some externally hosted files were ignored (use " "--allow-external to allow).") if self.need_warn_unverified: logger.warn("Some insecure and unverifiable files were ignored" " (use --allow-unverified %s to allow)." % req.name) raise DistributionNotFound( 'No distributions matching the version for %s' % req) if applicable_versions[0][1] is INSTALLED_VERSION: # We have an existing version, and its the best version logger.info( 'Installed version (%s) is most up-to-date (past versions: %s)' % (req.satisfied_by.version, ', '.join([ version for parsed_version, link, version in applicable_versions[1:] ]) or 'none')) raise BestVersionAlreadyInstalled if len(applicable_versions) > 1: logger.info( 'Using version %s (newest of versions: %s)' % (applicable_versions[0][2], ', '.join([ version for parsed_version, link, version in applicable_versions ]))) selected_version = applicable_versions[0][1] # TODO: Remove after 1.4 has been released # if (selected_version.internal is not None # and not selected_version.internal): # logger.warn("You are installing an externally hosted file. Future " # "versions of pip will default to disallowing " # "externally hosted files.") # if (selected_version.verifiable is not None # and not selected_version.verifiable): # logger.warn("You are installing a potentially insecure and " # "unverifiable file. Future versions of pip will " # "default to disallowing insecure files.") if selected_version._deprecated_regex: logger.deprecated( "1.7", "%s discovered using a deprecated method of parsing, " "in the future it will no longer be discovered" % req.name) return selected_version
def main(self, args): options, args = self.parse_args(args) level = 1 # Notify level += options.verbose level -= options.quiet level = logger.level_for_integer(4 - level) complete_log = [] logger.add_consumers( (level, sys.stdout), (logger.DEBUG, complete_log.append), ) if options.log_explicit_levels: logger.explicit_levels = True self.setup_logging() #TODO: try to get these passing down from the command? # without resorting to os.environ to hold these. if options.no_input: os.environ['PIP_NO_INPUT'] = '1' if options.exists_action: os.environ['PIP_EXISTS_ACTION'] = ' '.join(options.exists_action) if options.require_venv: # If a venv is required check if it can really be found if not running_under_virtualenv(): logger.fatal( 'Could not find an activated virtualenv (required).') sys.exit(VIRTUALENV_NOT_FOUND) if options.log: log_fp = open_logfile(options.log, 'a') logger.add_consumers((logger.DEBUG, log_fp)) else: log_fp = None exit = SUCCESS store_log = False try: status = self.run(options, args) # FIXME: all commands should return an exit status # and when it is done, isinstance is not needed anymore if isinstance(status, int): exit = status except PreviousBuildDirError: e = sys.exc_info()[1] logger.fatal(str(e)) logger.info('Exception information:\n%s' % format_exc()) store_log = True exit = PREVIOUS_BUILD_DIR_ERROR except (InstallationError, UninstallationError): e = sys.exc_info()[1] logger.fatal(str(e)) logger.info('Exception information:\n%s' % format_exc()) store_log = True exit = ERROR except BadCommand: e = sys.exc_info()[1] logger.fatal(str(e)) logger.info('Exception information:\n%s' % format_exc()) store_log = True exit = ERROR except CommandError: e = sys.exc_info()[1] logger.fatal('ERROR: %s' % e) logger.info('Exception information:\n%s' % format_exc()) exit = ERROR except KeyboardInterrupt: logger.fatal('Operation cancelled by user') logger.info('Exception information:\n%s' % format_exc()) store_log = True exit = ERROR except: logger.fatal('Exception:\n%s' % format_exc()) store_log = True exit = UNKNOWN_ERROR if store_log: log_file_fn = options.log_file text = '\n'.join(complete_log) try: log_file_fp = open_logfile(log_file_fn, 'w') except IOError: temp = tempfile.NamedTemporaryFile(delete=False) log_file_fn = temp.name log_file_fp = open_logfile(log_file_fn, 'w') logger.fatal('Storing debug log for failure in %s' % log_file_fn) log_file_fp.write(text) log_file_fp.close() if log_fp is not None: log_fp.close() return exit
temp_location = os.path.join(temp_dir, filename) download_hash = _download_url(resp, link, temp_location) if link.md5_hash: _check_md5(download_hash, link) if only_download: _copy_file(temp_location, location, content_type, link) else: unpack_file(temp_location, location, content_type, link) if target_file and target_file != temp_location: cache_download(target_file, temp_location, content_type) if target_file is None: os.unlink(temp_location) os.rmdir(temp_dir) def _get_response_from_url(target_url, link): try: resp = urlopen(target_url) except urllib2.HTTPError, e: logger.fatal("HTTP error %s while getting %s" % (e.code, link)) raise except IOError, e: # Typically an FTP error logger.fatal("Error %s while getting %s" % (e, link)) raise return resp class Urllib2HeadRequest(urllib2.Request): def get_method(self): return "HEAD"
urlopen.setup(proxystr=options.proxy, prompting=not options.no_input) exit = 0 try: self.run(options, args) except (InstallationError, UninstallationError), e: logger.fatal(str(e)) logger.info('Exception information:\n%s' % format_exc()) exit = 1 except BadCommand, e: logger.fatal(str(e)) logger.info('Exception information:\n%s' % format_exc()) exit = 1 except: logger.fatal('Exception:\n%s' % format_exc()) exit = 2 if log_fp is not None: log_fp.close() if exit: log_fn = options.log_file text = '\n'.join(complete_log) logger.fatal('Storing complete log in %s' % log_fn) log_fp = open_logfile(log_fn, 'w') log_fp.write(text) log_fp.close() return exit
def unpack_http_url(link, location, download_cache, download_dir=None, session=None): if session is None: session = PipSession() temp_dir = tempfile.mkdtemp('-unpack', 'pip-') temp_location = None target_url = link.url.split('#', 1)[0] already_cached = False cache_file = None cache_content_type_file = None download_hash = None if download_cache: cache_file = os.path.join(download_cache, urllib.quote(target_url, '')) cache_content_type_file = cache_file + '.content-type' already_cached = ( os.path.exists(cache_file) and os.path.exists(cache_content_type_file) ) if not os.path.isdir(download_cache): create_download_cache_folder(download_cache) already_downloaded = None if download_dir: already_downloaded = os.path.join(download_dir, link.filename) if not os.path.exists(already_downloaded): already_downloaded = None if already_downloaded: temp_location = already_downloaded content_type = mimetypes.guess_type(already_downloaded)[0] logger.notify('File was already downloaded %s' % already_downloaded) if link.hash: download_hash = _get_hash_from_file(temp_location, link) try: _check_hash(download_hash, link) except HashMismatch: logger.warn( 'Previously-downloaded file %s has bad hash, ' 're-downloading.' % temp_location ) temp_location = None os.unlink(already_downloaded) already_downloaded = None # We have a cached file, and we haven't already found a good downloaded copy if already_cached and not temp_location: with open(cache_content_type_file) as fp: content_type = fp.read().strip() temp_location = cache_file logger.notify('Using download cache from %s' % cache_file) if link.hash and link.hash_name: download_hash = _get_hash_from_file(cache_file, link) try: _check_hash(download_hash, link) except HashMismatch: logger.warn( 'Cached file %s has bad hash, ' 're-downloading.' % temp_location ) temp_location = None os.unlink(cache_file) os.unlink(cache_content_type_file) already_cached = False # We don't have either a cached or a downloaded copy if not temp_location: try: resp = session.get(target_url, stream=True) resp.raise_for_status() except requests.HTTPError as exc: logger.fatal("HTTP error %s while getting %s" % (exc.response.status_code, link)) raise content_type = resp.headers.get('content-type', '') filename = link.filename # fallback # Have a look at the Content-Disposition header for a better guess content_disposition = resp.headers.get('content-disposition') if content_disposition: type, params = cgi.parse_header(content_disposition) # We use ``or`` here because we don't want to use an "empty" value # from the filename param. filename = params.get('filename') or filename ext = splitext(filename)[1] if not ext: ext = mimetypes.guess_extension(content_type) if ext: filename += ext if not ext and link.url != resp.url: ext = os.path.splitext(resp.url)[1] if ext: filename += ext temp_location = os.path.join(temp_dir, filename) download_hash = _download_url(resp, link, temp_location) if link.hash and link.hash_name: _check_hash(download_hash, link) if download_dir and not already_downloaded: _copy_file(temp_location, download_dir, content_type, link) unpack_file(temp_location, location, content_type, link) if cache_file and not already_cached: cache_download(cache_file, temp_location, content_type) if not (already_cached or already_downloaded): os.unlink(temp_location) os.rmdir(temp_dir)
def main(self, args): options, args = self.parse_args(args) level = 1 # Notify level += options.verbose level -= options.quiet level = logger.level_for_integer(4 - level) complete_log = [] logger.add_consumers( (level, sys.stdout), (logger.DEBUG, complete_log.append), ) if options.log_explicit_levels: logger.explicit_levels = True self.setup_logging() # TODO: try to get these passing down from the command? # without resorting to os.environ to hold these. if options.no_input: os.environ['PIP_NO_INPUT'] = '1' if options.exists_action: os.environ['PIP_EXISTS_ACTION'] = ' '.join(options.exists_action) if options.require_venv: # If a venv is required check if it can really be found if not running_under_virtualenv(): logger.fatal( 'Could not find an activated virtualenv (required).' ) sys.exit(VIRTUALENV_NOT_FOUND) if options.log: log_fp = open_logfile(options.log, 'a') logger.add_consumers((logger.DEBUG, log_fp)) else: log_fp = None exit = SUCCESS store_log = False try: status = self.run(options, args) # FIXME: all commands should return an exit status # and when it is done, isinstance is not needed anymore if isinstance(status, int): exit = status except PreviousBuildDirError as exc: logger.fatal(str(exc)) logger.info('Exception information:\n%s' % format_exc()) store_log = True exit = PREVIOUS_BUILD_DIR_ERROR except (InstallationError, UninstallationError) as exc: logger.fatal(str(exc)) logger.info('Exception information:\n%s' % format_exc()) store_log = True exit = ERROR except BadCommand as exc: logger.fatal(str(exc)) logger.info('Exception information:\n%s' % format_exc()) store_log = True exit = ERROR except CommandError as exc: logger.fatal('ERROR: %s' % exc) logger.info('Exception information:\n%s' % format_exc()) exit = ERROR except KeyboardInterrupt: logger.fatal('Operation cancelled by user') logger.info('Exception information:\n%s' % format_exc()) store_log = True exit = ERROR except: logger.fatal('Exception:\n%s' % format_exc()) store_log = True exit = UNKNOWN_ERROR if store_log: log_file_fn = options.log_file text = '\n'.join(complete_log) try: log_file_fp = open_logfile(log_file_fn, 'w') except IOError: temp = tempfile.NamedTemporaryFile(delete=False) log_file_fn = temp.name log_file_fp = open_logfile(log_file_fn, 'w') logger.fatal('Storing debug log for failure in %s' % log_file_fn) log_file_fp.write(text) log_file_fp.close() if log_fp is not None: log_fp.close() return exit
def find_requirement(self, req, upgrade): def mkurl_pypi_url(url): loc = posixpath.join(url, url_name) # For maximum compatibility with easy_install, ensure the path # ends in a trailing slash. Although this isn't in the spec # (and PyPI can handle it without the slash) some other index # implementations might break if they relied on easy_install's behavior. if not loc.endswith('/'): loc = loc + '/' return loc url_name = req.url_name # Only check main index if index URL is given: main_index_url = None if self.index_urls: # Check that we have the url_name correctly spelled: main_index_url = Link(mkurl_pypi_url(self.index_urls[0]), trusted=True) # This will also cache the page, so it's okay that we get it again later: page = self._get_page(main_index_url, req) if page is None: url_name = self._find_url_name(Link(self.index_urls[0], trusted=True), url_name, req) or req.url_name # Combine index URLs with mirror URLs here to allow # adding more index URLs from requirements files all_index_urls = self.index_urls + self.mirror_urls if url_name is not None: locations = [ mkurl_pypi_url(url) for url in all_index_urls] + self.find_links else: locations = list(self.find_links) for version in req.absolute_versions: if url_name is not None and main_index_url is not None: locations = [ posixpath.join(main_index_url.url, version)] + locations file_locations, url_locations = self._sort_locations(locations) _flocations, _ulocations = self._sort_locations(self.dependency_links) file_locations.extend(_flocations) # We trust every url that the user has given us whether it was given # via --index-url, --user-mirrors/--mirror, or --find-links or a # default option thereof locations = [Link(url, trusted=True) for url in url_locations] # We explicitly do not trust links that came from dependency_links locations.extend([Link(url) for url in _ulocations]) logger.debug('URLs to search for versions for %s:' % req) for location in locations: logger.debug('* %s' % location) found_versions = [] found_versions.extend( self._package_versions( # We trust every directly linked archive in find_links [Link(url, '-f', trusted=True) for url in self.find_links], req.name.lower())) page_versions = [] for page in self._get_pages(locations, req): logger.debug('Analyzing links from page %s' % page.url) logger.indent += 2 try: page_versions.extend(self._package_versions(page.links, req.name.lower())) finally: logger.indent -= 2 dependency_versions = list(self._package_versions( [Link(url) for url in self.dependency_links], req.name.lower())) if dependency_versions: logger.info('dependency_links found: %s' % ', '.join([link.url for parsed, link, version in dependency_versions])) file_versions = list(self._package_versions( [Link(url) for url in file_locations], req.name.lower())) if not found_versions and not page_versions and not dependency_versions and not file_versions: logger.fatal('Could not find any downloads that satisfy the requirement %s' % req) if self.need_warn_external: logger.warn("Some externally hosted files were ignored (use " "--allow-external %s to allow)." % req.name) if self.need_warn_insecure: logger.warn("Some insecure and unverifiable files were ignored" " (use --allow-insecure %s to allow)." % req.name) raise DistributionNotFound('No distributions at all found for %s' % req) installed_version = [] if req.satisfied_by is not None: installed_version = [(req.satisfied_by.parsed_version, InfLink, req.satisfied_by.version)] if file_versions: file_versions.sort(reverse=True) logger.info('Local files found: %s' % ', '.join([url_to_path(link.url) for parsed, link, version in file_versions])) #this is an intentional priority ordering all_versions = installed_version + file_versions + found_versions + page_versions + dependency_versions applicable_versions = [] for (parsed_version, link, version) in all_versions: if version not in req.req: logger.info("Ignoring link %s, version %s doesn't match %s" % (link, version, ','.join([''.join(s) for s in req.req.specs]))) continue elif is_prerelease(version) and not (self.allow_all_prereleases or req.prereleases): # If this version isn't the already installed one, then # ignore it if it's a pre-release. if link is not InfLink: logger.info("Ignoring link %s, version %s is a pre-release (use --pre to allow)." % (link, version)) continue applicable_versions.append((parsed_version, link, version)) applicable_versions = self._sort_versions(applicable_versions) existing_applicable = bool([link for parsed_version, link, version in applicable_versions if link is InfLink]) if not upgrade and existing_applicable: if applicable_versions[0][1] is InfLink: logger.info('Existing installed version (%s) is most up-to-date and satisfies requirement' % req.satisfied_by.version) else: logger.info('Existing installed version (%s) satisfies requirement (most up-to-date version is %s)' % (req.satisfied_by.version, applicable_versions[0][2])) return None if not applicable_versions: logger.fatal('Could not find a version that satisfies the requirement %s (from versions: %s)' % (req, ', '.join([version for parsed_version, link, version in all_versions]))) if self.need_warn_external: logger.warn("Some externally hosted files were ignored (use " "--allow-external to allow).") if self.need_warn_insecure: logger.warn("Some insecure and unverifiable files were ignored" " (use --allow-insecure %s to allow)." % req.name) raise DistributionNotFound('No distributions matching the version for %s' % req) if applicable_versions[0][1] is InfLink: # We have an existing version, and its the best version logger.info('Installed version (%s) is most up-to-date (past versions: %s)' % (req.satisfied_by.version, ', '.join([version for parsed_version, link, version in applicable_versions[1:]]) or 'none')) raise BestVersionAlreadyInstalled if len(applicable_versions) > 1: logger.info('Using version %s (newest of versions: %s)' % (applicable_versions[0][2], ', '.join([version for parsed_version, link, version in applicable_versions]))) selected_version = applicable_versions[0][1] # TODO: Remove after 1.4 has been released if (selected_version.internal is not None and not selected_version.internal): logger.warn("You are installing an externally hosted file. Future " "versions of pip will default to disallowing " "externally hosted files.") if (selected_version.verifiable is not None and not selected_version.verifiable): logger.warn("You are installing a potentially insecure and " "unverifiable file. Future versions of pip will " "default to disallowing insecure files.") return selected_version
class Command(object): name = None usage = None hidden = False def __init__(self): assert self.name self.parser = ConfigOptionParser( usage=self.usage, prog='%s %s' % (sys.argv[0], self.name), version=parser.version, formatter=UpdatingDefaultsHelpFormatter(), name=self.name) for option in parser.option_list: if not option.dest or option.dest == 'help': # -h, --version, etc continue self.parser.add_option(option) command_dict[self.name] = self def merge_options(self, initial_options, options): # Make sure we have all global options carried over for attr in [ 'log', 'venv', 'proxy', 'venv_base', 'require_venv', 'respect_venv', 'log_explicit_levels', 'log_file', 'timeout', 'default_vcs', 'skip_requirements_regex', 'no_input' ]: setattr(options, attr, getattr(initial_options, attr) or getattr(options, attr)) options.quiet += initial_options.quiet options.verbose += initial_options.verbose def setup_logging(self): pass def main(self, complete_args, args, initial_options): options, args = self.parser.parse_args(args) self.merge_options(initial_options, options) level = 1 # Notify level += options.verbose level -= options.quiet level = logger.level_for_integer(4 - level) complete_log = [] logger.consumers.extend([(level, sys.stdout), (logger.DEBUG, complete_log.append)]) if options.log_explicit_levels: logger.explicit_levels = True self.setup_logging() if options.require_venv and not options.venv: # If a venv is required check if it can really be found if not os.environ.get('VIRTUAL_ENV'): logger.fatal( 'Could not find an activated virtualenv (required).') sys.exit(3) # Automatically install in currently activated venv if required options.respect_venv = True if args and args[-1] == '___VENV_RESTART___': ## FIXME: We don't do anything this this value yet: args = args[:-2] options.venv = None else: # If given the option to respect the activated environment # check if no venv is given as a command line parameter if options.respect_venv and os.environ.get('VIRTUAL_ENV'): if options.venv and os.path.exists(options.venv): # Make sure command line venv and environmental are the same if (os.path.realpath(os.path.expanduser(options.venv)) != os.path.realpath(os.environ.get('VIRTUAL_ENV'))): logger.fatal( "Given virtualenv (%s) doesn't match " "currently activated virtualenv (%s)." % (options.venv, os.environ.get('VIRTUAL_ENV'))) sys.exit(3) else: options.venv = os.environ.get('VIRTUAL_ENV') logger.info('Using already activated environment %s' % options.venv) if options.venv: logger.info('Running in environment %s' % options.venv) site_packages = False if options.site_packages: site_packages = True restart_in_venv(options.venv, options.venv_base, site_packages, complete_args) # restart_in_venv should actually never return, but for clarity... return ## FIXME: not sure if this sure come before or after venv restart if options.log: log_fp = open_logfile(options.log, 'a') logger.consumers.append((logger.DEBUG, log_fp)) else: log_fp = None socket.setdefaulttimeout(options.timeout or None) urlopen.setup(proxystr=options.proxy, prompting=not options.no_input) exit = 0 try: self.run(options, args) except (InstallationError, UninstallationError), e: logger.fatal(str(e)) logger.info('Exception information:\n%s' % format_exc()) exit = 1 except BadCommand, e: logger.fatal(str(e)) logger.info('Exception information:\n%s' % format_exc()) exit = 1
def unpack_http_url(link, location, download_cache, download_dir=None, session=None): if session is None: session = PipSession() temp_dir = tempfile.mkdtemp("-unpack", "pip-") temp_location = None target_url = link.url.split("#", 1)[0] already_cached = False cache_file = None cache_content_type_file = None download_hash = None # If a download cache is specified, is the file cached there? if download_cache: cache_file = os.path.join(download_cache, urllib.quote(target_url, "")) cache_content_type_file = cache_file + ".content-type" already_cached = os.path.exists(cache_file) and os.path.exists(cache_content_type_file) if not os.path.isdir(download_cache): create_download_cache_folder(download_cache) # If a download dir is specified, is the file already downloaded there? already_downloaded = None if download_dir: already_downloaded = os.path.join(download_dir, link.filename) if not os.path.exists(already_downloaded): already_downloaded = None # If already downloaded, does it's hash match? if already_downloaded: temp_location = already_downloaded content_type = mimetypes.guess_type(already_downloaded)[0] logger.notify("File was already downloaded %s" % already_downloaded) if link.hash: download_hash = _get_hash_from_file(temp_location, link) try: _check_hash(download_hash, link) except HashMismatch: logger.warn("Previously-downloaded file %s has bad hash, " "re-downloading." % temp_location) temp_location = None os.unlink(already_downloaded) already_downloaded = None # If not a valid download, let's confirm the cached file is valid if already_cached and not temp_location: with open(cache_content_type_file) as fp: content_type = fp.read().strip() temp_location = cache_file logger.notify("Using download cache from %s" % cache_file) if link.hash and link.hash_name: download_hash = _get_hash_from_file(cache_file, link) try: _check_hash(download_hash, link) except HashMismatch: logger.warn("Cached file %s has bad hash, " "re-downloading." % temp_location) temp_location = None os.unlink(cache_file) os.unlink(cache_content_type_file) already_cached = False # We don't have either a cached or a downloaded copy # let's download to a tmp dir if not temp_location: try: resp = session.get(target_url, stream=True) resp.raise_for_status() except requests.HTTPError as exc: logger.fatal("HTTP error %s while getting %s" % (exc.response.status_code, link)) raise content_type = resp.headers.get("content-type", "") filename = link.filename # fallback # Have a look at the Content-Disposition header for a better guess content_disposition = resp.headers.get("content-disposition") if content_disposition: type, params = cgi.parse_header(content_disposition) # We use ``or`` here because we don't want to use an "empty" value # from the filename param. filename = params.get("filename") or filename ext = splitext(filename)[1] if not ext: ext = mimetypes.guess_extension(content_type) if ext: filename += ext if not ext and link.url != resp.url: ext = os.path.splitext(resp.url)[1] if ext: filename += ext temp_location = os.path.join(temp_dir, filename) download_hash = _download_url(resp, link, temp_location) if link.hash and link.hash_name: _check_hash(download_hash, link) # a download dir is specified; let's copy the archive there if download_dir and not already_downloaded: _copy_file(temp_location, download_dir, content_type, link) # unpack the archive to the build dir location. even when only downloading # archives, they have to be unpacked to parse dependencies unpack_file(temp_location, location, content_type, link) # if using a download cache, cache it, if needed if cache_file and not already_cached: cache_download(cache_file, temp_location, content_type) if not (already_cached or already_downloaded): os.unlink(temp_location) os.rmdir(temp_dir)
def main(self, args, initial_options): options, args = self.parser.parse_args(args) self.merge_options(initial_options, options) level = 1 # Notify level += options.verbose level -= options.quiet level = logger.level_for_integer(4 - level) complete_log = [] logger.consumers.extend([(level, sys.stdout), (logger.DEBUG, complete_log.append)]) if options.log_explicit_levels: logger.explicit_levels = True self.setup_logging() if options.no_input: os.environ["PIP_NO_INPUT"] = "1" if options.exists_action: os.environ["PIP_EXISTS_ACTION"] = "".join(options.exists_action) if options.require_venv: # If a venv is required check if it can really be found if not os.environ.get("VIRTUAL_ENV"): logger.fatal("Could not find an activated virtualenv (required).") sys.exit(VIRTUALENV_NOT_FOUND) if options.log: log_fp = open_logfile(options.log, "a") logger.consumers.append((logger.DEBUG, log_fp)) else: log_fp = None socket.setdefaulttimeout(options.timeout or None) urlopen.setup(proxystr=options.proxy, prompting=not options.no_input) exit = SUCCESS store_log = False try: status = self.run(options, args) # FIXME: all commands should return an exit status # and when it is done, isinstance is not needed anymore if isinstance(status, int): exit = status except (InstallationError, UninstallationError): e = sys.exc_info()[1] logger.fatal(str(e)) logger.info("Exception information:\n%s" % format_exc()) store_log = True exit = ERROR except BadCommand: e = sys.exc_info()[1] logger.fatal(str(e)) logger.info("Exception information:\n%s" % format_exc()) store_log = True exit = ERROR except CommandError: e = sys.exc_info()[1] logger.fatal("ERROR: %s" % e) logger.info("Exception information:\n%s" % format_exc()) exit = ERROR except KeyboardInterrupt: logger.fatal("Operation cancelled by user") logger.info("Exception information:\n%s" % format_exc()) store_log = True exit = ERROR except: logger.fatal("Exception:\n%s" % format_exc()) store_log = True exit = UNKNOWN_ERROR if log_fp is not None: log_fp.close() if store_log: log_fn = options.log_file text = "\n".join(complete_log) logger.fatal("Storing complete log in %s" % log_fn) log_fp = open_logfile(log_fn, "w") log_fp.write(text) log_fp.close() return exit
def main(self, args, initial_options): options, args = self.parser.parse_args(args) self.merge_options(initial_options, options) level = 1 # Notify level += options.verbose level -= options.quiet level = logger.level_for_integer(4 - level) complete_log = [] logger.consumers.extend([(level, sys.stdout), (logger.DEBUG, complete_log.append)]) if options.log_explicit_levels: logger.explicit_levels = True self.setup_logging() #TODO: try to get these passing down from the command? # without resorting to os.environ to hold these. if options.no_input: os.environ['PIP_NO_INPUT'] = '1' if options.exists_action: os.environ['PIP_EXISTS_ACTION'] = ''.join(options.exists_action) if not ssl and options.insecure: os.environ['PIP_INSECURE'] = '1' if options.cert: os.environ['PIP_CERT'] = options.cert if options.require_venv: # If a venv is required check if it can really be found if not os.environ.get('VIRTUAL_ENV'): logger.fatal( 'Could not find an activated virtualenv (required).') sys.exit(VIRTUALENV_NOT_FOUND) if options.log: log_fp = open_logfile(options.log, 'a') logger.consumers.append((logger.DEBUG, log_fp)) else: log_fp = None socket.setdefaulttimeout(options.timeout or None) urlopen.setup(proxystr=options.proxy, prompting=not options.no_input) exit = SUCCESS store_log = False try: status = self.run(options, args) # FIXME: all commands should return an exit status # and when it is done, isinstance is not needed anymore if isinstance(status, int): exit = status except (InstallationError, UninstallationError): e = sys.exc_info()[1] logger.fatal(str(e)) logger.info('Exception information:\n%s' % format_exc()) store_log = True exit = ERROR except BadCommand: e = sys.exc_info()[1] logger.fatal(str(e)) logger.info('Exception information:\n%s' % format_exc()) store_log = True exit = ERROR except CommandError: e = sys.exc_info()[1] logger.fatal('ERROR: %s' % e) logger.info('Exception information:\n%s' % format_exc()) exit = ERROR except KeyboardInterrupt: logger.fatal('Operation cancelled by user') logger.info('Exception information:\n%s' % format_exc()) store_log = True exit = ERROR except: logger.fatal('Exception:\n%s' % format_exc()) store_log = True exit = UNKNOWN_ERROR if log_fp is not None: log_fp.close() if store_log: log_fn = options.log_file text = '\n'.join(complete_log) try: log_fp = open_logfile(log_fn, 'w') except IOError: temp = tempfile.NamedTemporaryFile(delete=False) log_fn = temp.name log_fp = open_logfile(log_fn, 'w') logger.fatal('Storing complete log in %s' % log_fn) log_fp.write(text) log_fp.close() return exit
def call_subprocess(cmd, show_stdout=True, filter_stdout=None, cwd=None, raise_on_returncode=True, command_level=logger.DEBUG, command_desc=None, extra_environ=None): if command_desc is None: cmd_parts = [] for part in cmd: if ' ' in part or '\n' in part or '"' in part or "'" in part: part = '"%s"' % part.replace('"', '\\"') cmd_parts.append(part) command_desc = ' '.join(cmd_parts) if show_stdout: stdout = None else: stdout = subprocess.PIPE logger.log(command_level, "Running command %s" % command_desc) env = os.environ.copy() if extra_environ: env.update(extra_environ) try: proc = subprocess.Popen( cmd, stderr=subprocess.STDOUT, stdin=None, stdout=stdout, cwd=cwd, env=env) except Exception: e = sys.exc_info()[1] logger.fatal( "Error %s while executing command %s" % (e, command_desc)) raise all_output = [] if stdout is not None: stdout = proc.stdout while 1: line = console_to_str(stdout.readline()) if not line: break line = line.rstrip() all_output.append(line + '\n') if filter_stdout: level = filter_stdout(line) if isinstance(level, tuple): level, line = level logger.log(level, line) if not logger.stdout_level_matches(level): logger.show_progress() else: logger.info(line) else: returned_stdout, returned_stderr = proc.communicate() all_output = [returned_stdout or ''] proc.wait() if proc.returncode: if raise_on_returncode: if all_output: logger.notify('Complete output from command %s:' % command_desc) logger.notify('\n'.join(all_output) + '\n----------------------------------------') raise InstallationError( "Command %s failed with error code %s in %s" % (command_desc, proc.returncode, cwd)) else: logger.warn( "Command %s had error code %s in %s" % (command_desc, proc.returncode, cwd)) if stdout is not None: return ''.join(all_output)
def prepare_files(self, finder, force_root_egg_info=False, bundle=False): """Prepare process. Create temp directories, download and/or unpack files.""" unnamed = list(self.unnamed_requirements) reqs = list(self.requirements.values()) while reqs or unnamed: if unnamed: req_to_install = unnamed.pop(0) else: req_to_install = reqs.pop(0) install = True best_installed = False not_found = None if not self.ignore_installed and not req_to_install.editable: req_to_install.check_if_exists() if req_to_install.satisfied_by: substitute = self.install_req_checker.get_available_substitute( req_to_install) # if the req_to_install is identified as the best available substitue # AND # ( no version with req_to_install.name has been installed # OR a different version of req_to_install.name has been installed # ) # then set the self.upgrade flag to True to install req_to_install if (req_to_install == substitute.requirement and (req_to_install.name not in self.install_req_checker.pre_installed or self.install_req_checker.pre_installed[ req_to_install.name].requirement is not req_to_install)): self.upgrade = True if self.upgrade: if not self.force_reinstall and not req_to_install.url: try: url = finder.find_requirement( req_to_install, self.upgrade) except BestVersionAlreadyInstalled: best_installed = True install = False except DistributionNotFound: not_found = sys.exc_info()[1] else: # Avoid the need to call find_requirement again req_to_install.url = url.url if not best_installed: #don't uninstall conflict if user install and conflict is not user install if not (self.use_user_site and not dist_in_usersite( req_to_install.satisfied_by)): req_to_install.conflicts_with = req_to_install.satisfied_by req_to_install.satisfied_by = None else: install = False if req_to_install.satisfied_by: if best_installed: logger.notify('Requirement already up-to-date: %s' % req_to_install) else: logger.notify('Requirement already satisfied ' '(use --upgrade to upgrade): %s' % req_to_install) if req_to_install.editable: logger.notify('Obtaining %s' % req_to_install) elif install: logger.notify('Downloading/unpacking %s' % req_to_install) logger.indent += 2 try: is_bundle = False if req_to_install.editable: if req_to_install.source_dir is None: location = req_to_install.build_location(self.src_dir) req_to_install.source_dir = location else: location = req_to_install.source_dir if not os.path.exists(self.build_dir): _make_build_dir(self.build_dir) req_to_install.update_editable(not self.is_download) if self.is_download: req_to_install.run_egg_info() req_to_install.archive(self.download_dir) else: req_to_install.run_egg_info() elif install: ##@@ if filesystem packages are not marked ##editable in a req, a non deterministic error ##occurs when the script attempts to unpack the ##build directory # NB: This call can result in the creation of a temporary build directory location = req_to_install.build_location( self.build_dir, not self.is_download) ## FIXME: is the existance of the checkout good enough to use it? I don't think so. unpack = True url = None if not os.path.exists(os.path.join(location, 'setup.py')): ## FIXME: this won't upgrade when there's an existing package unpacked in `location` if req_to_install.url is None: if not_found: raise not_found url = finder.find_requirement(req_to_install, upgrade=self.upgrade) else: ## FIXME: should req_to_install.url already be a link? url = Link(req_to_install.url) assert url if url: try: self.unpack_url(url, location, self.is_download) except HTTPError: e = sys.exc_info()[1] logger.fatal( 'Could not install requirement %s because of error %s' % (req_to_install, e)) raise InstallationError( 'Could not install requirement %s because of HTTP error %s for URL %s' % (req_to_install, e, url)) else: unpack = False if unpack: is_bundle = req_to_install.is_bundle if is_bundle: req_to_install.move_bundle_files( self.build_dir, self.src_dir) for subreq in req_to_install.bundle_requirements(): reqs.append(subreq) self.add_requirement(subreq) elif self.is_download: req_to_install.source_dir = location req_to_install.run_egg_info() if url and url.scheme in vcs.all_schemes: req_to_install.archive(self.download_dir) else: req_to_install.source_dir = location req_to_install.run_egg_info() if force_root_egg_info: # We need to run this to make sure that the .egg-info/ # directory is created for packing in the bundle req_to_install.run_egg_info( force_root_egg_info=True) req_to_install.assert_source_matches_version() #@@ sketchy way of identifying packages not grabbed from an index if bundle and req_to_install.url: self.copy_to_build_dir(req_to_install) install = False # req_to_install.req is only avail after unpack for URL pkgs # repeat check_if_exists to uninstall-on-upgrade (#14) req_to_install.check_if_exists() if req_to_install.satisfied_by: if self.upgrade or self.ignore_installed: #don't uninstall conflict if user install and and conflict is not user install if not (self.use_user_site and not dist_in_usersite( req_to_install.satisfied_by)): req_to_install.conflicts_with = req_to_install.satisfied_by req_to_install.satisfied_by = None else: install = False if not is_bundle: ## FIXME: shouldn't be globally added: finder.add_dependency_links( req_to_install.dependency_links) if (req_to_install.extras): logger.notify("Installing extra requirements: %r" % ','.join(req_to_install.extras)) if not self.ignore_dependencies: for req in req_to_install.requirements( req_to_install.extras): try: name = pkg_resources.Requirement.parse( req).project_name except ValueError: e = sys.exc_info()[1] ## FIXME: proper warning logger.error( 'Invalid requirement: %r (%s) in requirement %s' % (req, e, req_to_install)) continue if self.has_requirement(name): ## FIXME: check for conflict continue subreq = InstallRequirement(req, req_to_install) reqs.append(subreq) self.add_requirement(subreq) if req_to_install.editable and req_to_install.source_dir: for subreq in self.install_requirements_txt( req_to_install): if self.add_requirement(subreq): reqs.append(subreq) if not self.has_requirement(req_to_install.name): #'unnamed' requirements will get added here self.add_requirement(req_to_install) if self.is_download or req_to_install._temp_build_dir is not None: self.reqs_to_cleanup.append(req_to_install) else: self.reqs_to_cleanup.append(req_to_install) if install: self.successfully_downloaded.append(req_to_install) if bundle and (req_to_install.url and req_to_install.url.startswith('file:///')): self.copy_to_build_dir(req_to_install) finally: logger.indent -= 2
def unpack_http_url(link, location, download_cache, download_dir=None, session=None): if session is None: session = PipSession() temp_dir = tempfile.mkdtemp('-unpack', 'pip-') temp_location = None target_url = link.url.split('#', 1)[0] already_cached = False cache_file = None cache_content_type_file = None download_hash = None # If a download cache is specified, is the file cached there? if download_cache: cache_file = os.path.join( download_cache, urllib.quote(target_url, '') ) cache_content_type_file = cache_file + '.content-type' already_cached = ( os.path.exists(cache_file) and os.path.exists(cache_content_type_file) ) if not os.path.isdir(download_cache): create_download_cache_folder(download_cache) # If a download dir is specified, is the file already downloaded there? already_downloaded = None if download_dir: already_downloaded = os.path.join(download_dir, link.filename) if not os.path.exists(already_downloaded): already_downloaded = None # If already downloaded, does its hash match? if already_downloaded: temp_location = already_downloaded content_type = mimetypes.guess_type(already_downloaded)[0] logger.notify('File was already downloaded %s' % already_downloaded) if link.hash: download_hash = _get_hash_from_file(temp_location, link) try: _check_hash(download_hash, link) except HashMismatch: logger.warn( 'Previously-downloaded file %s has bad hash, ' 're-downloading.' % temp_location ) temp_location = None os.unlink(already_downloaded) already_downloaded = None # If not a valid download, let's confirm the cached file is valid if already_cached and not temp_location: with open(cache_content_type_file) as fp: content_type = fp.read().strip() temp_location = cache_file logger.notify('Using download cache from %s' % cache_file) if link.hash and link.hash_name: download_hash = _get_hash_from_file(cache_file, link) try: _check_hash(download_hash, link) except HashMismatch: logger.warn( 'Cached file %s has bad hash, ' 're-downloading.' % temp_location ) temp_location = None os.unlink(cache_file) os.unlink(cache_content_type_file) already_cached = False # We don't have either a cached or a downloaded copy # let's download to a tmp dir if not temp_location: try: resp = session.get( target_url, # We use Accept-Encoding: identity here because requests # defaults to accepting compressed responses. This breaks in # a variety of ways depending on how the server is configured. # - Some servers will notice that the file isn't a compressible # file and will leave the file alone and with an empty # Content-Encoding # - Some servers will notice that the file is already # compressed and will leave the file alone and will add a # Content-Encoding: gzip header # - Some servers won't notice anything at all and will take # a file that's already been compressed and compress it again # and set the Content-Encoding: gzip header # By setting this to request only the identity encoding We're # hoping to eliminate the third case. Hopefully there does not # exist a server which when given a file will notice it is # already compressed and that you're not asking for a # compressed file and will then decompress it before sending # because if that's the case I don't think it'll ever be # possible to make this work. headers={"Accept-Encoding": "identity"}, stream=True, ) resp.raise_for_status() except requests.HTTPError as exc: logger.fatal("HTTP error %s while getting %s" % (exc.response.status_code, link)) raise content_type = resp.headers.get('content-type', '') filename = link.filename # fallback # Have a look at the Content-Disposition header for a better guess content_disposition = resp.headers.get('content-disposition') if content_disposition: type, params = cgi.parse_header(content_disposition) # We use ``or`` here because we don't want to use an "empty" value # from the filename param. filename = params.get('filename') or filename ext = splitext(filename)[1] if not ext: ext = mimetypes.guess_extension(content_type) if ext: filename += ext if not ext and link.url != resp.url: ext = os.path.splitext(resp.url)[1] if ext: filename += ext temp_location = os.path.join(temp_dir, filename) download_hash = _download_url(resp, link, temp_location) if link.hash and link.hash_name: _check_hash(download_hash, link) # a download dir is specified; let's copy the archive there if download_dir and not already_downloaded: _copy_file(temp_location, download_dir, content_type, link) # unpack the archive to the build dir location. even when only downloading # archives, they have to be unpacked to parse dependencies unpack_file(temp_location, location, content_type, link) # if using a download cache, cache it, if needed if cache_file and not already_cached: cache_download(cache_file, temp_location, content_type) if not (already_cached or already_downloaded): os.unlink(temp_location) os.rmdir(temp_dir)
def main(self, args, initial_options): options, args = self.parser.parse_args(args) self.merge_options(initial_options, options) level = 1 # Notify level += options.verbose level -= options.quiet level = logger.level_for_integer(4 - level) complete_log = [] logger.consumers.extend( [(level, sys.stdout), (logger.DEBUG, complete_log.append)]) if options.log_explicit_levels: logger.explicit_levels = True self.setup_logging() #TODO: try to get these passing down from the command? # without resorting to os.environ to hold these. if options.no_input: os.environ['PIP_NO_INPUT'] = '1' if options.exists_action: os.environ['PIP_EXISTS_ACTION'] = ''.join(options.exists_action) if not ssl and options.insecure: os.environ['PIP_INSECURE'] = '1' if options.cert: os.environ['PIP_CERT'] = options.cert if options.require_venv: # If a venv is required check if it can really be found if not os.environ.get('VIRTUAL_ENV'): logger.fatal('Could not find an activated virtualenv (required).') sys.exit(VIRTUALENV_NOT_FOUND) if options.log: log_fp = open_logfile(options.log, 'a') logger.consumers.append((logger.DEBUG, log_fp)) else: log_fp = None socket.setdefaulttimeout(options.timeout or None) urlopen.setup(proxystr=options.proxy, prompting=not options.no_input) exit = SUCCESS store_log = False try: status = self.run(options, args) # FIXME: all commands should return an exit status # and when it is done, isinstance is not needed anymore if isinstance(status, int): exit = status except (InstallationError, UninstallationError): e = sys.exc_info()[1] logger.fatal(str(e)) logger.info('Exception information:\n%s' % format_exc()) store_log = True exit = ERROR except BadCommand: e = sys.exc_info()[1] logger.fatal(str(e)) logger.info('Exception information:\n%s' % format_exc()) store_log = True exit = ERROR except CommandError: e = sys.exc_info()[1] logger.fatal('ERROR: %s' % e) logger.info('Exception information:\n%s' % format_exc()) exit = ERROR except KeyboardInterrupt: logger.fatal('Operation cancelled by user') logger.info('Exception information:\n%s' % format_exc()) store_log = True exit = ERROR except: logger.fatal('Exception:\n%s' % format_exc()) store_log = True exit = UNKNOWN_ERROR if log_fp is not None: log_fp.close() if store_log: log_fn = options.log_file text = '\n'.join(complete_log) try: log_fp = open_logfile(log_fn, 'w') except IOError: temp = tempfile.NamedTemporaryFile(delete=False) log_fn = temp.name log_fp = open_logfile(log_fn, 'w') logger.fatal('Storing complete log in %s' % log_fn) log_fp.write(text) log_fp.close() return exit
def _get_response_from_url(target_url, link): try: resp = urlopen(target_url) except urllib2.HTTPError, e: logger.fatal("HTTP error %s while getting %s" % (e.code, link)) raise
def prepare_files(self, finder, force_root_egg_info=False, bundle=False): """ Prepare process. Create temp directories, download and/or unpack files. """ unnamed = list(self.unnamed_requirements) reqs = list(self.requirements.values()) while reqs or unnamed: if unnamed: req_to_install = unnamed.pop(0) else: req_to_install = reqs.pop(0) install = True best_installed = False not_found = None # ############################################# # # # Search for archive to fulfill requirement # # # ############################################# # if not self.ignore_installed and not req_to_install.editable: req_to_install.check_if_exists() if req_to_install.satisfied_by: if self.upgrade: if not self.force_reinstall and not req_to_install.url: try: url = finder.find_requirement( req_to_install, self.upgrade) except BestVersionAlreadyInstalled: best_installed = True install = False except DistributionNotFound as exc: not_found = exc else: # Avoid the need to call find_requirement again req_to_install.url = url.url if not best_installed: # don't uninstall conflict if user install and # conflict is not user install if not (self.use_user_site and not dist_in_usersite( req_to_install.satisfied_by )): req_to_install.conflicts_with = \ req_to_install.satisfied_by req_to_install.satisfied_by = None else: install = False if req_to_install.satisfied_by: if best_installed: logger.notify('Requirement already up-to-date: %s' % req_to_install) else: logger.notify('Requirement already satisfied ' '(use --upgrade to upgrade): %s' % req_to_install) if req_to_install.editable: logger.notify('Obtaining %s' % req_to_install) elif install: if (req_to_install.url and req_to_install.url.lower().startswith('file:')): logger.notify( 'Unpacking %s' % display_path(url_to_path(req_to_install.url)) ) else: logger.notify('Downloading/unpacking %s' % req_to_install) logger.indent += 2 # ################################ # # # vcs update or unpack archive # # # ################################ # try: is_bundle = False is_wheel = False if req_to_install.editable: if req_to_install.source_dir is None: location = req_to_install.build_location(self.src_dir) req_to_install.source_dir = location else: location = req_to_install.source_dir if not os.path.exists(self.build_dir): _make_build_dir(self.build_dir) req_to_install.update_editable(not self.is_download) if self.is_download: req_to_install.run_egg_info() req_to_install.archive(self.download_dir) else: req_to_install.run_egg_info() elif install: # @@ if filesystem packages are not marked # editable in a req, a non deterministic error # occurs when the script attempts to unpack the # build directory # NB: This call can result in the creation of a temporary # build directory location = req_to_install.build_location( self.build_dir, not self.is_download, ) unpack = True url = None # In the case where the req comes from a bundle, we should # assume a build dir exists and move on if req_to_install.from_bundle: pass # If a checkout exists, it's unwise to keep going. version # inconsistencies are logged later, but do not fail the # installation. elif os.path.exists(os.path.join(location, 'setup.py')): raise PreviousBuildDirError( "pip can't proceed with requirements '%s' due to a" " pre-existing buld directory (%s). This is likely" " due to a previous installation that failed. pip " "is being responsible and not assuming it can " "delete this. Please delete it and try again." % (req_to_install, location) ) else: # FIXME: this won't upgrade when there's an existing # package unpacked in `location` if req_to_install.url is None: if not_found: raise not_found url = finder.find_requirement( req_to_install, upgrade=self.upgrade, ) else: # FIXME: should req_to_install.url already be a # link? url = Link(req_to_install.url) assert url if url: try: if ( url.filename.endswith(wheel_ext) and self.wheel_download_dir ): # when doing 'pip wheel` download_dir = self.wheel_download_dir do_download = True else: download_dir = self.download_dir do_download = self.is_download self.unpack_url( url, location, download_dir, do_download, ) except HTTPError as exc: logger.fatal( 'Could not install requirement %s because ' 'of error %s' % (req_to_install, exc) ) raise InstallationError( 'Could not install requirement %s because ' 'of HTTP error %s for URL %s' % (req_to_install, exc, url) ) else: unpack = False if unpack: is_bundle = req_to_install.is_bundle is_wheel = url and url.filename.endswith(wheel_ext) if is_bundle: req_to_install.move_bundle_files( self.build_dir, self.src_dir, ) for subreq in req_to_install.bundle_requirements(): reqs.append(subreq) self.add_requirement(subreq) elif self.is_download: req_to_install.source_dir = location if not is_wheel: # FIXME:https://github.com/pypa/pip/issues/1112 req_to_install.run_egg_info() if url and url.scheme in vcs.all_schemes: req_to_install.archive(self.download_dir) elif is_wheel: req_to_install.source_dir = location req_to_install.url = url.url else: req_to_install.source_dir = location req_to_install.run_egg_info() if force_root_egg_info: # We need to run this to make sure that the # .egg-info/ directory is created for packing # in the bundle req_to_install.run_egg_info( force_root_egg_info=True, ) req_to_install.assert_source_matches_version() # @@ sketchy way of identifying packages not # grabbed from an index if bundle and req_to_install.url: self.copy_to_build_dir(req_to_install) install = False # req_to_install.req is only avail after unpack for URL # pkgs repeat check_if_exists to uninstall-on-upgrade # (#14) if not self.ignore_installed: req_to_install.check_if_exists() if req_to_install.satisfied_by: if self.upgrade or self.ignore_installed: # don't uninstall conflict if user install and # conflict is not user install if not (self.use_user_site and not dist_in_usersite( req_to_install.satisfied_by)): req_to_install.conflicts_with = \ req_to_install.satisfied_by req_to_install.satisfied_by = None else: logger.notify( 'Requirement already satisfied (use ' '--upgrade to upgrade): %s' % req_to_install ) install = False # ###################### # # # parse dependencies # # # ###################### # if is_wheel: dist = list( pkg_resources.find_distributions(location) )[0] if not req_to_install.req: req_to_install.req = dist.as_requirement() self.add_requirement(req_to_install) if not self.ignore_dependencies: for subreq in dist.requires( req_to_install.extras): if self.has_requirement( subreq.project_name): continue subreq = InstallRequirement(str(subreq), req_to_install) reqs.append(subreq) self.add_requirement(subreq) # sdists elif not is_bundle: if (req_to_install.extras): logger.notify( "Installing extra requirements: %r" % ','.join(req_to_install.extras) ) if not self.ignore_dependencies: for req in req_to_install.requirements( req_to_install.extras): try: name = pkg_resources.Requirement.parse( req ).project_name except ValueError as exc: # FIXME: proper warning logger.error( 'Invalid requirement: %r (%s) in ' 'requirement %s' % (req, exc, req_to_install) ) continue if self.has_requirement(name): # FIXME: check for conflict continue subreq = InstallRequirement(req, req_to_install) reqs.append(subreq) self.add_requirement(subreq) if not self.has_requirement(req_to_install.name): # 'unnamed' requirements will get added here self.add_requirement(req_to_install) # cleanup tmp src if not is_bundle: if ( self.is_download or req_to_install._temp_build_dir is not None ): self.reqs_to_cleanup.append(req_to_install) if install: self.successfully_downloaded.append(req_to_install) if (bundle and ( req_to_install.url and req_to_install.url.startswith('file:///') )): self.copy_to_build_dir(req_to_install) finally: logger.indent -= 2
def main(self, complete_args, args, initial_options): options, args = self.parser.parse_args(args) self.merge_options(initial_options, options) level = 1 # Notify level += options.verbose level -= options.quiet level = logger.level_for_integer(4-level) complete_log = [] logger.consumers.extend( [(level, sys.stdout), (logger.DEBUG, complete_log.append)]) if options.log_explicit_levels: logger.explicit_levels = True self.setup_logging() if options.require_venv and not options.venv: # If a venv is required check if it can really be found if not os.environ.get('VIRTUAL_ENV'): logger.fatal('Could not find an activated virtualenv (required).') sys.exit(3) # Automatically install in currently activated venv if required options.respect_venv = True if args and args[-1] == '___VENV_RESTART___': ## FIXME: We don't do anything this this value yet: args = args[:-2] options.venv = None else: # If given the option to respect the activated environment # check if no venv is given as a command line parameter if options.respect_venv and os.environ.get('VIRTUAL_ENV'): if options.venv and os.path.exists(options.venv): # Make sure command line venv and environmental are the same if (os.path.realpath(os.path.expanduser(options.venv)) != os.path.realpath(os.environ.get('VIRTUAL_ENV'))): logger.fatal("Given virtualenv (%s) doesn't match " "currently activated virtualenv (%s)." % (options.venv, os.environ.get('VIRTUAL_ENV'))) sys.exit(3) else: options.venv = os.environ.get('VIRTUAL_ENV') logger.info('Using already activated environment %s' % options.venv) if options.venv: logger.info('Running in environment %s' % options.venv) site_packages=False if options.site_packages: site_packages=True restart_in_venv(options.venv, options.venv_base, site_packages, complete_args) # restart_in_venv should actually never return, but for clarity... return ## FIXME: not sure if this sure come before or after venv restart if options.log: log_fp = open_logfile(options.log, 'a') logger.consumers.append((logger.DEBUG, log_fp)) else: log_fp = None socket.setdefaulttimeout(options.timeout or None) urlopen.setup(proxystr=options.proxy, prompting=not options.no_input) exit = 0 try: self.run(options, args) except (InstallationError, UninstallationError), e: logger.fatal(str(e)) logger.info('Exception information:\n%s' % format_exc()) exit = 1
def find_requirement(self, req, upgrade): url_name = req.url_name # Only check main index if index URL is given: main_index_url = None if self.index_urls: # Check that we have the url_name correctly spelled: main_index_url = Link(posixpath.join(self.index_urls[0], url_name)) # This will also cache the page, so it's okay that we get it again later: page = self._get_page(main_index_url, req) if page is None: url_name = self._find_url_name(Link(self.index_urls[0]), url_name, req) or req.url_name # Combine index URLs with mirror URLs here to allow # adding more index URLs from requirements files all_index_urls = self.index_urls + self.mirror_urls def mkurl_pypi_url(url): loc = posixpath.join(url, url_name) # For maximum compatibility with easy_install, ensure the path # ends in a trailing slash. Although this isn't in the spec # (and PyPI can handle it without the slash) some other index # implementations might break if they relied on easy_install's behavior. if not loc.endswith('/'): loc = loc + '/' return loc if url_name is not None: locations = [ mkurl_pypi_url(url) for url in all_index_urls] + self.find_links else: locations = list(self.find_links) locations.extend(self.dependency_links) for version in req.absolute_versions: if url_name is not None and main_index_url is not None: locations = [ posixpath.join(main_index_url.url, version)] + locations file_locations, url_locations = self._sort_locations(locations) locations = [Link(url) for url in url_locations] logger.debug('URLs to search for versions for %s:' % req) for location in locations: logger.debug('* %s' % location) found_versions = [] found_versions.extend( self._package_versions( [Link(url, '-f') for url in self.find_links], req.name.lower())) page_versions = [] for page in self._get_pages(locations, req): logger.debug('Analyzing links from page %s' % page.url) logger.indent += 2 try: page_versions.extend(self._package_versions(page.links, req.name.lower())) finally: logger.indent -= 2 dependency_versions = list(self._package_versions( [Link(url) for url in self.dependency_links], req.name.lower())) if dependency_versions: logger.info('dependency_links found: %s' % ', '.join([link.url for parsed, link, version in dependency_versions])) file_versions = list(self._package_versions( [Link(url) for url in file_locations], req.name.lower())) if not found_versions and not page_versions and not dependency_versions and not file_versions: logger.fatal('Could not find any downloads that satisfy the requirement %s' % req) raise DistributionNotFound('No distributions at all found for %s' % req) if req.satisfied_by is not None: found_versions.append((req.satisfied_by.parsed_version, Inf, req.satisfied_by.version)) if file_versions: file_versions.sort(reverse=True) logger.info('Local files found: %s' % ', '.join([url_to_path(link.url) for parsed, link, version in file_versions])) found_versions = file_versions + found_versions all_versions = found_versions + page_versions + dependency_versions applicable_versions = [] for (parsed_version, link, version) in all_versions: if version not in req.req: logger.info("Ignoring link %s, version %s doesn't match %s" % (link, version, ','.join([''.join(s) for s in req.req.specs]))) continue applicable_versions.append((link, version)) applicable_versions = sorted(applicable_versions, key=lambda v: pkg_resources.parse_version(v[1]), reverse=True) existing_applicable = bool([link for link, version in applicable_versions if link is Inf]) if not upgrade and existing_applicable: if applicable_versions[0][1] is Inf: logger.info('Existing installed version (%s) is most up-to-date and satisfies requirement' % req.satisfied_by.version) raise BestVersionAlreadyInstalled else: logger.info('Existing installed version (%s) satisfies requirement (most up-to-date version is %s)' % (req.satisfied_by.version, applicable_versions[0][1])) return None if not applicable_versions: logger.fatal('Could not find a version that satisfies the requirement %s (from versions: %s)' % (req, ', '.join([version for parsed_version, link, version in found_versions]))) raise DistributionNotFound('No distributions matching the version for %s' % req) if applicable_versions[0][0] is Inf: # We have an existing version, and its the best version logger.info('Installed version (%s) is most up-to-date (past versions: %s)' % (req.satisfied_by.version, ', '.join([version for link, version in applicable_versions[1:]]) or 'none')) raise BestVersionAlreadyInstalled if len(applicable_versions) > 1: logger.info('Using version %s (newest of versions: %s)' % (applicable_versions[0][1], ', '.join([version for link, version in applicable_versions]))) return applicable_versions[0][0]
def main(self, complete_args, args, initial_options): options, args = self.parser.parse_args(args) self.merge_options(initial_options, options) level = 1 # Notify level += options.verbose level -= options.quiet level = logger.level_for_integer(4-level) complete_log = [] logger.consumers.extend( [(level, sys.stdout), (logger.DEBUG, complete_log.append)]) if options.log_explicit_levels: logger.explicit_levels = True self.setup_logging() if options.require_venv: # If a venv is required check if it can really be found if not os.environ.get('VIRTUAL_ENV'): logger.fatal('Could not find an activated virtualenv (required).') sys.exit(3) if options.log: log_fp = open_logfile(options.log, 'a') logger.consumers.append((logger.DEBUG, log_fp)) else: log_fp = None socket.setdefaulttimeout(options.timeout or None) urlopen.setup(proxystr=options.proxy, prompting=not options.no_input) exit = 0 try: self.run(options, args) except (InstallationError, UninstallationError): e = sys.exc_info()[1] logger.fatal(str(e)) logger.info('Exception information:\n%s' % format_exc()) exit = 1 except BadCommand: e = sys.exc_info()[1] logger.fatal(str(e)) logger.info('Exception information:\n%s' % format_exc()) exit = 1 except KeyboardInterrupt: logger.fatal('Operation cancelled by user') logger.info('Exception information:\n%s' % format_exc()) exit = 1 except: logger.fatal('Exception:\n%s' % format_exc()) exit = 2 if log_fp is not None: log_fp.close() if exit: log_fn = options.log_file text = '\n'.join(complete_log) logger.fatal('Storing complete log in %s' % log_fn) log_fp = open_logfile(log_fn, 'w') log_fp.write(text) log_fp.close() return exit
def unpack_http_url(link, location, download_cache, download_dir=None, session=None): if session is None: session = PipSession() temp_dir = tempfile.mkdtemp('-unpack', 'pip-') temp_location = None target_url = link.url.split('#', 1)[0] already_cached = False cache_file = None cache_content_type_file = None download_hash = None # If a download cache is specified, is the file cached there? if download_cache: cache_file = os.path.join(download_cache, urllib.quote(target_url, '')) cache_content_type_file = cache_file + '.content-type' already_cached = (os.path.exists(cache_file) and os.path.exists(cache_content_type_file)) if not os.path.isdir(download_cache): create_download_cache_folder(download_cache) # If a download dir is specified, is the file already downloaded there? already_downloaded = None if download_dir: already_downloaded = os.path.join(download_dir, link.filename) if not os.path.exists(already_downloaded): already_downloaded = None # If already downloaded, does it's hash match? if already_downloaded: temp_location = already_downloaded content_type = mimetypes.guess_type(already_downloaded)[0] logger.notify('File was already downloaded %s' % already_downloaded) if link.hash: download_hash = _get_hash_from_file(temp_location, link) try: _check_hash(download_hash, link) except HashMismatch: logger.warn('Previously-downloaded file %s has bad hash, ' 're-downloading.' % temp_location) temp_location = None os.unlink(already_downloaded) already_downloaded = None # If not a valid download, let's confirm the cached file is valid if already_cached and not temp_location: with open(cache_content_type_file) as fp: content_type = fp.read().strip() temp_location = cache_file logger.notify('Using download cache from %s' % cache_file) if link.hash and link.hash_name: download_hash = _get_hash_from_file(cache_file, link) try: _check_hash(download_hash, link) except HashMismatch: logger.warn('Cached file %s has bad hash, ' 're-downloading.' % temp_location) temp_location = None os.unlink(cache_file) os.unlink(cache_content_type_file) already_cached = False # We don't have either a cached or a downloaded copy # let's download to a tmp dir if not temp_location: try: resp = session.get(target_url, stream=True) resp.raise_for_status() except requests.HTTPError as exc: logger.fatal("HTTP error %s while getting %s" % (exc.response.status_code, link)) raise content_type = resp.headers.get('content-type', '') filename = link.filename # fallback # Have a look at the Content-Disposition header for a better guess content_disposition = resp.headers.get('content-disposition') if content_disposition: type, params = cgi.parse_header(content_disposition) # We use ``or`` here because we don't want to use an "empty" value # from the filename param. filename = params.get('filename') or filename ext = splitext(filename)[1] if not ext: ext = mimetypes.guess_extension(content_type) if ext: filename += ext if not ext and link.url != resp.url: ext = os.path.splitext(resp.url)[1] if ext: filename += ext temp_location = os.path.join(temp_dir, filename) download_hash = _download_url(resp, link, temp_location) if link.hash and link.hash_name: _check_hash(download_hash, link) # a download dir is specified; let's copy the archive there if download_dir and not already_downloaded: _copy_file(temp_location, download_dir, content_type, link) # unpack the archive to the build dir location. even when only downloading # archives, they have to be unpacked to parse dependencies unpack_file(temp_location, location, content_type, link) # if using a download cache, cache it, if needed if cache_file and not already_cached: cache_download(cache_file, temp_location, content_type) if not (already_cached or already_downloaded): os.unlink(temp_location) os.rmdir(temp_dir)
def find_requirement(self, req, upgrade): url_name = req.url_name # Only check main index if index URL is given: main_index_url = None if self.index_urls: # Check that we have the url_name correctly spelled: main_index_url = Link(posixpath.join(self.index_urls[0], url_name)) # This will also cache the page, so it's okay that we get it again later: page = self._get_page(main_index_url, req) if page is None: url_name = self._find_url_name(Link(self.index_urls[0]), url_name, req) or req.url_name # Combine index URLs with mirror URLs here to allow # adding more index URLs from requirements files all_index_urls = self.index_urls + self.mirror_urls def mkurl_pypi_url(url): loc = posixpath.join(url, url_name) # For maximum compatibility with easy_install, ensure the path # ends in a trailing slash. Although this isn't in the spec # (and PyPI can handle it without the slash) some other index # implementations might break if they relied on easy_install's behavior. if not loc.endswith('/'): loc = loc + '/' return loc if url_name is not None: locations = [mkurl_pypi_url(url) for url in all_index_urls] + self.find_links else: locations = list(self.find_links) locations.extend(self.dependency_links) for version in req.absolute_versions: if url_name is not None and main_index_url is not None: locations = [posixpath.join(main_index_url.url, version) ] + locations file_locations, url_locations = self._sort_locations(locations) locations = [Link(url) for url in url_locations] logger.debug('URLs to search for versions for %s:' % req) for location in locations: logger.debug('* %s' % location) found_versions = [] found_versions.extend( self._package_versions( [Link(url, '-f') for url in self.find_links], req.name.lower())) page_versions = [] for page in self._get_pages(locations, req): logger.debug('Analyzing links from page %s' % page.url) logger.indent += 2 try: page_versions.extend( self._package_versions(page.links, req.name.lower())) finally: logger.indent -= 2 dependency_versions = list( self._package_versions( [Link(url) for url in self.dependency_links], req.name.lower())) if dependency_versions: logger.info('dependency_links found: %s' % ', '.join( [link.url for parsed, link, version in dependency_versions])) file_versions = list( self._package_versions([Link(url) for url in file_locations], req.name.lower())) if not found_versions and not page_versions and not dependency_versions and not file_versions: logger.fatal( 'Could not find any downloads that satisfy the requirement %s' % req) raise DistributionNotFound('No distributions at all found for %s' % req) if req.satisfied_by is not None: found_versions.append((req.satisfied_by.parsed_version, Inf, req.satisfied_by.version)) if file_versions: file_versions.sort(reverse=True) logger.info('Local files found: %s' % ', '.join([ url_to_path(link.url) for parsed, link, version in file_versions ])) found_versions = file_versions + found_versions all_versions = found_versions + page_versions + dependency_versions applicable_versions = [] for (parsed_version, link, version) in all_versions: if version not in req.req: logger.info("Ignoring link %s, version %s doesn't match %s" % (link, version, ','.join( [''.join(s) for s in req.req.specs]))) continue applicable_versions.append((link, version)) applicable_versions = sorted( applicable_versions, key=lambda v: pkg_resources.parse_version(v[1]), reverse=True) existing_applicable = bool( [link for link, version in applicable_versions if link is Inf]) if not upgrade and existing_applicable: if applicable_versions[0][1] is Inf: logger.info( 'Existing installed version (%s) is most up-to-date and satisfies requirement' % req.satisfied_by.version) raise BestVersionAlreadyInstalled else: logger.info( 'Existing installed version (%s) satisfies requirement (most up-to-date version is %s)' % (req.satisfied_by.version, applicable_versions[0][1])) return None if not applicable_versions: logger.fatal( 'Could not find a version that satisfies the requirement %s (from versions: %s)' % (req, ', '.join([ version for parsed_version, link, version in found_versions ]))) raise DistributionNotFound( 'No distributions matching the version for %s' % req) if applicable_versions[0][0] is Inf: # We have an existing version, and its the best version logger.info( 'Installed version (%s) is most up-to-date (past versions: %s)' % (req.satisfied_by.version, ', '.join( [version for link, version in applicable_versions[1:]]) or 'none')) raise BestVersionAlreadyInstalled if len(applicable_versions) > 1: logger.info('Using version %s (newest of versions: %s)' % (applicable_versions[0][1], ', '.join( [version for link, version in applicable_versions]))) return applicable_versions[0][0]
def prepare_files(self, finder, force_root_egg_info=False, bundle=False): """Prepare process. Create temp directories, download and/or unpack files.""" unnamed = list(self.unnamed_requirements) reqs = list(self.requirements.values()) while reqs or unnamed: if unnamed: req_to_install = unnamed.pop(0) else: req_to_install = reqs.pop(0) install = True best_installed = False if not self.ignore_installed and not req_to_install.editable: req_to_install.check_if_exists() if req_to_install.satisfied_by: if self.upgrade: if not self.force_reinstall: try: url = finder.find_requirement( req_to_install, self.upgrade) except BestVersionAlreadyInstalled: best_installed = True install = False else: # Avoid the need to call find_requirement again req_to_install.url = url.url if not best_installed: req_to_install.conflicts_with = req_to_install.satisfied_by req_to_install.satisfied_by = None else: install = False if req_to_install.satisfied_by: if best_installed: logger.notify('Requirement already up-to-date: %s' % req_to_install) else: logger.notify('Requirement already satisfied ' '(use --upgrade to upgrade): %s' % req_to_install) if req_to_install.editable: logger.notify('Obtaining %s' % req_to_install) elif install: if req_to_install.url and req_to_install.url.lower().startswith('file:'): logger.notify('Unpacking %s' % display_path(url_to_path(req_to_install.url))) else: logger.notify('Downloading/unpacking %s' % req_to_install) logger.indent += 2 try: is_bundle = False if req_to_install.editable: if req_to_install.source_dir is None: location = req_to_install.build_location(self.src_dir) req_to_install.source_dir = location else: location = req_to_install.source_dir if not os.path.exists(self.build_dir): _make_build_dir(self.build_dir) req_to_install.update_editable(not self.is_download) if self.is_download: req_to_install.run_egg_info() req_to_install.archive(self.download_dir) else: req_to_install.run_egg_info() elif install: ##@@ if filesystem packages are not marked ##editable in a req, a non deterministic error ##occurs when the script attempts to unpack the ##build directory location = req_to_install.build_location(self.build_dir, not self.is_download) ## FIXME: is the existance of the checkout good enough to use it? I don't think so. unpack = True url = None if not os.path.exists(os.path.join(location, 'setup.py')): ## FIXME: this won't upgrade when there's an existing package unpacked in `location` if req_to_install.url is None: url = finder.find_requirement(req_to_install, upgrade=self.upgrade) else: ## FIXME: should req_to_install.url already be a link? url = Link(req_to_install.url) assert url if url: try: self.unpack_url(url, location, self.is_download) except HTTPError: e = sys.exc_info()[1] logger.fatal('Could not install requirement %s because of error %s' % (req_to_install, e)) raise InstallationError( 'Could not install requirement %s because of HTTP error %s for URL %s' % (req_to_install, e, url)) else: unpack = False if unpack: is_bundle = req_to_install.is_bundle if is_bundle: req_to_install.move_bundle_files(self.build_dir, self.src_dir) for subreq in req_to_install.bundle_requirements(): reqs.append(subreq) self.add_requirement(subreq) elif self.is_download: req_to_install.source_dir = location req_to_install.run_egg_info() if url and url.scheme in vcs.all_schemes: req_to_install.archive(self.download_dir) else: req_to_install.source_dir = location req_to_install.run_egg_info() if force_root_egg_info: # We need to run this to make sure that the .egg-info/ # directory is created for packing in the bundle req_to_install.run_egg_info(force_root_egg_info=True) req_to_install.assert_source_matches_version() #@@ sketchy way of identifying packages not grabbed from an index if bundle and req_to_install.url: self.copy_to_build_dir(req_to_install) install = False # req_to_install.req is only avail after unpack for URL pkgs # repeat check_if_exists to uninstall-on-upgrade (#14) req_to_install.check_if_exists() if req_to_install.satisfied_by: if self.upgrade or self.ignore_installed: req_to_install.conflicts_with = req_to_install.satisfied_by req_to_install.satisfied_by = None else: install = False if not is_bundle: ## FIXME: shouldn't be globally added: finder.add_dependency_links(req_to_install.dependency_links) if (req_to_install.extras): logger.notify("Installing extra requirements: %r" % ','.join(req_to_install.extras)) if not self.ignore_dependencies: for req in req_to_install.requirements(req_to_install.extras): try: name = pkg_resources.Requirement.parse(req).project_name except ValueError: e = sys.exc_info()[1] ## FIXME: proper warning logger.error('Invalid requirement: %r (%s) in requirement %s' % (req, e, req_to_install)) continue subreq = InstallRequirement(req, req_to_install) if self.has_requirement(name): investigate.append([ self.get_requirement(name), subreq ]) continue reqs.append(subreq) self.add_requirement(subreq) if req_to_install.name not in self.requirements: self.requirements[req_to_install.name] = req_to_install if self.is_download: self.reqs_to_cleanup.append(req_to_install) else: self.reqs_to_cleanup.append(req_to_install) if install: self.successfully_downloaded.append(req_to_install) if bundle and (req_to_install.url and req_to_install.url.startswith('file:///')): self.copy_to_build_dir(req_to_install) finally: logger.indent -= 2