def move_bundle_files(self, dest_build_dir, dest_src_dir): base = self._temp_build_dir assert base src_dir = os.path.join(base, 'src') build_dir = os.path.join(base, 'build') bundle_build_dirs = [] bundle_editable_dirs = [] for source_dir, dest_dir, dir_collection in [ (src_dir, dest_src_dir, bundle_editable_dirs), (build_dir, dest_build_dir, bundle_build_dirs)]: if os.path.exists(source_dir): for dirname in os.listdir(source_dir): dest = os.path.join(dest_dir, dirname) dir_collection.append(dest) if os.path.exists(dest): logger.warn( 'The directory %s (containing package %s) already ' 'exists; cannot move source from bundle %s' % (dest, dirname, self) ) continue if not os.path.exists(dest_dir): logger.info('Creating directory %s' % dest_dir) os.makedirs(dest_dir) shutil.move(os.path.join(source_dir, dirname), dest) if not os.listdir(source_dir): os.rmdir(source_dir) self._temp_build_dir = None self._bundle_build_dirs = bundle_build_dirs self._bundle_editable_dirs = bundle_editable_dirs
def list(self, options, args): if args: raise InstallationError("You cannot give an argument with --list") for path in sorted(self.paths()): if not os.path.exists(path): continue basename = os.path.basename(path.rstrip(os.path.sep)) if os.path.isfile(path) and zipfile.is_zipfile(path): if os.path.dirname(path) not in self.paths(): logger.notify("Zipped egg: %s" % display_path(path)) continue if ( basename != "site-packages" and basename != "dist-packages" and not path.replace("\\", "/").endswith("lib/python") ): continue logger.notify("In %s:" % display_path(path)) logger.indent += 2 zipped = [] unzipped = [] try: for filename in sorted(os.listdir(path)): ext = os.path.splitext(filename)[1].lower() if ext in (".pth", ".egg-info", ".egg-link"): continue if ext == ".py": logger.info("Not displaying %s: not a package" % display_path(filename)) continue full = os.path.join(path, filename) if os.path.isdir(full): unzipped.append((filename, self.count_package(full))) elif zipfile.is_zipfile(full): zipped.append(filename) else: logger.info("Unknown file: %s" % display_path(filename)) if zipped: logger.notify("Zipped packages:") logger.indent += 2 try: for filename in zipped: logger.notify(filename) finally: logger.indent -= 2 else: logger.notify("No zipped packages.") if unzipped: if options.sort_files: unzipped.sort(key=lambda x: -x[1]) logger.notify("Unzipped packages:") logger.indent += 2 try: for filename, count in unzipped: logger.notify("%s (%i files)" % (filename, count)) finally: logger.indent -= 2 else: logger.notify("No unzipped packages.") finally: logger.indent -= 2
def cmd(self): if self._cmd is not None: return self._cmd command = find_command(self.name) logger.info('Found command {0!r} at {1!r}'.format(self.name, command)) self._cmd = command return command
def cmd(self): if self._cmd is not None: return self._cmd command = find_command(self.name) logger.info('Found command %r at %r' % (self.name, command)) self._cmd = command return command
def get_info(self, location): """Returns (url, revision), where both are strings""" assert not location.rstrip('/').endswith(self.dirname), \ 'Bad directory: %s' % location output = call_subprocess( [self.cmd, 'info', location], show_stdout=False, extra_environ={'LANG': 'C'}, ) match = _svn_url_re.search(output) if not match: logger.warn( 'Cannot determine URL of svn checkout %s' % display_path(location) ) return None, None url = match.group(1).strip() match = _svn_revision_re.search(output) if not match: logger.warn( 'Cannot determine revision of svn checkout %s' % display_path(location) ) logger.info('Output that cannot be parsed: \n%s' % output) return url, None return url, match.group(1)
def remove_filename_from_pth(self, filename): for pth in self.pth_files(): f = open(pth, 'r') lines = f.readlines() f.close() new_lines = [ l for l in lines if l.strip() != filename] if lines != new_lines: logger.info('Removing reference to %s from .pth file %s' % (display_path(filename), display_path(pth))) if not [line for line in new_lines if line]: logger.info( '%s file would be empty: deleting' % display_path(pth) ) if not self.simulate: os.unlink(pth) else: if not self.simulate: f = open(pth, 'wb') f.writelines(new_lines) f.close() return logger.warn( 'Cannot find a reference to %s in any .pth file' % display_path(filename) )
def move_bundle_files(self, dest_build_dir, dest_src_dir): base = self._temp_build_dir assert base src_dir = os.path.join(base, 'src') build_dir = os.path.join(base, 'build') bundle_build_dirs = [] bundle_editable_dirs = [] for source_dir, dest_dir, dir_collection in [ (src_dir, dest_src_dir, bundle_editable_dirs), (build_dir, dest_build_dir, bundle_build_dirs) ]: if os.path.exists(source_dir): for dirname in os.listdir(source_dir): dest = os.path.join(dest_dir, dirname) dir_collection.append(dest) if os.path.exists(dest): logger.warn( 'The directory %s (containing package %s) already ' 'exists; cannot move source from bundle %s' % (dest, dirname, self)) continue if not os.path.exists(dest_dir): logger.info('Creating directory %s' % dest_dir) os.makedirs(dest_dir) shutil.move(os.path.join(source_dir, dirname), dest) if not os.listdir(source_dir): os.rmdir(source_dir) self._temp_build_dir = None self._bundle_build_dirs = bundle_build_dirs self._bundle_editable_dirs = bundle_editable_dirs
def update_editable(self, obtain=True): if not self.url: logger.info( "Cannot update repository at %s; repository location is " "unknown" % self.source_dir ) return assert self.editable assert self.source_dir if self.url.startswith('file:'): # Static paths don't get updated return assert '+' in self.url, "bad url: %r" % self.url if not self.update: return vc_type, url = self.url.split('+', 1) backend = vcs.get_backend(vc_type) if backend: vcs_backend = backend(self.url) if obtain: vcs_backend.obtain(self.source_dir) else: vcs_backend.export(self.source_dir) else: assert 0, ( 'Unexpected version control type (in %s): %s' % (self.url, vc_type))
def update_editable(self, obtain=True): if not self.url: logger.info( "Cannot update repository at %s; repository location is " "unknown" % self.source_dir) return assert self.editable assert self.source_dir if self.url.startswith('file:'): # Static paths don't get updated return assert '+' in self.url, "bad url: %r" % self.url if not self.update: return vc_type, url = self.url.split('+', 1) backend = vcs.get_backend(vc_type) if backend: vcs_backend = backend(self.url) if obtain: vcs_backend.obtain(self.source_dir) else: vcs_backend.export(self.source_dir) else: assert 0, ('Unexpected version control type (in %s): %s' % (self.url, vc_type))
def copy_to_build_dir(self, req_to_install): target_dir = req_to_install.editable and self.src_dir or self.build_dir logger.info("Copying %s to %s" % (req_to_install.name, target_dir)) dest = os.path.join(target_dir, req_to_install.name) shutil.copytree(req_to_install.source_dir, dest) call_subprocess(["python", "%s/setup.py" % dest, "clean"], cwd=dest, command_desc='python setup.py clean')
def remove(self, auto_confirm=False): """Remove paths in ``self.paths`` with confirmation (unless ``auto_confirm`` is True).""" if not self._can_uninstall(): return if not self.paths: logger.notify("Can't uninstall '%s'. No files were found to uninstall." % self.dist.project_name) return logger.notify("Uninstalling %s:" % self.dist.project_name) logger.indent += 2 paths = sorted(self.compact(self.paths)) try: if auto_confirm: response = "y" else: for path in paths: logger.notify(path) response = ask("Proceed (y/n)? ", ("y", "n")) if self._refuse: logger.notify("Not removing or modifying (outside of prefix):") for path in self.compact(self._refuse): logger.notify(path) if response == "y": self.save_dir = tempfile.mkdtemp(suffix="-uninstall", prefix="pip-") for path in paths: new_path = self._stash(path) logger.info("Removing file or directory %s" % path) self._moved_paths.append(path) renames(path, new_path) for pth in self.pth.values(): pth.remove() logger.notify("Successfully uninstalled %s" % self.dist.project_name) finally: logger.indent -= 2
def list(self, options, args): if args: raise InstallationError('You cannot give an argument with --list') for path in sorted(self.paths()): if not os.path.exists(path): continue basename = os.path.basename(path.rstrip(os.path.sep)) if os.path.isfile(path) and zipfile.is_zipfile(path): if os.path.dirname(path) not in self.paths(): logger.notify('Zipped egg: %s' % display_path(path)) continue if (basename != 'site-packages' and basename != 'dist-packages' and not path.replace('\\', '/').endswith('lib/python')): continue logger.notify('In %s:' % display_path(path)) logger.indent += 2 zipped = [] unzipped = [] try: for filename in sorted(os.listdir(path)): ext = os.path.splitext(filename)[1].lower() if ext in ('.pth', '.egg-info', '.egg-link'): continue if ext == '.py': logger.info('Not displaying %s: not a package' % display_path(filename)) continue full = os.path.join(path, filename) if os.path.isdir(full): unzipped.append((filename, self.count_package(full))) elif zipfile.is_zipfile(full): zipped.append(filename) else: logger.info('Unknown file: %s' % display_path(filename)) if zipped: logger.notify('Zipped packages:') logger.indent += 2 try: for filename in zipped: logger.notify(filename) finally: logger.indent -= 2 else: logger.notify('No zipped packages.') if unzipped: if options.sort_files: unzipped.sort(key=lambda x: -x[1]) logger.notify('Unzipped packages:') logger.indent += 2 try: for filename, count in unzipped: logger.notify('%s (%i files)' % (filename, count)) finally: logger.indent -= 2 else: logger.notify('No unzipped packages.') finally: logger.indent -= 2
def list(self, options, args): if args: raise InstallationError( 'You cannot give an argument with --list') for path in sorted(self.paths()): if not os.path.exists(path): continue basename = os.path.basename(path.rstrip(os.path.sep)) if os.path.isfile(path) and zipfile.is_zipfile(path): if os.path.dirname(path) not in self.paths(): logger.notify('Zipped egg: %s' % display_path(path)) continue if (basename != 'site-packages' and basename != 'dist-packages' and not path.replace('\\', '/').endswith('lib/python')): continue logger.notify('In %s:' % display_path(path)) logger.indent += 2 zipped = [] unzipped = [] try: for filename in sorted(os.listdir(path)): ext = os.path.splitext(filename)[1].lower() if ext in ('.pth', '.egg-info', '.egg-link'): continue if ext == '.py': logger.info('Not displaying %s: not a package' % display_path(filename)) continue full = os.path.join(path, filename) if os.path.isdir(full): unzipped.append((filename, self.count_package(full))) elif zipfile.is_zipfile(full): zipped.append(filename) else: logger.info('Unknown file: %s' % display_path(filename)) if zipped: logger.notify('Zipped packages:') logger.indent += 2 try: for filename in zipped: logger.notify(filename) finally: logger.indent -= 2 else: logger.notify('No zipped packages.') if unzipped: if options.sort_files: unzipped.sort(key=lambda x: -x[1]) logger.notify('Unzipped packages:') logger.indent += 2 try: for filename, count in unzipped: logger.notify('%s (%i files)' % (filename, count)) finally: logger.indent -= 2 else: logger.notify('No unzipped packages.') finally: logger.indent -= 2
def _download_url(resp, link, temp_location): fp = open(temp_location, 'wb') download_hash = None if link.hash and link.hash_name: try: download_hash = hashlib.new(link.hash_name) except ValueError: logger.warn("Unsupported hash name %s for package %s" % (link.hash_name, link)) try: total_length = int(resp.headers['content-length']) except (ValueError, KeyError, TypeError): total_length = 0 downloaded = 0 show_progress = total_length > 40 * 1000 or not total_length show_url = link.show_url try: if show_progress: ## FIXME: the URL can get really long in this message: if total_length: logger.start_progress('Downloading %s (%s): ' % (show_url, format_size(total_length))) else: logger.start_progress('Downloading %s (unknown size): ' % show_url) else: logger.notify('Downloading %s' % show_url) logger.info('Downloading from URL %s' % link) def resp_read(chunk_size): try: # Special case for urllib3. try: for chunk in resp.raw.stream( chunk_size, decode_content=False): yield chunk except IncompleteRead as e: raise ChunkedEncodingError(e) except AttributeError: # Standard file-like object. while True: chunk = resp.raw.read(chunk_size) if not chunk: break yield chunk for chunk in resp_read(4096): downloaded += len(chunk) if show_progress: if not total_length: logger.show_progress('%s' % format_size(downloaded)) else: logger.show_progress('%3i%% %s' % (100 * downloaded / total_length, format_size(downloaded))) if download_hash is not None: download_hash.update(chunk) fp.write(chunk) fp.close() finally: if show_progress: logger.end_progress('%s downloaded' % format_size(downloaded)) return download_hash
def cmd(self): if self._cmd is not None: return self._cmd command = find_command(self.name) if command is None: raise BadCommand('Cannot find command %r' % self.name) logger.info('Found command %r at %r' % (self.name, command)) self._cmd = command return command
def check_destination(self, dest, url, rev_options, rev_display): """ Prepare a location to receive a checkout/clone. Return True if the location is ready for (and requires) a checkout/clone, False otherwise. """ checkout = True prompt = False if os.path.exists(dest): checkout = False if os.path.exists(os.path.join(dest, self.dirname)): existing_url = self.get_url(dest) if self.compare_urls(existing_url, url): logger.info('%s in %s exists, and has correct URL (%s)' % (self.repo_name.title(), display_path(dest), url)) logger.notify('Updating %s %s%s' % (display_path(dest), self.repo_name, rev_display)) self.update(dest, rev_options) else: logger.warn('%s %s in %s exists with URL %s' % (self.name, self.repo_name, display_path(dest), existing_url)) prompt = ('(s)witch, (i)gnore, (w)ipe, (b)ackup ', ('s', 'i', 'w', 'b')) else: logger.warn('Directory %s already exists, ' 'and is not a %s %s.' % (dest, self.name, self.repo_name)) prompt = ('(i)gnore, (w)ipe, (b)ackup ', ('i', 'w', 'b')) if prompt: logger.warn('The plan is to install the %s repository %s' % (self.name, url)) response = ask_path_exists('What to do? %s' % prompt[0], prompt[1]) if response == 's': logger.notify('Switching %s %s to %s%s' % (self.repo_name, display_path(dest), url, rev_display)) self.switch(dest, url, rev_options) elif response == 'i': # do nothing pass elif response == 'w': logger.warn('Deleting %s' % display_path(dest)) rmtree(dest) checkout = True elif response == 'b': dest_dir = backup_dir(dest) logger.warn('Backing up %s to %s' % (display_path(dest), dest_dir)) shutil.move(dest, dest_dir) checkout = True return checkout
def check_destination(self, dest, url, rev_options, rev_display): """ Prepare a location to receive a checkout/clone. Return True if the location is ready for (and requires) a checkout/clone, False otherwise. """ checkout = True prompt = False if os.path.exists(dest): checkout = False if os.path.exists(os.path.join(dest, self.dirname)): existing_url = self.get_url(dest) if self.compare_urls(existing_url, url): logger.info( '%s in %s exists, and has correct URL (%s)' % (self.repo_name.title(), display_path(dest), url)) logger.notify( 'Updating %s %s%s' % (display_path(dest), self.repo_name, rev_display)) self.update(dest, rev_options) else: logger.warn('%s %s in %s exists with URL %s' % (self.name, self.repo_name, display_path(dest), existing_url)) prompt = ('(s)witch, (i)gnore, (w)ipe, (b)ackup ', ('s', 'i', 'w', 'b')) else: logger.warn('Directory %s already exists, ' 'and is not a %s %s.' % (dest, self.name, self.repo_name)) prompt = ('(i)gnore, (w)ipe, (b)ackup ', ('i', 'w', 'b')) if prompt: logger.warn('The plan is to install the %s repository %s' % (self.name, url)) response = ask_path_exists('What to do? %s' % prompt[0], prompt[1]) if response == 's': logger.notify( 'Switching %s %s to %s%s' % (self.repo_name, display_path(dest), url, rev_display)) self.switch(dest, url, rev_options) elif response == 'i': # do nothing pass elif response == 'w': logger.warn('Deleting %s' % display_path(dest)) rmtree(dest) checkout = True elif response == 'b': dest_dir = backup_dir(dest) logger.warn('Backing up %s to %s' % (display_path(dest), dest_dir)) shutil.move(dest, dest_dir) checkout = True return checkout
def rollback(self): if self._saved_lines is None: logger.error("Cannot roll back changes to %s, none were made" % self.file) return False logger.info("Rolling %s back to previous state" % self.file) fh = open(self.file, "wb") fh.writelines(self._saved_lines) fh.close() return True
def zip_package(self, module_name, filename, no_pyc): orig_filename = filename logger.notify('Zip %s (in %s)' % (module_name, display_path(filename))) logger.indent += 2 if filename.endswith('.egg'): dest_filename = filename else: dest_filename = filename + '.zip' try: # FIXME: I think this needs to be undoable: if filename == dest_filename: filename = backup_dir(orig_filename) logger.notify( 'Moving %s aside to %s' % (orig_filename, filename) ) if not self.simulate: shutil.move(orig_filename, filename) try: logger.info( 'Creating zip file in %s' % display_path(dest_filename) ) if not self.simulate: zip = zipfile.ZipFile(dest_filename, 'w') zip.writestr(module_name + '/', '') for dirpath, dirnames, filenames in os.walk(filename): if no_pyc: filenames = [f for f in filenames if not f.lower().endswith('.pyc')] for fns, is_dir in [ (dirnames, True), (filenames, False)]: for fn in fns: full = os.path.join(dirpath, fn) dest = os.path.join( module_name, dirpath[len(filename):].lstrip( os.path.sep ), fn, ) if is_dir: zip.writestr(dest + '/', '') else: zip.write(full, dest) zip.close() logger.info( 'Removing old directory %s' % display_path(filename) ) if not self.simulate: rmtree(filename) except: # FIXME: need to do an undo here raise # FIXME: should also be undone: self.add_filename_to_pth(dest_filename) finally: logger.indent -= 2
def rollback(self): if self._saved_lines is None: logger.error('Cannot roll back changes to %s, none were made' % self.file) return False logger.info('Rolling %s back to previous state' % self.file) fh = open(self.file, 'wb') fh.writelines(self._saved_lines) fh.close() return True
def unzip_package(self, module_name, filename): zip_filename = os.path.dirname(filename) if not os.path.isfile(zip_filename) and zipfile.is_zipfile( zip_filename): raise InstallationError( 'Module %s (in %s) isn\'t located in a zip file in %s' % (module_name, filename, zip_filename)) package_path = os.path.dirname(zip_filename) if not package_path in self.paths(): logger.warn( 'Unpacking %s into %s, but %s is not on sys.path' % (display_path(zip_filename), display_path(package_path), display_path(package_path))) logger.notify('Unzipping %s (in %s)' % (module_name, display_path(zip_filename))) if self.simulate: logger.notify( 'Skipping remaining operations because of --simulate') return logger.indent += 2 try: ## FIXME: this should be undoable: zip = zipfile.ZipFile(zip_filename) to_save = [] for info in zip.infolist(): name = info.filename if name.startswith(module_name + os.path.sep): content = zip.read(name) dest = os.path.join(package_path, name) if not os.path.exists(os.path.dirname(dest)): os.makedirs(os.path.dirname(dest)) if not content and dest.endswith(os.path.sep): if not os.path.exists(dest): os.makedirs(dest) else: f = open(dest, 'wb') f.write(content) f.close() else: to_save.append((name, zip.read(name))) zip.close() if not to_save: logger.info('Removing now-empty zip file %s' % display_path(zip_filename)) os.unlink(zip_filename) self.remove_filename_from_pth(zip_filename) else: logger.info('Removing entries in %s/ from zip file %s' % (module_name, display_path(zip_filename))) zip = zipfile.ZipFile(zip_filename, 'w') for name, content in to_save: zip.writestr(name, content) zip.close() finally: logger.indent -= 2
def remove_temporary_source(self): """Remove the source files from this requirement, if they are marked for deletion""" if self.is_bundle or os.path.exists(self.delete_marker_filename): logger.info('Removing source in %s' % self.source_dir) if self.source_dir: rmtree(self.source_dir) self.source_dir = None if self._temp_build_dir and os.path.exists(self._temp_build_dir): rmtree(self._temp_build_dir) self._temp_build_dir = None
def zip_package(self, module_name, filename, no_pyc): orig_filename = filename logger.notify('Zip %s (in %s)' % (module_name, display_path(filename))) logger.indent += 2 if filename.endswith('.egg'): dest_filename = filename else: dest_filename = filename + '.zip' try: # FIXME: I think this needs to be undoable: if filename == dest_filename: filename = backup_dir(orig_filename) logger.notify('Moving %s aside to %s' % (orig_filename, filename)) if not self.simulate: shutil.move(orig_filename, filename) try: logger.info('Creating zip file in %s' % display_path(dest_filename)) if not self.simulate: zip = zipfile.ZipFile(dest_filename, 'w') zip.writestr(module_name + '/', '') for dirpath, dirnames, filenames in os.walk(filename): if no_pyc: filenames = [ f for f in filenames if not f.lower().endswith('.pyc') ] for fns, is_dir in [(dirnames, True), (filenames, False)]: for fn in fns: full = os.path.join(dirpath, fn) dest = os.path.join( module_name, dirpath[len(filename):].lstrip( os.path.sep), fn, ) if is_dir: zip.writestr(dest + '/', '') else: zip.write(full, dest) zip.close() logger.info('Removing old directory %s' % display_path(filename)) if not self.simulate: rmtree(filename) except: # FIXME: need to do an undo here raise # FIXME: should also be undone: self.add_filename_to_pth(dest_filename) finally: logger.indent -= 2
def _clean_one(self, req): base_args = self._base_setup_args(req) logger.info('Running setup.py clean for %s', req.name) clean_args = base_args + ['clean', '--all'] try: call_subprocess(clean_args, cwd=req.source_dir, show_stdout=False) return True except: logger.error('Failed cleaning build dir for %s', req.name) return False
def rollback(self): """Rollback the changes previously made by remove().""" if self.save_dir is None: logger.error("Can't roll back %s; was not uninstalled" % self.dist.project_name) return False logger.notify("Rolling back uninstall of %s" % self.dist.project_name) for path in self._moved_paths: tmp_path = self._stash(path) logger.info("Replacing %s" % path) renames(tmp_path, path) for pth in self.pth: pth.rollback()
def run(self, options, args): if not options.req_repository: logger.notify('You need to specify a repository. This utility ' 'does not upload to PyPI') return 1 options.build_dir = os.path.abspath(options.req_cache_dir) options.src_dir = os.path.abspath(options.req_cache_dir) options.no_install = True options.ignore_installed = True install_options = options.install_options or [] global_options = options.global_options or [] index_urls = [options.index_url] + options.extra_index_urls if options.no_index: logger.notify('Ignoring indexes: %s' % ','.join(index_urls)) index_urls = [] finder = self._build_package_finder(options, index_urls) requirement_set = RequirementSet(build_dir=options.build_dir, src_dir=options.src_dir, download_dir=options.download_dir, download_cache=options.download_cache, upgrade=options.upgrade, ignore_installed=options.ignore_installed, ignore_dependencies=options.ignore_dependencies) req_req = open(os.path.abspath(options.req_req_requirements), 'w') req_req.writelines(['--index-url=%s' % self.repository_url(options), '\n']) try: for filename in options.requirements: for req in parse_requirements(filename, finder=finder, options=options): logger.info('req ' + str(req.req)) req_req.writelines([str(req.req), '\n']) requirement_set.add_requirement(req) finally: req_req.close() if not options.no_download: requirement_set.prepare_files(finder, force_root_egg_info=False, bundle=False) else: requirement_set.locate_files() if not os.path.exists(os.path.abspath(options.req_cache_dir)): os.mkdir(os.path.abspath(options.req_cache_dir)) if not options.req_no_upload: self.upload_to_repository(options) if options.req_clean_cache: requirement_set.cleanup_files(bundle=False) return requirement_set
def __init__(self, find_links, index_urls, use_mirrors=False, mirrors=None, main_mirror_url=None): self.find_links = find_links self.index_urls = index_urls self.dependency_links = [] self.cache = PageCache() # These are boring links that have already been logged somehow: self.logged_links = set() if use_mirrors: self.mirror_urls = self._get_mirror_urls(mirrors, main_mirror_url) logger.info('Using PyPI mirrors: %s' % ', '.join(self.mirror_urls)) else: self.mirror_urls = []
def rollback(self): """Rollback the changes previously made by remove().""" if self.save_dir is None: logger.error("Can't roll back %s; was not uninstalled" % self.dist.project_name) return False logger.notify('Rolling back uninstall of %s' % self.dist.project_name) for path in self._moved_paths: tmp_path = self._stash(path) logger.info('Replacing %s' % path) renames(tmp_path, path) for pth in self.pth.values(): pth.rollback()
def unzip_package(self, module_name, filename): zip_filename = os.path.dirname(filename) if not os.path.isfile(zip_filename) and zipfile.is_zipfile(zip_filename): raise InstallationError( 'Module %s (in %s) isn\'t located in a zip file in %s' % (module_name, filename, zip_filename)) package_path = os.path.dirname(zip_filename) if not package_path in self.paths(): logger.warn( 'Unpacking %s into %s, but %s is not on sys.path' % (display_path(zip_filename), display_path(package_path), display_path(package_path))) logger.notify('Unzipping %s (in %s)' % (module_name, display_path(zip_filename))) if self.simulate: logger.notify('Skipping remaining operations because of --simulate') return logger.indent += 2 try: ## FIXME: this should be undoable: zip = zipfile.ZipFile(zip_filename) to_save = [] for info in zip.infolist(): name = info.filename if name.startswith(module_name + os.path.sep): content = zip.read(name) dest = os.path.join(package_path, name) if not os.path.exists(os.path.dirname(dest)): os.makedirs(os.path.dirname(dest)) if not content and dest.endswith(os.path.sep): if not os.path.exists(dest): os.makedirs(dest) else: f = open(dest, 'wb') f.write(content) f.close() else: to_save.append((name, zip.read(name))) zip.close() if not to_save: logger.info('Removing now-empty zip file %s' % display_path(zip_filename)) os.unlink(zip_filename) self.remove_filename_from_pth(zip_filename) else: logger.info('Removing entries in %s/ from zip file %s' % (module_name, display_path(zip_filename))) zip = zipfile.ZipFile(zip_filename, 'w') for name, content in to_save: zip.writestr(name, content) zip.close() finally: logger.indent -= 2
def _download_url(resp, link, temp_location): fp = open(temp_location, 'wb') download_hash = None if link.hash and link.hash_name: try: download_hash = hashlib.new(link.hash_name) except ValueError: logger.warn("Unsupported hash name %s for package %s" % (link.hash_name, link)) try: total_length = int(resp.info()['content-length']) except (ValueError, KeyError, TypeError): total_length = 0 downloaded = 0 show_progress = total_length > 40 * 1000 or not total_length show_url = link.show_url try: if show_progress: ## FIXME: the URL can get really long in this message: if total_length: logger.start_progress('Downloading %s (%s): ' % (show_url, format_size(total_length))) else: logger.start_progress('Downloading %s (unknown size): ' % show_url) else: logger.notify('Downloading %s' % show_url) logger.info('Downloading from URL %s' % link) while True: chunk = resp.read(4096) if not chunk: break downloaded += len(chunk) if show_progress: if not total_length: logger.show_progress('%s' % format_size(downloaded)) else: logger.show_progress('%3i%% %s' % (100 * downloaded / total_length, format_size(downloaded))) if download_hash is not None: download_hash.update(chunk) fp.write(chunk) fp.close() finally: if show_progress: logger.end_progress('%s downloaded' % format_size(downloaded)) return download_hash
def __init__(self, find_links, index_urls, use_mirrors=False, mirrors=None, main_mirror_url=None, use_wheel=False, allow_external=[], allow_insecure=[], allow_all_external=False, allow_all_insecure=False, allow_all_prereleases=False): self.find_links = find_links self.index_urls = index_urls self.dependency_links = [] self.cache = PageCache() # These are boring links that have already been logged somehow: self.logged_links = set() if use_mirrors: self.mirror_urls = self._get_mirror_urls(mirrors, main_mirror_url) logger.info('Using PyPI mirrors: %s' % ', '.join(self.mirror_urls)) else: self.mirror_urls = [] self.use_wheel = use_wheel # Do we allow (safe and verifiable) externally hosted files? self.allow_external = set(normalize_name(n) for n in allow_external) # Which names are allowed to install insecure and unverifiable files? self.allow_insecure = set(normalize_name(n) for n in allow_insecure) # Do we allow all (safe and verifiable) externally hosted files? self.allow_all_external = allow_all_external # Do we allow unsafe and unverifiable files? self.allow_all_insecure = allow_all_insecure # Stores if we ignored any external links so that we can instruct # end users how to install them if no distributions are available self.need_warn_external = False # Stores if we ignored any unsafe links so that we can instruct # end users how to install them if no distributions are available self.need_warn_insecure = False # Do we want to allow _all_ pre-releases? self.allow_all_prereleases = allow_all_prereleases
def cleanup_files(self): """Clean up files, remove builds.""" logger.notify('Cleaning up...') logger.indent += 2 for req in self.reqs_to_cleanup: req.remove_temporary_source() remove_dir = [] if self._pip_has_created_build_dir(): remove_dir.append(self.build_dir) for dir in remove_dir: if os.path.exists(dir): logger.info('Removing temporary dir %s...' % dir) rmtree(dir) logger.indent -= 2
def _download_url(resp, link, temp_location): fp = open(temp_location, 'wb') download_hash = None if link.hash and link.hash_name: try: download_hash = hashlib.new(link.hash_name) except ValueError: logger.warn("Unsupported hash name %s for package %s" % (link.hash_name, link)) try: total_length = int(resp.info()['content-length']) except (ValueError, KeyError, TypeError): total_length = 0 downloaded = 0 show_progress = total_length > 40*1000 or not total_length show_url = link.show_url try: if show_progress: ## FIXME: the URL can get really long in this message: if total_length: logger.start_progress('Downloading %s (%s): ' % (show_url, format_size(total_length))) else: logger.start_progress('Downloading %s (unknown size): ' % show_url) else: logger.notify('Downloading %s' % show_url) logger.info('Downloading from URL %s' % link) while True: chunk = resp.read(4096) if not chunk: break downloaded += len(chunk) if show_progress: if not total_length: logger.show_progress('%s' % format_size(downloaded)) else: logger.show_progress('%3i%% %s' % (100*downloaded/total_length, format_size(downloaded))) if download_hash is not None: download_hash.update(chunk) fp.write(chunk) fp.close() finally: if show_progress: logger.end_progress('%s downloaded' % format_size(downloaded)) return download_hash
def remove(self): logger.info('Removing pth entries from %s:' % self.file) fh = open(self.file, 'rb') # windows uses '\r\n' with py3k, but uses '\n' with py2.x lines = fh.readlines() self._saved_lines = lines fh.close() if any(b('\r\n') in line for line in lines): endline = '\r\n' else: endline = '\n' for entry in self.entries: try: logger.info('Removing entry: %s' % entry) lines.remove(b(entry + endline)) except ValueError: pass fh = open(self.file, 'wb') fh.writelines(lines) fh.close()
def remove(self): logger.info("Removing pth entries from %s:" % self.file) fh = open(self.file, "rb") # windows uses '\r\n' with py3k, but uses '\n' with py2.x lines = fh.readlines() self._saved_lines = lines fh.close() if any(b("\r\n") in line for line in lines): endline = "\r\n" else: endline = "\n" for entry in self.entries: try: logger.info("Removing entry: %s" % entry) lines.remove(b(entry + endline)) except ValueError: pass fh = open(self.file, "wb") fh.writelines(lines) fh.close()
def remove_filename_from_pth(self, filename): for pth in self.pth_files(): f = open(pth, 'r') lines = f.readlines() f.close() new_lines = [ l for l in lines if l.strip() != filename] if lines != new_lines: logger.info('Removing reference to %s from .pth file %s' % (display_path(filename), display_path(pth))) if not filter(None, new_lines): logger.info('%s file would be empty: deleting' % display_path(pth)) if not self.simulate: os.unlink(pth) else: if not self.simulate: f = open(pth, 'wb') f.writelines(new_lines) f.close() return logger.warn('Cannot find a reference to %s in any .pth file' % display_path(filename))
def _build_one(self, req, output_dir, python_tag=None): """Build one wheel. :return: The filename of the built wheel, or None if the build failed. """ tempd = tempfile.mkdtemp('pip-wheel-') try: if self.__build_one(req, tempd, python_tag=python_tag): try: wheel_name = os.listdir(tempd)[0] wheel_path = os.path.join(output_dir, wheel_name) shutil.move(os.path.join(tempd, wheel_name), wheel_path) logger.info('Stored in directory: %s', output_dir) return wheel_path except: pass # Ignore return, we can't do anything else useful. self._clean_one(req) return None finally: rmtree(tempd)
def remove(self, auto_confirm=False): """Remove paths in ``self.paths`` with confirmation (unless ``auto_confirm`` is True).""" if not self._can_uninstall(): return if not self.paths: logger.notify( "Can't uninstall '%s'. No files were found to uninstall." % self.dist.project_name) return logger.notify('Uninstalling %s:' % self.dist.project_name) logger.indent += 2 paths = sorted(self.compact(self.paths)) try: if auto_confirm: response = 'y' else: for path in paths: logger.notify(path) response = ask('Proceed (y/n)? ', ('y', 'n')) if self._refuse: logger.notify('Not removing or modifying (outside of prefix):') for path in self.compact(self._refuse): logger.notify(path) if response == 'y': self.save_dir = tempfile.mkdtemp(suffix='-uninstall', prefix='pip-') for path in paths: new_path = self._stash(path) logger.info('Removing file or directory %s' % path) self._moved_paths.append(path) renames(path, new_path) for pth in self.pth.values(): pth.remove() logger.notify('Successfully uninstalled %s' % self.dist.project_name) finally: logger.indent -= 2
def cleanup_files(self, bundle=False): """Clean up files, remove builds.""" logger.notify('Cleaning up...') logger.indent += 2 for req in self.reqs_to_cleanup: req.remove_temporary_source() remove_dir = [] if self._pip_has_created_build_dir(): remove_dir.append(self.build_dir) # The source dir of a bundle can always be removed. # FIXME: not if it pre-existed the bundle! if bundle: remove_dir.append(self.src_dir) for dir in remove_dir: if os.path.exists(dir): logger.info('Removing temporary dir %s...' % dir) rmtree(dir) logger.indent -= 2
def __init__(self, *args, **kwargs): use_gssapi = kwargs.pop('gssapi', None) super(PipSession, self).__init__(*args, **kwargs) if use_gssapi: # Use Kerberos authentication logger.info("Using Kerberos for authentication") try: from requests_kerberos import HTTPKerberosAuth self.auth = HTTPKerberosAuth() except ImportError as e: logger.error("""Can't import module requests_kerberos: %s Falling back to basic auth..""", e) # Attach our User Agent to the request self.headers["User-Agent"] = user_agent() if not self.auth: # Attach our Authentication handler to the session self.auth = MultiDomainBasicAuth() # Enable file:// urls self.mount("file://", LocalFSAdapter())
def main(self, args): options, args = self.parse_args(args) level = 1 # Notify level += options.verbose level -= options.quiet level = logger.level_for_integer(4 - level) complete_log = [] logger.add_consumers( (level, sys.stdout), (logger.DEBUG, complete_log.append), ) if options.log_explicit_levels: logger.explicit_levels = True self.setup_logging() #TODO: try to get these passing down from the command? # without resorting to os.environ to hold these. if options.no_input: os.environ['PIP_NO_INPUT'] = '1' if options.exists_action: os.environ['PIP_EXISTS_ACTION'] = ' '.join(options.exists_action) if options.require_venv: # If a venv is required check if it can really be found if not running_under_virtualenv(): logger.fatal( 'Could not find an activated virtualenv (required).') sys.exit(VIRTUALENV_NOT_FOUND) if options.log: log_fp = open_logfile(options.log, 'a') logger.add_consumers((logger.DEBUG, log_fp)) else: log_fp = None exit = SUCCESS store_log = False try: status = self.run(options, args) # FIXME: all commands should return an exit status # and when it is done, isinstance is not needed anymore if isinstance(status, int): exit = status except PreviousBuildDirError: e = sys.exc_info()[1] logger.fatal(str(e)) logger.info('Exception information:\n%s' % format_exc()) store_log = True exit = PREVIOUS_BUILD_DIR_ERROR except (InstallationError, UninstallationError): e = sys.exc_info()[1] logger.fatal(str(e)) logger.info('Exception information:\n%s' % format_exc()) store_log = True exit = ERROR except BadCommand: e = sys.exc_info()[1] logger.fatal(str(e)) logger.info('Exception information:\n%s' % format_exc()) store_log = True exit = ERROR except CommandError: e = sys.exc_info()[1] logger.fatal('ERROR: %s' % e) logger.info('Exception information:\n%s' % format_exc()) exit = ERROR except KeyboardInterrupt: logger.fatal('Operation cancelled by user') logger.info('Exception information:\n%s' % format_exc()) store_log = True exit = ERROR except: logger.fatal('Exception:\n%s' % format_exc()) store_log = True exit = UNKNOWN_ERROR if store_log: log_file_fn = options.log_file text = '\n'.join(complete_log) try: log_file_fp = open_logfile(log_file_fn, 'w') except IOError: temp = tempfile.NamedTemporaryFile(delete=False) log_file_fn = temp.name log_file_fp = open_logfile(log_file_fn, 'w') logger.fatal('Storing debug log for failure in %s' % log_file_fn) log_file_fp.write(text) log_file_fp.close() if log_fp is not None: log_fp.close() return exit
def call_subprocess(cmd, show_stdout=True, filter_stdout=None, cwd=None, raise_on_returncode=True, command_level=logger.DEBUG, command_desc=None, extra_environ=None): if command_desc is None: cmd_parts = [] for part in cmd: if ' ' in part or '\n' in part or '"' in part or "'" in part: part = '"%s"' % part.replace('"', '\\"') cmd_parts.append(part) command_desc = ' '.join(cmd_parts) if show_stdout: stdout = None else: stdout = subprocess.PIPE logger.log(command_level, "Running command %s" % command_desc) env = os.environ.copy() if extra_environ: env.update(extra_environ) try: proc = subprocess.Popen( cmd, stderr=subprocess.STDOUT, stdin=None, stdout=stdout, cwd=cwd, env=env) except Exception: e = sys.exc_info()[1] logger.fatal( "Error %s while executing command %s" % (e, command_desc)) raise all_output = [] if stdout is not None: stdout = proc.stdout while 1: line = console_to_str(stdout.readline()) if not line: break line = line.rstrip() all_output.append(line + '\n') if filter_stdout: level = filter_stdout(line) if isinstance(level, tuple): level, line = level logger.log(level, line) if not logger.stdout_level_matches(level): logger.show_progress() else: logger.info(line) else: returned_stdout, returned_stderr = proc.communicate() all_output = [returned_stdout or ''] proc.wait() if proc.returncode: if raise_on_returncode: if all_output: logger.notify('Complete output from command %s:' % command_desc) logger.notify('\n'.join(all_output) + '\n----------------------------------------') raise InstallationError( "Command %s failed with error code %s in %s" % (command_desc, proc.returncode, cwd)) else: logger.warn( "Command %s had error code %s in %s" % (command_desc, proc.returncode, cwd)) if stdout is not None: return ''.join(all_output)
def find_requirement(self, req, upgrade): def mkurl_pypi_url(url): loc = posixpath.join(url, url_name) # For maximum compatibility with easy_install, ensure the path # ends in a trailing slash. Although this isn't in the spec # (and PyPI can handle it without the slash) some other index # implementations might break if they relied on easy_install's behavior. if not loc.endswith('/'): loc = loc + '/' return loc url_name = req.url_name # Only check main index if index URL is given: main_index_url = None if self.index_urls: # Check that we have the url_name correctly spelled: main_index_url = Link(mkurl_pypi_url(self.index_urls[0]), trusted=True) # This will also cache the page, so it's okay that we get it again later: page = self._get_page(main_index_url, req) if page is None: url_name = self._find_url_name( Link(self.index_urls[0], trusted=True), url_name, req) or req.url_name if url_name is not None: locations = [mkurl_pypi_url(url) for url in self.index_urls] + self.find_links else: locations = list(self.find_links) for version in req.absolute_versions: if url_name is not None and main_index_url is not None: locations = [posixpath.join(main_index_url.url, version) ] + locations file_locations, url_locations = self._sort_locations(locations) _flocations, _ulocations = self._sort_locations(self.dependency_links) file_locations.extend(_flocations) # We trust every url that the user has given us whether it was given # via --index-url or --find-links locations = [Link(url, trusted=True) for url in url_locations] # We explicitly do not trust links that came from dependency_links locations.extend([Link(url) for url in _ulocations]) logger.debug('URLs to search for versions for %s:' % req) for location in locations: logger.debug('* %s' % location) # Determine if this url used a secure transport mechanism parsed = urlparse.urlparse(str(location)) if parsed.scheme in INSECURE_SCHEMES: secure_schemes = INSECURE_SCHEMES[parsed.scheme] if len(secure_schemes) == 1: ctx = (location, parsed.scheme, secure_schemes[0], parsed.netloc) logger.warn("%s uses an insecure transport scheme (%s). " "Consider using %s if %s has it available" % ctx) elif len(secure_schemes) > 1: ctx = (location, parsed.scheme, ", ".join(secure_schemes), parsed.netloc) logger.warn("%s uses an insecure transport scheme (%s). " "Consider using one of %s if %s has any of " "them available" % ctx) else: ctx = (location, parsed.scheme) logger.warn("%s uses an insecure transport scheme (%s)." % ctx) found_versions = [] found_versions.extend( self._package_versions( # We trust every directly linked archive in find_links [Link(url, '-f', trusted=True) for url in self.find_links], req.name.lower())) page_versions = [] for page in self._get_pages(locations, req): logger.debug('Analyzing links from page %s' % page.url) logger.indent += 2 try: page_versions.extend( self._package_versions(page.links, req.name.lower())) finally: logger.indent -= 2 dependency_versions = list( self._package_versions( [Link(url) for url in self.dependency_links], req.name.lower())) if dependency_versions: logger.info('dependency_links found: %s' % ', '.join( [link.url for parsed, link, version in dependency_versions])) file_versions = list( self._package_versions([Link(url) for url in file_locations], req.name.lower())) if not found_versions and not page_versions and not dependency_versions and not file_versions: logger.fatal( 'Could not find any downloads that satisfy the requirement %s' % req) if self.need_warn_external: logger.warn("Some externally hosted files were ignored (use " "--allow-external %s to allow)." % req.name) if self.need_warn_unverified: logger.warn("Some insecure and unverifiable files were ignored" " (use --allow-unverified %s to allow)." % req.name) raise DistributionNotFound('No distributions at all found for %s' % req) installed_version = [] if req.satisfied_by is not None: installed_version = [(req.satisfied_by.parsed_version, INSTALLED_VERSION, req.satisfied_by.version)] if file_versions: file_versions.sort(reverse=True) logger.info('Local files found: %s' % ', '.join([ url_to_path(link.url) for parsed, link, version in file_versions ])) #this is an intentional priority ordering all_versions = installed_version + file_versions + found_versions + page_versions + dependency_versions applicable_versions = [] for (parsed_version, link, version) in all_versions: if version not in req.req: logger.info("Ignoring link %s, version %s doesn't match %s" % (link, version, ','.join( [''.join(s) for s in req.req.specs]))) continue elif is_prerelease(version) and not (self.allow_all_prereleases or req.prereleases): # If this version isn't the already installed one, then # ignore it if it's a pre-release. if link is not INSTALLED_VERSION: logger.info( "Ignoring link %s, version %s is a pre-release (use --pre to allow)." % (link, version)) continue applicable_versions.append((parsed_version, link, version)) applicable_versions = self._sort_versions(applicable_versions) existing_applicable = bool([ link for parsed_version, link, version in applicable_versions if link is INSTALLED_VERSION ]) if not upgrade and existing_applicable: if applicable_versions[0][1] is INSTALLED_VERSION: logger.info( 'Existing installed version (%s) is most up-to-date and satisfies requirement' % req.satisfied_by.version) else: logger.info( 'Existing installed version (%s) satisfies requirement (most up-to-date version is %s)' % (req.satisfied_by.version, applicable_versions[0][2])) return None if not applicable_versions: logger.fatal( 'Could not find a version that satisfies the requirement %s (from versions: %s)' % (req, ', '.join([ version for parsed_version, link, version in all_versions ]))) if self.need_warn_external: logger.warn("Some externally hosted files were ignored (use " "--allow-external to allow).") if self.need_warn_unverified: logger.warn("Some insecure and unverifiable files were ignored" " (use --allow-unverified %s to allow)." % req.name) raise DistributionNotFound( 'No distributions matching the version for %s' % req) if applicable_versions[0][1] is INSTALLED_VERSION: # We have an existing version, and its the best version logger.info( 'Installed version (%s) is most up-to-date (past versions: %s)' % (req.satisfied_by.version, ', '.join([ version for parsed_version, link, version in applicable_versions[1:] ]) or 'none')) raise BestVersionAlreadyInstalled if len(applicable_versions) > 1: logger.info( 'Using version %s (newest of versions: %s)' % (applicable_versions[0][2], ', '.join([ version for parsed_version, link, version in applicable_versions ]))) selected_version = applicable_versions[0][1] # TODO: Remove after 1.4 has been released # if (selected_version.internal is not None # and not selected_version.internal): # logger.warn("You are installing an externally hosted file. Future " # "versions of pip will default to disallowing " # "externally hosted files.") # if (selected_version.verifiable is not None # and not selected_version.verifiable): # logger.warn("You are installing a potentially insecure and " # "unverifiable file. Future versions of pip will " # "default to disallowing insecure files.") if selected_version._deprecated_regex: logger.deprecated( "1.7", "%s discovered using a deprecated method of parsing, " "in the future it will no longer be discovered" % req.name) return selected_version
def main(self, complete_args, args, initial_options): options, args = self.parser.parse_args(args) self.merge_options(initial_options, options) if options.require_venv and not options.venv: # If a venv is required check if it can really be found if not os.environ.get('VIRTUAL_ENV'): print 'Could not find an activated virtualenv (required).' sys.exit(3) # Automatically install in currently activated venv if required options.respect_venv = True if args and args[-1] == '___VENV_RESTART___': ## FIXME: We don't do anything this this value yet: venv_location = args[-2] args = args[:-2] options.venv = None else: # If given the option to respect the activated environment # check if no venv is given as a command line parameter if options.respect_venv and os.environ.get('VIRTUAL_ENV'): if options.venv and os.path.exists(options.venv): # Make sure command line venv and environmental are the same if (os.path.realpath(os.path.expanduser(options.venv)) != os.path.realpath(os.environ.get('VIRTUAL_ENV'))): print ("Given virtualenv (%s) doesn't match " "currently activated virtualenv (%s)." % (options.venv, os.environ.get('VIRTUAL_ENV'))) sys.exit(3) else: options.venv = os.environ.get('VIRTUAL_ENV') print 'Using already activated environment %s' % options.venv level = 1 # Notify level += options.verbose level -= options.quiet level = logger.level_for_integer(4-level) complete_log = [] logger.consumers.extend( [(level, sys.stdout), (logger.DEBUG, complete_log.append)]) if options.log_explicit_levels: logger.explicit_levels = True if options.venv: if options.verbose > 0: # The logger isn't setup yet print 'Running in environment %s' % options.venv site_packages=False if options.site_packages: site_packages=True restart_in_venv(options.venv, options.venv_base, site_packages, complete_args) # restart_in_venv should actually never return, but for clarity... return ## FIXME: not sure if this sure come before or after venv restart if options.log: log_fp = open_logfile_append(options.log) logger.consumers.append((logger.DEBUG, log_fp)) else: log_fp = None socket.setdefaulttimeout(options.timeout or None) setup_proxy_handler(options.proxy) exit = 0 try: self.run(options, args) except (InstallationError, UninstallationError), e: logger.fatal(str(e)) logger.info('Exception information:\n%s' % format_exc()) exit = 1
def run(self, options, args): logger.info("fake") if self.error: raise SystemExit(1)
def main(self, complete_args, args, initial_options): options, args = self.parser.parse_args(args) self.merge_options(initial_options, options) level = 1 # Notify level += options.verbose level -= options.quiet level = logger.level_for_integer(4-level) complete_log = [] logger.consumers.extend( [(level, sys.stdout), (logger.DEBUG, complete_log.append)]) if options.log_explicit_levels: logger.explicit_levels = True self.setup_logging() if options.require_venv and not options.venv: # If a venv is required check if it can really be found if not os.environ.get('VIRTUAL_ENV'): logger.fatal('Could not find an activated virtualenv (required).') sys.exit(3) # Automatically install in currently activated venv if required options.respect_venv = True if args and args[-1] == '___VENV_RESTART___': ## FIXME: We don't do anything this this value yet: args = args[:-2] options.venv = None else: # If given the option to respect the activated environment # check if no venv is given as a command line parameter if options.respect_venv and os.environ.get('VIRTUAL_ENV'): if options.venv and os.path.exists(options.venv): # Make sure command line venv and environmental are the same if (os.path.realpath(os.path.expanduser(options.venv)) != os.path.realpath(os.environ.get('VIRTUAL_ENV'))): logger.fatal("Given virtualenv (%s) doesn't match " "currently activated virtualenv (%s)." % (options.venv, os.environ.get('VIRTUAL_ENV'))) sys.exit(3) else: options.venv = os.environ.get('VIRTUAL_ENV') logger.info('Using already activated environment %s' % options.venv) if options.venv: logger.info('Running in environment %s' % options.venv) site_packages=False if options.site_packages: site_packages=True restart_in_venv(options.venv, options.venv_base, site_packages, complete_args) # restart_in_venv should actually never return, but for clarity... return ## FIXME: not sure if this sure come before or after venv restart if options.log: log_fp = open_logfile(options.log, 'a') logger.consumers.append((logger.DEBUG, log_fp)) else: log_fp = None socket.setdefaulttimeout(options.timeout or None) urlopen.setup(proxystr=options.proxy, prompting=not options.no_input) exit = 0 try: self.run(options, args) except (InstallationError, UninstallationError), e: logger.fatal(str(e)) logger.info('Exception information:\n%s' % format_exc()) exit = 1
def call_subprocess(cmd, show_stdout=True, filter_stdout=None, cwd=None, raise_on_returncode=True, command_level=logger.DEBUG, command_desc=None, extra_environ=None): if command_desc is None: cmd_parts = [] for part in cmd: if ' ' in part or '\n' in part or '"' in part or "'" in part: part = '"%s"' % part.replace('"', '\\"') cmd_parts.append(part) command_desc = ' '.join(cmd_parts) if show_stdout: stdout = None else: stdout = subprocess.PIPE logger.log(command_level, "Running command %s" % command_desc) env = os.environ.copy() if extra_environ: env.update(extra_environ) try: proc = subprocess.Popen(cmd, stderr=subprocess.STDOUT, stdin=None, stdout=stdout, cwd=cwd, env=env) except Exception: e = sys.exc_info()[1] logger.fatal("Error %s while executing command %s" % (e, command_desc)) raise all_output = [] if stdout is not None: stdout = proc.stdout while 1: line = console_to_str(stdout.readline()) if not line: break line = line.rstrip() all_output.append(line + '\n') if filter_stdout: level = filter_stdout(line) if isinstance(level, tuple): level, line = level logger.log(level, line) if not logger.stdout_level_matches(level): logger.show_progress() else: logger.info(line) else: returned_stdout, returned_stderr = proc.communicate() all_output = [returned_stdout or ''] proc.wait() if proc.returncode: if raise_on_returncode: if all_output: logger.notify('Complete output from command %s:' % command_desc) logger.notify('\n'.join(all_output) + '\n----------------------------------------') raise InstallationError( "Command %s failed with error code %s in %s" % (command_desc, proc.returncode, cwd)) else: logger.warn("Command %s had error code %s in %s" % (command_desc, proc.returncode, cwd)) if stdout is not None: return ''.join(all_output)
def main(self, complete_args, args, initial_options): options, args = self.parser.parse_args(args) self.merge_options(initial_options, options) level = 1 # Notify level += options.verbose level -= options.quiet level = logger.level_for_integer(4-level) complete_log = [] logger.consumers.extend( [(level, sys.stdout), (logger.DEBUG, complete_log.append)]) if options.log_explicit_levels: logger.explicit_levels = True self.setup_logging() if options.require_venv: # If a venv is required check if it can really be found if not os.environ.get('VIRTUAL_ENV'): logger.fatal('Could not find an activated virtualenv (required).') sys.exit(3) if options.log: log_fp = open_logfile(options.log, 'a') logger.consumers.append((logger.DEBUG, log_fp)) else: log_fp = None socket.setdefaulttimeout(options.timeout or None) urlopen.setup(proxystr=options.proxy, prompting=not options.no_input) exit = 0 try: self.run(options, args) except (InstallationError, UninstallationError): e = sys.exc_info()[1] logger.fatal(str(e)) logger.info('Exception information:\n%s' % format_exc()) exit = 1 except BadCommand: e = sys.exc_info()[1] logger.fatal(str(e)) logger.info('Exception information:\n%s' % format_exc()) exit = 1 except KeyboardInterrupt: logger.fatal('Operation cancelled by user') logger.info('Exception information:\n%s' % format_exc()) exit = 1 except: logger.fatal('Exception:\n%s' % format_exc()) exit = 2 if log_fp is not None: log_fp.close() if exit: log_fn = options.log_file text = '\n'.join(complete_log) logger.fatal('Storing complete log in %s' % log_fn) log_fp = open_logfile(log_fn, 'w') log_fp.write(text) log_fp.close() return exit
stdout = proc.stdout while 1: line = stdout.readline() if not line: break line = line.rstrip() all_output.append(line + '\n') if filter_stdout: level = filter_stdout(line) if isinstance(level, tuple): level, line = level logger.log(level, line) if not logger.stdout_level_matches(level): logger.show_progress() else: logger.info(line) else: returned_stdout, returned_stderr = proc.communicate() all_output = [returned_stdout or ''] proc.wait() if proc.returncode: if raise_on_returncode: if all_output: logger.notify('Complete output from command %s:' % command_desc) logger.notify('\n'.join(all_output) + '\n----------------------------------------') raise InstallationError( "Command %s failed with error code %s" % (command_desc, proc.returncode)) else: logger.warn( "Command %s had error code %s"
def find_requirement(self, req, upgrade): url_name = req.url_name # Only check main index if index URL is given: main_index_url = None if self.index_urls: # Check that we have the url_name correctly spelled: main_index_url = Link(posixpath.join(self.index_urls[0], url_name)) # This will also cache the page, so it's okay that we get it again later: page = self._get_page(main_index_url, req) if page is None: url_name = self._find_url_name(Link(self.index_urls[0]), url_name, req) or req.url_name # Combine index URLs with mirror URLs here to allow # adding more index URLs from requirements files all_index_urls = self.index_urls + self.mirror_urls def mkurl_pypi_url(url): loc = posixpath.join(url, url_name) # For maximum compatibility with easy_install, ensure the path # ends in a trailing slash. Although this isn't in the spec # (and PyPI can handle it without the slash) some other index # implementations might break if they relied on easy_install's behavior. if not loc.endswith('/'): loc = loc + '/' return loc if url_name is not None: locations = [mkurl_pypi_url(url) for url in all_index_urls] + self.find_links else: locations = list(self.find_links) locations.extend(self.dependency_links) for version in req.absolute_versions: if url_name is not None and main_index_url is not None: locations = [posixpath.join(main_index_url.url, version) ] + locations file_locations, url_locations = self._sort_locations(locations) locations = [Link(url) for url in url_locations] logger.debug('URLs to search for versions for %s:' % req) for location in locations: logger.debug('* %s' % location) found_versions = [] found_versions.extend( self._package_versions( [Link(url, '-f') for url in self.find_links], req.name.lower())) page_versions = [] for page in self._get_pages(locations, req): logger.debug('Analyzing links from page %s' % page.url) logger.indent += 2 try: page_versions.extend( self._package_versions(page.links, req.name.lower())) finally: logger.indent -= 2 dependency_versions = list( self._package_versions( [Link(url) for url in self.dependency_links], req.name.lower())) if dependency_versions: logger.info('dependency_links found: %s' % ', '.join( [link.url for parsed, link, version in dependency_versions])) file_versions = list( self._package_versions([Link(url) for url in file_locations], req.name.lower())) if not found_versions and not page_versions and not dependency_versions and not file_versions: logger.fatal( 'Could not find any downloads that satisfy the requirement %s' % req) raise DistributionNotFound('No distributions at all found for %s' % req) if req.satisfied_by is not None: found_versions.append((req.satisfied_by.parsed_version, Inf, req.satisfied_by.version)) if file_versions: file_versions.sort(reverse=True) logger.info('Local files found: %s' % ', '.join([ url_to_path(link.url) for parsed, link, version in file_versions ])) found_versions = file_versions + found_versions all_versions = found_versions + page_versions + dependency_versions applicable_versions = [] for (parsed_version, link, version) in all_versions: if version not in req.req: logger.info("Ignoring link %s, version %s doesn't match %s" % (link, version, ','.join( [''.join(s) for s in req.req.specs]))) continue applicable_versions.append((link, version)) applicable_versions = sorted( applicable_versions, key=lambda v: pkg_resources.parse_version(v[1]), reverse=True) existing_applicable = bool( [link for link, version in applicable_versions if link is Inf]) if not upgrade and existing_applicable: if applicable_versions[0][1] is Inf: logger.info( 'Existing installed version (%s) is most up-to-date and satisfies requirement' % req.satisfied_by.version) raise BestVersionAlreadyInstalled else: logger.info( 'Existing installed version (%s) satisfies requirement (most up-to-date version is %s)' % (req.satisfied_by.version, applicable_versions[0][1])) return None if not applicable_versions: logger.fatal( 'Could not find a version that satisfies the requirement %s (from versions: %s)' % (req, ', '.join([ version for parsed_version, link, version in found_versions ]))) raise DistributionNotFound( 'No distributions matching the version for %s' % req) if applicable_versions[0][0] is Inf: # We have an existing version, and its the best version logger.info( 'Installed version (%s) is most up-to-date (past versions: %s)' % (req.satisfied_by.version, ', '.join( [version for link, version in applicable_versions[1:]]) or 'none')) raise BestVersionAlreadyInstalled if len(applicable_versions) > 1: logger.info('Using version %s (newest of versions: %s)' % (applicable_versions[0][1], ', '.join( [version for link, version in applicable_versions]))) return applicable_versions[0][0]
def main(self, args, initial_options): options, args = self.parser.parse_args(args) self.merge_options(initial_options, options) level = 1 # Notify level += options.verbose level -= options.quiet level = logger.level_for_integer(4 - level) complete_log = [] logger.consumers.extend([(level, sys.stdout), (logger.DEBUG, complete_log.append)]) if options.log_explicit_levels: logger.explicit_levels = True self.setup_logging() #TODO: try to get these passing down from the command? # without resorting to os.environ to hold these. if options.no_input: os.environ['PIP_NO_INPUT'] = '1' if options.exists_action: os.environ['PIP_EXISTS_ACTION'] = ''.join(options.exists_action) if not ssl and options.insecure: os.environ['PIP_INSECURE'] = '1' if options.cert: os.environ['PIP_CERT'] = options.cert if options.require_venv: # If a venv is required check if it can really be found if not os.environ.get('VIRTUAL_ENV'): logger.fatal( 'Could not find an activated virtualenv (required).') sys.exit(VIRTUALENV_NOT_FOUND) if options.log: log_fp = open_logfile(options.log, 'a') logger.consumers.append((logger.DEBUG, log_fp)) else: log_fp = None socket.setdefaulttimeout(options.timeout or None) urlopen.setup(proxystr=options.proxy, prompting=not options.no_input) exit = SUCCESS store_log = False try: status = self.run(options, args) # FIXME: all commands should return an exit status # and when it is done, isinstance is not needed anymore if isinstance(status, int): exit = status except (InstallationError, UninstallationError): e = sys.exc_info()[1] logger.fatal(str(e)) logger.info('Exception information:\n%s' % format_exc()) store_log = True exit = ERROR except BadCommand: e = sys.exc_info()[1] logger.fatal(str(e)) logger.info('Exception information:\n%s' % format_exc()) store_log = True exit = ERROR except CommandError: e = sys.exc_info()[1] logger.fatal('ERROR: %s' % e) logger.info('Exception information:\n%s' % format_exc()) exit = ERROR except KeyboardInterrupt: logger.fatal('Operation cancelled by user') logger.info('Exception information:\n%s' % format_exc()) store_log = True exit = ERROR except: logger.fatal('Exception:\n%s' % format_exc()) store_log = True exit = UNKNOWN_ERROR if log_fp is not None: log_fp.close() if store_log: log_fn = options.log_file text = '\n'.join(complete_log) try: log_fp = open_logfile(log_fn, 'w') except IOError: temp = tempfile.NamedTemporaryFile(delete=False) log_fn = temp.name log_fp = open_logfile(log_fn, 'w') logger.fatal('Storing complete log in %s' % log_fn) log_fp.write(text) log_fp.close() return exit