def upgrade(self, package_name, json, version): ''' Upgrade a package to the most recent version. ''' logger.info('Removing {0} old version', package_name) self.remove_files(package_name) args_manager['install']['upgrade'] = True logger.info('Upgrading {0} to {1}', package_name, version) logger.indent += 8 for release in json['urls']: logger.info('Installing {0}...', release['filename']) logger.indent += 4 try: Installer.from_url(release['url']) break except Exception as e: try: msg = e.args[0] except IndexError: msg = repr(e) logger.error('Error: An error occurred while installing {0}: {1}', package_name, msg) logger.info('Trying another file...') logger.indent -= 4 else: logger.warn('Error: Did not find any installable release on PyPI for {0}', package_name) try: Requirement('{0}=={1}'.format(package_name, version))._install_from_links(args_manager['install']['packages_url']) except Exception as e: logger.fatal('Error: {0}', e, exc=InstallationError) logger.info('Restoring uninstalled files') self.restore_files(package_name) logger.indent = 0
def remove_files(self, package): uninst = Uninstaller(package, yes=True) to_del = uninst.find_files() if not to_del: logger.info('No files to remove found') return # XXX: tempfile.mktemp is deprecated, but all other functions create # the directory and shutil does not want that. tempdir = tempfile.mktemp() self.removed[package] = {} self.removed[package][tempdir] = [] for path in to_del: self.removed[package][tempdir].append(path) # We store files-to-delete into a temporary directory: # if something goes wrong during the upgrading we can # restore the original files. p = os.path.join(tempdir, os.path.basename(path)) try: shutil.copy2(path, p) # It is a directory except IOError: try: shutil.copytree(path, p) except OSError: logger.debug('debug: shutil.copytree raised OSError') continue logger.enabled = False uninst.uninstall() logger.enabled = True
def update(self): ''' Searches for updates for every package in the WorkingSet. Then calls :meth:`~pyg.inst.Updater.upgrade`. ''' logger.info('Searching for updates') for dist in self.working_set: package = dist.project_name version = Version(dist.version) logger.verbose('Found: {0}=={1}', package, version) try: json = PyPIJson(package).retrieve() new_version = Version(json['info']['version']) except Exception as e: logger.error('Error: Failed to fetch data for {0} ({1})', package, e) continue if version >= new_version: continue txt = 'A new release is avaiable for {0}: {1!s} (old {2}), update'.format(package, new_version, dist.version) u = logger.ask(txt, bool=('upgrade version', 'keep working version'), dont_ask=self.yes) if u: self.upgrade(package, json, new_version) else: logger.info('{0} has not been upgraded', package) self._clean() logger.success('Updating finished successfully')
def _find_develop(self, dir, req): try: logger.info('Looking for a local package...') try: dist = Develop(req.name) except (ValueError, AttributeError): logger.error('Cannot find a local distribution for {0}', req.name, exc=PygError) else: # XXX: Add a `location` attribute in pkgtools' next release location = os.path.abspath(dist._arg_name) path = os.path.dirname(location) if not req.match(Version(dist.version)): logger.error('Found {0}, but it does not match the requirement', path, exc=PygError) setup_py = os.path.join(path, 'setup.py') if not os.path.exists(setup_py): logger.error('Cannot find setup.py for {0}', req.name, exc=PygError) logger.info('Found a matching package in {0}', path) with TempDir() as tempdir: code, output = call_setup(path, ['sdist', '-d', tempdir]) if code != 0: logger.fatal('setup.py failed to create the source distribution') print_output(output, 'setup.py sdist') raise PygError arch = glob.glob(os.path.join(tempdir, '*{0}*'.format(req.name)))[0] shutil.move(arch, dir) arch_name = os.path.join(dir, os.path.basename(arch)) unpack(arch_name) return arch_name except (PygError, IndexError, ValueError): return False
def install(self): try: r = Requirement(self.req) updater = Updater(skip=True) if self.upgrading: updater.remove_files(self.req) r.install() # Now let's install dependencies Installer._install_deps(r.reqset, r.name, updater) logger.success('{0} installed successfully', r.name) except (KeyboardInterrupt, Exception) as e: try: msg = e.args[0] except IndexError: msg = repr(e) if isinstance(e, KeyboardInterrupt): logger.warn('Process interrupted...') else: logger.warn('Error: An error occurred during the {0} of {1}: {2}', 'upgrading' if self.upgrading else 'installation', self.req, msg) if self.upgrading: logger.info('Restoring files...') updater.restore_files(self.req) else: logger.info('Removing broken files...') Uninstaller(self.req).uninstall() logger.error(msg, exc=InstallationError)
def remove_func(packname, req_file, yes, info): if info: for p in packname: logger.info('{0}:', p) files = Uninstaller(p).find_files() logger.indent = 8 for path in files: logger.info(path) logger.indent = 0 return check_and_exit() ## Little Easter egg... if len(packname) == 1 and packname[0] == 'yourself': return Uninstaller('pyg', yes).uninstall() if req_file: with open(os.path.abspath(req_file)) as f: for line in f: try: Uninstaller(line.strip(), yes).uninstall() except PygError: continue for p in packname: try: Uninstaller(p, yes).uninstall() except PygError: continue
def call_cmd(self, args): ## Ensure that the destination directory exists self.check_dest() if self.ARGS is not None: args = self.ARGS + args logger.info('Copying data from {0} to {1}', self.url, self.dest) return call_subprocess([self.cmd, self.method] + args, cwd=self.dest)
def print_output(output, cmd): '''Print to sys.stderr the complete output of a failed command''' logger.info('Complete output from command `{0}`:', cmd) logger.indent += 8 for line in output.splitlines(): logger.error(line) logger.indent -= 8
def develop(self): logger.info('Running setup.py develop for {0}', self.package_name) code, output = call_setup(self.dest, ['develop']) if code != 0: logger.fatal('Error: setup.py develop did not install {0}', self.package_name) print_output(output, 'setup.py develop') raise InstallationError('setup.py did not install {0}'.format(self.package_name)) logger.info('{0} installed succesfully', self.package_name)
def from_url(url, packname=None): with TempDir() as tempdir: packname = packname or urlparse.urlsplit(url).path.split('/')[-1] if '#egg=' in url: url, packname = url.split('#egg=') path = os.path.join(tempdir, packname) logger.info('Installing {0} from {1}', packname, url) with open(path, 'w') as f: f.write(request(url)) Installer.from_file(path, packname)
def unlink(path): path = os.path.abspath(path) with open(PYG_LINKS) as f: lines = f.readlines() with open(PYG_LINKS, 'w') as f: for line in lines: if line.strip() == path: logger.info('Removing {0} from {1}...', path, PYG_LINKS) continue f.write(line)
def develop(self): self.retrieve_data() if not os.path.exists(os.path.join(self.dir, 'setup.py')): logger.fatal('Error: The repository must have a top-level setup.py file', exc=InstallationError) logger.info('Running setup.py develop for {0}', self.package_name) code, output = call_setup(self.dir, ['develop']) if code != 0: logger.fatal('Error: setup.py develop did not install {0}', self.package_name) print_output(output, 'setup.py develop') raise InstallationError('setup.py did not install {0}'.format(self.package_name)) logger.info('{0} installed succesfully', self.package_name)
def __init__(self, skip=False): ## You should use skip=True when you want to upgrade a single package. ## Just do: ##>>> u = Updater(skip=True) ##>>> u.upgrade(package_name, json, version) if not skip: logger.debug('Loading list of installed packages... ', addn=False) self.working_set = list(iter(pkg_resources.working_set)) logger.info('{0} packages loaded', len(self.working_set)) self.removed = {}
def __init__(self, req): self.upgrading = False if is_installed(req): self.upgrading = True if not args_manager['install']['upgrade']: logger.info('{0} is already installed, use -U, --upgrade to upgrade', req) raise AlreadyInstalled ## We don't set args_manager['upgrade'] = False ## because we want to propagate it to dependencies logger.info('{0} is already installed, upgrading...', req) self.req = req
def install(self): ''' Install a bundle. For every package runs ``setup.py install``. ''' with TempDir() as tempdir: with ZipFile(self.path) as z: z.extractall(tempdir) location = os.path.join(tempdir, 'build') for f in os.listdir(location): logger.info('Installing {0}...', f) fullpath = os.path.join(location, f) Dir(fullpath, f, tempdir).install() logger.success('Bundle installed successfully')
def download_func(args): pref = None if args.prefer: pref = ['.' + args.prefer.strip('.')] name = args.packname dest = args_manager['download']['download_dir'] unpk = args_manager['download']['unpack'] downloader = ReqManager(Requirement(name), pref) downloader.download(dest) if downloader.downloaded_name is None: logger.fatal('Error: Did not find any files for {0}', name, exc=PygError) if unpk: path = os.path.abspath(downloader.downloaded_name) logger.info('Unpacking {0} to {1}', os.path.basename(path), os.getcwd()) unpack(path)
def link(path): path = os.path.abspath(path) if not os.path.exists(path): logger.error('{0} does not exist', path) if not os.path.exists(PYG_LINKS): if not os.path.exists(os.path.dirname(PYG_LINKS)): os.makedirs(os.path.dirname(PYG_LINKS)) open(PYG_LINKS, 'w').close() path = os.path.abspath(path) logger.info('Linking {0} in {1}...', path, PYG_LINKS) if path in open(PYG_LINKS, 'r').read(): logger.warn('{0} is already linked, exiting now...', path) with open(PYG_LINKS, 'a') as f: f.write(path) f.write('\n')
def _download_all(self, dir): reqs = list(self.reqs) already_downloaded = set() while reqs: r = reqs.pop() # Hack for virtualenvs if r.name.lower == 'python': continue if any(r.name == rq.name for rq in already_downloaded): logger.debug('debug: Already downloaded: {0}', r) continue if any(r == rq for rq in self.exclude): logger.info('Excluding {0}', r) continue logger.indent = 0 logger.info('{0}:', r) logger.indent = 8 try: dist = SDist(self._download(dir, r)) self.callback(r, dist) except ConfigParser.MissingSectionHeaderError: continue try: logger.info('Looking for {0} dependencies', r) logger.indent += 8 found = False try: requirements = dist.file('requires.txt') except KeyError: requirements = [] for requirement in requirements: rq = Requirement(requirement) if rq not in already_downloaded: logger.info('Found: {0}', requirement) reqs.append(rq) found = True if not found: logger.info('None found') except KeyError: logger.debug('debug: requires.txt not found for {0}', dist) try: as_req = dist.as_req except KeyError: as_req = str(r) already_downloaded.add(Requirement(as_req)) logger.indent = 0 logger.success('Finished processing dependencies')
def from_dir(path, name=None): name = name or os.path.basename(path) reqset = ReqSet(name) try: with TempDir() as tempdir: logger.info('Installing {0}', name) Dir(path, name, tempdir, reqset).install() except Exception as e: try: msg = e.args[0] except IndexError: msg = repr(e) logger.fatal('Error: {0}: cannot install the package', msg, exc=InstallationError) else: if reqset: Installer._install_deps(reqset) logger.success('{0} installed successfully', name)
def get_links(package, index_url=None): ## Correction for standard installations when index_url looks standard ## http://pypi.python.org/pypi. if index_url is None: index_url = args_manager['install']['packages_url'] logger.info('Looking for packages at {0}', index_url) urls = set() package_index = PygPackageIndex(index_url) req = pkg_resources.Requirement.parse(str(package)) for source in (True, False): package_index.fetch_distribution(req, None, force_scan=True, \ source=source, develop_ok=False) for url in package_index.urls: ## PackageIndex looks for local distributions too, and we ## don't want that. if url.startswith(('http', 'https')): urls.add(urlparse.urldefrag(url)[0]) return urls
def _install_package_from_name(package, ignore=False): if os.path.exists(package) and not ignore: path = os.path.abspath(package) logger.info('Installing {0}', path) if os.path.isfile(path): return Installer.from_file(path) elif os.path.isdir(path): if not os.path.exists(os.path.join(path, 'setup.py')): raise PygError('{0} must contain the setup.py file', path) return Installer.from_dir(path) else: raise PygError('Cannot install the package: {0} is neither a file nor a directory', path) if package.startswith(('http://', 'https://')): return Installer.from_url(package) for s in ('git+', 'hg+', 'bzr+', 'svn+'): if package.startswith(s): with TempDir() as tempdir: return vcs(package, tempdir).install() return Installer(package).install()
def download(self, dest): dest = os.path.abspath(dest) files = self.files() ## We need a placeholder because of the nested for loops success = False for p in self.pref: if success: break if not files[p]: logger.warn('{0} files not found. Continue searching...', p) continue for v, name, hash, url in files[p]: if success: break if p == '.egg' and not right_egg(name): logger.info('Found egg file for another Python version: {0}. Continue searching...', version_egg(name)) continue try: data = download(url, 'Retrieving data for {0}'.format(self.name)).getvalue() except (urllib2.URLError, urllib2.HTTPError) as e: logger.debug('urllib2 error: {0}', e.args) continue if not data: logger.debug('debug: Request failed') continue if not os.path.exists(dest): os.makedirs(dest) try: # Fix for packages with no version in the name if '-' not in name: name = '{0}-{1}{2}'.format(name, v, p) logger.info('Writing data into {0}', name) with open(os.path.join(dest, name), 'w') as f: f.write(data) except (IOError, OSError): logger.debug('debug: Error while writing data') continue logger.success('{0} downloaded successfully', self.name) success = True self.downloaded_name = name self.downloaded_version = v
def install_func(packname, req_file, editable, ignore, yes): check_and_exit() if editable: if len(packname) > 1: logger.error('Error: Unable to install multiple packages in editable mode') return package = packname[0] if os.path.exists(os.path.abspath(package)): package = 'dir+{0}#egg={1}'.format(os.path.abspath(package), os.path.basename(package)) return vcs(package).develop() if req_file: logger.info('Installing from requirements file') for rq in req_file: Installer.from_req_file(os.path.abspath(rq)) return if packname: for package in packname: _install_package_from_name(package, ignore)
def _download_and_install(self, url, filename, packname, e, hash=None): fobj = download(url, 'Downloading {0}'.format(self.name)) if hash is not None: logger.info('Checking md5 sum') if md5(fobj.getvalue()).hexdigest() != hash: logger.fatal('Error: {0} appears to be corrupted', self.name) return if e in ('.tar.gz', '.tar.bz2', '.zip'): installer = Archive(fobj, e, packname, self.reqset) elif e == '.egg': installer = Egg(fobj, filename, self.reqset, packname) elif is_windows() and e in WINDOWS_EXT: installer = Binary(fobj, e, packname) else: logger.error('Error: unknown filetype: {0}', e, exc=InstallationError) ## There is no need to catch exceptions now, this will be done by `pyg.inst.Installer.install` installer.install() self.success = True
def run_setup(path, name, global_args=[], args=[], exc=TypeError): ''' Run `setup.py install` for the given setup file. `name` is the package name; `global_args` are the arguments to pass before the `install` command; `args` are the options for the `install` command and `exc` is the exception to throw in case of a failed installation. The warning for `path` is the same as `call_setup`. ''' logger.info('Running setup.py install for {0}', name) if name == 'dreampie': ar = global_args + ['install'] + args else: ar = global_args + ['install', '--single-version-externally-managed', '--record', os.path.join(tempfile.mkdtemp(), '.pyg-install-record')] + args code, output = call_setup(path, ar) if code != 0: logger.fatal('Error: setup.py did not install {0}', name) print_output(output, 'setup.py install') raise exc('setup.py did not install {0}'.format(name))
def _install_deps(rs, name=None, updater=None): if not rs: return if args_manager['install']['no_deps']: logger.info('Skipping dependencies for {0}', name) logger.indent = 8 for req in rs: logger.info(req) logger.indent = 0 return logger.info('Installing dependencies...') dep_error = False newly_installed = [] for req in rs: if is_installed(req) and not args_manager['install']['upgrade_all']: logger.indent = 8 logger.info('{0} is already installed, use -A, --upgrade-all to upgrade dependencies', req) continue logger.indent = 0 logger.info('Installing {0} (from {1})', req, rs.comes_from) logger.indent = 8 try: Installer(req).install() newly_installed.append(req) except AlreadyInstalled: continue except InstallationError: dep_error = True logger.error('Error: {0} has not been installed correctly', req) continue logger.indent = 0 if dep_error: if updater: for req in newly_installed: updater.restore_files(req) updater.remove_files(rs.comes_from.name) updater.restore_files(rs.comes_from.name) logger.error("{0}'s dependencies installation failed", rs.comes_from.name, exc=InstallationError) else: logger.success('Finished installing dependencies for {0}', rs.comes_from.name)
def check_dest(self): ## If the target directory is empty we don't have to worry try: if not os.listdir(self.dest): return ## If self.dest does not exist we leave it as it is, because it will be ## created by the `else` clause (below) except OSError: pass if os.path.exists(self.dest): txt = 'The destination already exists: {0}\nWhat do you want to do ?'.format(self.dest) u = logger.ask(txt, choices={'destroy': 'd', 'backup': 'b', 'exit': 'x'}) if u == 'd': logger.info('Removing {0}...', self.dest) shutil.rmtree(self.dest) os.makedirs(self.dest) elif u == 'b': dst = os.path.join(os.path.dirname(self.dest), self.dest + '-pyg-backup') logger.info('Moving {0} to {1}', self.dest, dst) shutil.move(self.dest, dst) os.makedirs(self.dest) elif u == 'x': logger.info('Exiting...') sys.exit(0) else: os.makedirs(self.dest)
def from_req_file(filepath): path = os.path.abspath(filepath) not_installed = set() parser = init_parser() with open(path) as f: logger.info('{0}:', path) for line in f: line = line.strip() if line.startswith('#'): logger.debug('debug: Comment found: {0}', line) continue try: logger.indent = 8 logger.info('Installing: {0}', line) logger.indent = 16 parser.dispatch(argv=['install'] + line.split()) except AlreadyInstalled: continue except InstallationError: not_installed.add(line) except SystemExit as e: if e.code != 0: logger.warn('W: {0} tried to raise SystemExit: skipping installation') else: logger.info('{0} tried to raise SystemExit, but the exit code was 0') if not_installed: logger.warn('These packages have not been installed:') logger.indent = 8 for req in not_installed: logger.warn(req) logger.indent = 0 raise InstallationError()
def uninstall(self): path_re = re.compile(r'\./{0}-[\d\w\.]+-py\d\.\d.egg'.format(self.name), re.I) path_re2 = re.compile(r'\.{0}'.format(self.name), re.I) to_del = self.find_files() if not to_del: logger.warn('{0}: did not find any files to delete', self.name) raise PygError logger.info('Uninstalling {0}', self.name) logger.indent += 8 for d in to_del: logger.info(d) logger.indent -= 8 do_it = logger.ask('Proceed', bool=('remove files', 'cancel'), dont_ask=args_manager['install']['yes']) if do_it: for d in to_del: try: logger.verbose('Deleting: {0}', d) shutil.rmtree(d) except OSError: ## It is not a directory try: os.remove(d) except OSError: logger.error('Error: cannot delete {0}', d) logger.verbose('Removing egg path from easy_install.pth...') with open(EASY_INSTALL) as f: lines = f.readlines() with open(EASY_INSTALL, 'w') as f: for line in lines: if path_re.match(line) or path_re2.match(line): continue f.write(line) logger.success('{0} uninstalled succesfully', self.name) else: logger.info('{0} has not been uninstalled', self.name)
def install(self): ''' Given a pathname containing a :file:`setup.py` file, install the package. First runs `setup.py egg_info` to find out requirements, then runs ``setup.py install``. ''' if self.reqset is not None: logger.info('Running setup.py egg_info for {0}', self.name) call_setup(self.path, ['egg_info', '--egg-base', self.tempdir]) try: dist = DirTools(glob.glob(os.path.join(self.path, '*egg-info'))[0]) except (IndexError, ValueError): pass else: try: for r in dist.file('requires.txt'): self.reqset.add(r) except (KeyError, ConfigParser.MissingSectionHeaderError): logger.debug('debug: requires.txt not found') #try: # for r in dist.file('dependency_links.txt'): # self.reqset.add(r) #except (KeyError, ConfigParser.MissingSectionHeaderError): # logger.debug('debug: dependency_links.txt not found') args = [] if args_manager['install']['install_dir'] != INSTALL_DIR: dir = os.path.abspath(args_manager['install']['install_dir']) if not os.path.exists(dir): os.makedirs(dir) ## Setuptools would not install the package without this hack os.putenv('PYTHONPATH', dir) args += ['--install-purelib', dir, '--install-platlib', dir] if args_manager['install']['no_scripts']: args += ['--install-scripts', self.tempdir] if args_manager['install']['no_data']: args += ['--install-data', self.tempdir] if args_manager['install']['user']: args += ['--user'] run_setup(self.path, self.name, args=args, exc=InstallationError)