def test_latest_vulndb(self): dists = Distributions() pkg = 'vulndb' found = None pypi = CheeseShop(False) all_dists = dists.get_distributions('all', pkg, dists.get_highest_installed(pkg)) for dist, active in all_dists: project_name, versions = pypi.query_versions_pypi( dist.project_name) if versions: # PyPI returns them in chronological order, # but who knows if its guaranteed in the API? # Make sure we grab the highest version: newest = get_highest_version(versions) if newest != dist.version: # We may have newer than what PyPI knows about if pkg_resources.parse_version(dist.version) < \ pkg_resources.parse_version(newest): found = True if found: self.assertTrue(False, MESSAGE)
def get_fresh_updates(package_name="", version=""): userpath = expanduser("~") now = datetime.now() # Do we have a cache ? if isfile(userpath + "/.qyolk"): f = open(userpath + "/.qyolk", "r") cache = cPickle.load(f) check_time = now - timedelta(hours=24) if cache[0] > check_time: # fresh cache, use it return cache[1] # No cache, get updates and create the cache ret = [] pypi = CheeseShop() dists = Distributions() for pkg in get_pkglist(): for (dist, active) in dists.get_distributions( "all", pkg, dists.get_highest_installed(pkg)): (project_name, versions) = pypi.query_versions_pypi(dist.project_name) if versions: newest = get_highest_version(versions) if newest != dist.version: if pkg_resources.parse_version( dist.version) < pkg_resources.parse_version( newest): ret.append([project_name, dist.version, newest]) f = open(userpath + "/.qyolk", "w") cPickle.dump([now, ret], f) return ret
def get_fresh_updates(package_name="", version=""): userpath = expanduser("~") now = datetime.now() # Do we have a cache ? if isfile(userpath + "/.qyolk"): f = open(userpath + "/.qyolk", "r") cache = cPickle.load(f) check_time = now - timedelta(hours=24) if cache[0] > check_time: # fresh cache, use it return cache[1] # No cache, get updates and create the cache ret = [] pypi = CheeseShop() dists = Distributions() for pkg in get_pkglist(): for (dist, active) in dists.get_distributions("all", pkg, dists.get_highest_installed(pkg)): (project_name, versions) = pypi.query_versions_pypi(dist.project_name) if versions: newest = get_highest_version(versions) if newest != dist.version: if pkg_resources.parse_version(dist.version) < pkg_resources.parse_version(newest): ret.append([project_name, dist.version, newest]) f = open(userpath + "/.qyolk", "w") cPickle.dump([now, ret], f) return ret
class TestPyPi(BaseTestCase): """""" def setUp(self): self.pypi = CheeseShop() self.all_packages = [] for package in self.pypi.list_packages(): (pn, vers) = self.pypi.query_versions_pypi(package) for version in vers: try: url = self.pypi.get_download_urls(pn, version)[0] except IndexError: pass # TODO: log how many packages do not have URL else: self.all_packages.append((pn, version)) # TODO: cache entries with cPickle #def test_get_vars_against_pypi(self): #for package_name, version in self.all_packages: #try: #d = Enamer.get_vars(url, pn, version) #except: #pass ## TODO: maybe assert some of dictionary stuff? #self.fail('Fail!') def test_get_vars_against_pypi(self): for package_name, version in self.all_packages: #try: main(['create', package_name, version]) #except: #pass # TODO: maybe assert some of dictionary stuff? self.fail('Fail!')
def sync(self): """""" pypi = CheeseShop() for package in pypi.list_packages(): (pn, vers) = pypi.query_versions_pypi(package) for version in vers: # TODO: parse_* will not return anything for correct atoms atom = Enamer.construct_atom( Enamer.parse_pn(pn)[0], self.config.category, Enamer.parse_pv(version[0])) # we skip existing ebuilds if PortageUtils.ebuild_exists(atom): continue try: url = pypi.get_download_urls(pn, version)[0] # TODO: use setuptools way also except IndexError: log.warn('Skipping %s, no download url', atom) else: try: self.config.configs['argparse']['uri'] = url self.config.configs['argparse']['up_pn'] = pn self.config.configs['argparse']['up_pv'] = version gpypi = GPyPI(pn, version, self.config) gpypi.create_ebuilds() except KeyboardInterrupt: raise except: log.exception( 'Unexpected error occured during ebuild creation:')
def sync(self): """""" pypi = CheeseShop() for package in pypi.list_packages(): (pn, vers) = pypi.query_versions_pypi(package) for version in vers: # TODO: parse_* will not return anything for correct atoms atom = Enamer.construct_atom(Enamer.parse_pn(pn)[0], self.config.category, Enamer.parse_pv(version[0])) # we skip existing ebuilds if PortageUtils.ebuild_exists(atom): continue try: url = pypi.get_download_urls(pn, version)[0] # TODO: use setuptools way also except IndexError: log.warn('Skipping %s, no download url', atom) else: try: self.config.configs['argparse']['uri'] = url self.config.configs['argparse']['up_pn'] = pn self.config.configs['argparse']['up_pv'] = version gpypi = GPyPI(pn, version, self.config) gpypi.create_ebuilds() except KeyboardInterrupt: raise except: log.exception('Unexpected error occured during ebuild creation:')
def test_latest_vulndb(self): dists = Distributions() pkg = 'vulndb' found = None pypi = CheeseShop(False) all_dists = dists.get_distributions('all', pkg, dists.get_highest_installed(pkg)) for dist, active in all_dists: project_name, versions = pypi.query_versions_pypi(dist.project_name) if versions: # PyPI returns them in chronological order, # but who knows if its guaranteed in the API? # Make sure we grab the highest version: newest = get_highest_version(versions) if newest != dist.version: #We may have newer than what PyPI knows about if pkg_resources.parse_version(dist.version) < \ pkg_resources.parse_version(newest): found = True if found: self.assertTrue(False, MESSAGE)
def run(self): """ Perform actions based on CLI options @returns: status code """ opt_parser = setup_opt_parser() (self.options, remaining_args) = opt_parser.parse_args() logger = self.set_log_level() pkg_spec = validate_pypi_opts(opt_parser) if not pkg_spec: pkg_spec = remaining_args self.pkg_spec = pkg_spec if not self.options.pypi_search and (len(sys.argv) == 1 or\ len(remaining_args) > 2): opt_parser.print_help() return 2 #Options that depend on querying installed packages, not PyPI. #We find the proper case for package names if they are installed, #otherwise PyPI returns the correct case. if self.options.show_deps or self.options.show_all or \ self.options.show_active or self.options.show_non_active or \ (self.options.show_updates and pkg_spec): want_installed = True else: want_installed = False #show_updates may or may not have a pkg_spec if not want_installed or self.options.show_updates: self.pypi = CheeseShop(self.options.debug) #XXX: We should return 2 here if we couldn't create xmlrpc server if pkg_spec: (self.project_name, self.version, self.all_versions) = \ self.parse_pkg_ver(want_installed) if want_installed and not self.project_name: logger.error("%s is not installed." % pkg_spec[0]) return 1 #I could prefix all these with 'cmd_' and the methods also #and then iterate over the `options` dictionary keys... commands = [ 'show_deps', 'query_metadata_pypi', 'fetch', 'versions_available', 'show_updates', 'browse_website', 'show_download_links', 'pypi_search', 'show_pypi_changelog', 'show_pypi_releases', 'yolk_version', 'show_all', 'show_active', 'show_non_active', 'show_entry_map', 'show_entry_points' ] #Run first command it finds, and only the first command, then return #XXX: Check if more than one command was set in options and give error? for action in commands: if getattr(self.options, action): return getattr(self, action)() opt_parser.print_help()
def setUp(self): self.pypi = CheeseShop() self.all_packages = [] for package in self.pypi.list_packages(): (pn, vers) = self.pypi.query_versions_pypi(package) for version in vers: try: url = self.pypi.get_download_urls(pn, version)[0] except IndexError: pass # TODO: log how many packages do not have URL else: self.all_packages.append((pn, version))
def get_latest_version_number(package_name): """Receives a package name and returns the latest version its available in from pip""" pkg, all_versions = CheeseShop().query_versions_pypi(package_name) if len(all_versions): return all_versions[0] return None
def get_fresh_updates(package_name="", version=""): ret = [] pypi = CheeseShop() for pkg in get_pkglist(): for (dist, active) in get_distributions("all", pkg, get_highest_installed(pkg)): (project_name, versions) = pypi.query_versions_pypi(dist.project_name) if versions: newest = get_highest_version(versions) if newest != dist.version: if pkg_resources.parse_version( dist.version) < pkg_resources.parse_version( newest): ret.append([project_name, dist.version, newest]) return ret
def run(self): """ Perform actions based on CLI options @returns: status code """ opt_parser = setup_opt_parser() (self.options, remaining_args) = opt_parser.parse_args() logger = self.set_log_level() pkg_spec = validate_pypi_opts(opt_parser) if not pkg_spec: pkg_spec = remaining_args self.pkg_spec = pkg_spec if not self.options.pypi_search and (len(sys.argv) == 1 or\ len(remaining_args) > 2): opt_parser.print_help() return 2 #Options that depend on querying installed packages, not PyPI. #We find the proper case for package names if they are installed, #otherwise PyPI returns the correct case. if self.options.show_deps or self.options.show_all or \ self.options.show_active or self.options.show_non_active or \ (self.options.show_updates and pkg_spec): want_installed = True else: want_installed = False #show_updates may or may not have a pkg_spec if not want_installed or self.options.show_updates: self.pypi = CheeseShop(self.options.debug) #XXX: We should return 2 here if we couldn't create xmlrpc server if pkg_spec: (self.project_name, self.version, self.all_versions) = \ self.parse_pkg_ver(want_installed) if want_installed and not self.project_name: logger.error("%s is not installed." % pkg_spec[0]) return 1 #I could prefix all these with 'cmd_' and the methods also #and then iterate over the `options` dictionary keys... commands = ['show_deps', 'query_metadata_pypi', 'fetch', 'versions_available', 'show_updates', 'browse_website', 'show_download_links', 'pypi_search', 'show_pypi_changelog', 'show_pypi_releases', 'yolk_version', 'show_all', 'show_active', 'show_non_active', 'show_entry_map', 'show_entry_points'] #Run first command it finds, and only the first command, then return #XXX: Check if more than one command was set in options and give error? for action in commands: if getattr(self.options, action): return getattr(self, action)() opt_parser.print_help()
def run(self): """Perform actions based on CLI options. @returns: status code """ parser = setup_parser() try: import argcomplete argcomplete.autocomplete(parser) except ImportError: pass self.options = parser.parse_args() pkg_spec = validate_pypi_opts(parser) if not pkg_spec: pkg_spec = self.options.pkg_spec self.pkg_spec = pkg_spec if self.options.fields: self.options.fields = [ s.strip().lower() for s in self.options.fields.split(',') ] else: self.options.fields = [] if (not self.options.pypi_search and len(sys.argv) == 1): parser.print_help() return 2 # Options that depend on querying installed packages, not PyPI. # We find the proper case for package names if they are installed, # otherwise PyPI returns the correct case. if (self.options.show_deps or self.options.show_all or self.options.show_active or self.options.show_non_active or (self.options.show_updates and pkg_spec) or self.options.upgrade): want_installed = True else: want_installed = False # show_updates may or may not have a pkg_spec if (not want_installed or self.options.show_updates or self.options.upgrade): self.pypi = CheeseShop(self.options.debug) # XXX: We should return 2 here if we couldn't create xmlrpc server if pkg_spec: (self.project_name, self.version, self.all_versions) = self.parse_pkg_ver(want_installed) if want_installed and not self.project_name: print(u'{} is not installed'.format(pkg_spec), file=sys.stderr) return 1 # I could prefix all these with 'cmd_' and the methods also # and then iterate over the `options` dictionary keys... commands = [ 'show_deps', 'query_metadata_pypi', 'fetch', 'versions_available', 'show_updates', 'upgrade', 'browse_website', 'show_download_links', 'pypi_search', 'show_pypi_changelog', 'show_pypi_releases', 'yolk_version', 'show_all', 'show_active', 'show_non_active', 'show_entry_map', 'show_entry_points' ] # Run first command it finds, and only the first command, then return # XXX: Check if more than one command was set in options and give # error? for action in commands: if getattr(self.options, action): return getattr(self, action)() parser.print_help()
def __init__(self, package_name, version, options): self.package_name = package_name self.version = version self.options = options self.tree = [(package_name, version)] self.pypi = CheeseShop()
class GPyPI(object): """ Main class for GPyPi interface :param package_name: case-insensitive package name :type package_name: string :param version: package version :type version: string :param options: command-line options :type options: ArgParse options """ def __init__(self, package_name, version, options): self.package_name = package_name self.version = version self.options = options self.tree = [(package_name, version)] self.pypi = CheeseShop() def create_ebuilds(self): """ Create ebuild for given package_name and any ebuilds for dependencies if needed. If no version is given we use the highest available. """ while len(self.tree): (project_name, version) = self.tree.pop(0) self.package_name = project_name self.version = version requires = self.do_ebuild() if requires: for req in requires: if self.options.no_deps: pass else: self.handle_dependencies(req.project_name) # TODO: disable some options after first ebuild is created #self.options.overwrite = False #self.options.category = None def handle_dependencies(self, project_name): """Add dependency if not already in self.tree""" pkgs = [] if len(self.tree): for deps in self.tree: pkgs.append(deps[0]) if project_name not in pkgs: # TODO: document that we can not query pypi with version spec or use distutils2 # for dependencies self.tree.append((project_name, None)) log.info("Dependency needed: %s" % project_name) def url_from_pypi(self): """ Query PyPI to find a package's URI :returns: source URL string """ try: return self.pypi.get_download_urls(self.package_name, self.version, pkg_type="source")[0] except IndexError: return None def url_from_setuptools(self): """ Use setuptools to find a package's URI :returns: source URL string or None """ #if self.options.subversion: # src_uri = get_download_uri(self.package_name, "dev", "source") #else: return get_download_uri(self.package_name, self.version, "source", self.options.index_url) def find_uri(self, method="all"): """ Returns download URI for package If no package version was given it returns highest available Setuptools should find anything xml-rpc can and more. :param method: download method can be 'xml-rpc', 'setuptools', or 'all' :type method: string :returns download_url string """ download_url = None if method == "all" or method == "xml-rpc": download_url = self.url_from_pypi() if (method == "all" or method == "setuptools") and not download_url: #Sometimes setuptools can find a package URI if PyPI doesn't have it download_url = self.url_from_setuptools() # TODO: configuratior log.debug("Package URI: %s " % download_url) return download_url def do_ebuild(self): """ Get SRC_URI using PyPI and attempt to create ebuild :returns: tuple with exit code and pkg_resources requirement """ #Get proper case for project name: (self.package_name, versions) = self.pypi.query_versions_pypi(self.package_name) if not versions: log.error("No package %s on PyPi." % self.package_name) return if self.version and (self.version not in versions): log.error("No package %s for version %s on PyPi." % (self.package_name, self.version)) return else: self.version = get_highest_version(versions) # TODO: self.options.uri only for first ebuild # TODO: make find_uri method configurable download_url = self.find_uri() log.info('Generating ebuild: %s %s', self.package_name, self.version) log.debug('URI from PyPi: %s', download_url) self.options.configs['argparse']['uri'] = download_url self.options.configs['argparse']['up_pn'] = self.package_name self.options.configs['argparse']['up_pv'] = self.version ebuild = Ebuild(self.options) ebuild.set_metadata(self.query_metadata()) if self.options.command == 'echo': ebuild.print_formatted() else: ebuild.create() return ebuild.requires def query_metadata(self): """ Get package metadata from PyPI :returns: metadata text """ if self.version: return self.pypi.release_data(self.package_name, self.version) else: (pn, vers) = self.pypi.query_versions_pypi(self.package_name) return self.pypi.release_data(self.package_name, get_highest_version(vers))
class Yolk(object): """Main class for yolk.""" def __init__(self): # PyPI project name with proper case self.project_name = '' # PyPI project version self.version = '' # List of all versions not hidden on PyPI self.all_versions = [] self.pkg_spec = None self.options = None # Squelch output from setuptools # Add future offenders to this list. shut_up = ['distutils.log'] sys.stdout = StdOut(sys.stdout, shut_up) sys.stderr = StdOut(sys.stderr, shut_up) self.pypi = None def get_plugin(self, method): """Return plugin object if CLI option is activated and method exists. @param method: name of plugin's method we're calling @type method: string @returns: list of plugins with `method` """ all_plugins = [] for entry_point in pkg_resources.iter_entry_points('yolk.plugins'): plugin_obj = entry_point.load() plugin = plugin_obj() plugin.configure(self.options, None) if plugin.enabled: if not hasattr(plugin, method): plugin = None else: all_plugins.append(plugin) return all_plugins def run(self): """Perform actions based on CLI options. @returns: status code """ parser = setup_parser() try: import argcomplete argcomplete.autocomplete(parser) except ImportError: pass self.options = parser.parse_args() pkg_spec = validate_pypi_opts(parser) if not pkg_spec: pkg_spec = self.options.pkg_spec self.pkg_spec = pkg_spec if self.options.fields: self.options.fields = [s.strip().lower() for s in self.options.fields.split(',')] else: self.options.fields = [] if ( not self.options.pypi_search and len(sys.argv) == 1 ): parser.print_help() return 2 # Options that depend on querying installed packages, not PyPI. # We find the proper case for package names if they are installed, # otherwise PyPI returns the correct case. if ( self.options.show_deps or self.options.show_all or self.options.show_active or self.options.show_non_active or (self.options.show_updates and pkg_spec) or self.options.upgrade ): want_installed = True else: want_installed = False # show_updates may or may not have a pkg_spec if ( not want_installed or self.options.show_updates or self.options.upgrade ): self.pypi = CheeseShop(self.options.debug) # XXX: We should return 2 here if we couldn't create xmlrpc server if pkg_spec: (self.project_name, self.version, self.all_versions) = self.parse_pkg_ver(want_installed) if want_installed and not self.project_name: print('{} is not installed'.format(pkg_spec), file=sys.stderr) return 1 # I could prefix all these with 'cmd_' and the methods also # and then iterate over the `options` dictionary keys... commands = ['show_deps', 'query_metadata_pypi', 'fetch', 'versions_available', 'show_updates', 'upgrade', 'browse_website', 'show_download_links', 'pypi_search', 'show_pypi_changelog', 'show_pypi_releases', 'yolk_version', 'show_all', 'show_active', 'show_non_active', 'show_entry_map', 'show_entry_points'] # Run first command it finds, and only the first command, then return # XXX: Check if more than one command was set in options and give # error? for action in commands: if getattr(self.options, action): return getattr(self, action)() parser.print_help() def show_active(self): """Show installed active packages.""" return self.show_distributions('active') def show_non_active(self): """Show installed non-active packages.""" return self.show_distributions('nonactive') def show_all(self): """Show all installed packages.""" return self.show_distributions('all') def show_updates(self): """Check installed packages for available updates on PyPI. @param project_name: optional package name to check; checks every installed package if none specified @type project_name: string @returns: None """ if self.project_name: pkg_list = [self.project_name] else: pkg_list = get_pkglist() for (project_name, version, newest) in _updates( pkg_list, self.pypi, user_installs_only=self.options.user): print('{} {} ({})'.format(project_name, version, newest)) return 0 def upgrade(self): """Check installed packages for available updates on PyPI and upgrade. @param project_name: optional package name to check; checks every installed package if none specified @type project_name: string @returns: None """ if self.project_name: pkg_list = [self.project_name] else: pkg_list = get_pkglist() names = [values[0] for values in _updates(pkg_list, self.pypi, user_installs_only=self.options.user)] if names: subprocess.call( [sys.executable, '-m', 'pip', 'install', '--upgrade'] + (['--user'] if self.options.user else []) + names) return 0 def show_distributions(self, show): """Show list of installed activated OR non-activated packages. @param show: type of pkgs to show (all, active or nonactive) @type show: string @returns: None or 2 if error """ # Search for any plugins with active CLI options with add_column() # method. plugins = self.get_plugin('add_column') # Some locations show false positive for 'development' packages: ignores = ['/UNIONFS', '/KNOPPIX.IMG'] # See http://cheeseshop.python.org/pypi/workingenv.py for details. workingenv = os.environ.get('WORKING_ENV') if workingenv: ignores.append(workingenv) results = None for (dist, active) in yolklib.get_distributions(show, self.project_name, self.version): metadata = get_metadata(dist) for prefix in ignores: if dist.location.startswith(prefix): dist.location = dist.location.replace(prefix, '') # Case-insensitive search because of Windows. if dist.location.lower().startswith(get_python_lib().lower()): develop = '' else: develop = dist.location if metadata: add_column_text = '' for my_plugin in plugins: # See if package is 'owned' by a package manager such as # portage, apt, rpm etc. add_column_text += my_plugin.add_column(dist) + ' ' self.print_metadata(metadata, develop, active, add_column_text) else: print(str(dist) + ' has no metadata') results = True if not results and self.project_name: if self.version: pkg_spec = '{}=={}'.format(self.project_name, self.version) else: pkg_spec = self.project_name if show == 'all': print('There are no versions of {} installed'.format(pkg_spec), file=sys.stderr) else: print( 'There are no {} versions of {} installed'.format( show, pkg_spec), file=sys.stderr) return 2 elif show == 'all' and results and self.options.fields: print("Versions with '*' are non-active.") print("Versions with '!' are deployed in development mode.") def print_metadata(self, metadata, develop, active, installed_by): """Print out formatted metadata. @param metadata: package's metadata @type metadata: pkg_resources Distribution obj @param develop: path to pkg if its deployed in development mode @type develop: string @param active: show if package is activated or not @type active: boolean @param installed_by: Shows if pkg was installed by a package manager other than setuptools @type installed_by: string @returns: None """ show_metadata = self.options.metadata version = metadata['Version'] # When showing all packages, note which are not active: if active: if self.options.fields: active_status = '' else: active_status = 'active' else: if self.options.fields: active_status = '*' else: active_status = 'non-active' if develop: if self.options.fields: development_status = '! ({})'.format(develop) else: development_status = 'development ({})'.format(develop) else: development_status = installed_by status = '{} {}'.format(active_status, development_status) if self.options.fields: print( '{} ({}){} {}'.format(metadata['Name'], version, active_status, development_status)) else: # Need intelligent justification. print(metadata['Name'].ljust(15) + ' - ' + version.ljust(12) + ' - ' + status) if self.options.fields: for field in metadata.keys(): if field.lower() in self.options.fields: print(' {}: {}'.format(field, metadata[field])) print() elif show_metadata: for field in metadata.keys(): if field != 'Name' and field != 'Summary': print(' {}: {}'.format(field, metadata[field])) def show_deps(self): """Show dependencies for package(s) @returns: 0 - success 1 - No dependency info supplied """ pkgs = pkg_resources.Environment() for pkg in pkgs[self.project_name]: if not self.version: print(pkg.project_name, pkg.version) i = len(list(pkg._dep_map.values())[0]) if i: while i: if ( not self.version or self.version and pkg.version == self.version ): if self.version and i == len(list( pkg._dep_map.values())[0]): print(pkg.project_name, pkg.version) print(' ' + str(list( pkg._dep_map.values())[0][i - 1])) i -= 1 else: return 1 return 0 def show_pypi_changelog(self): """Show detailed PyPI ChangeLog for the last `hours` @returns: 0 = success or 1 if failed to retrieve from XML-RPC server """ hours = self.options.show_pypi_changelog if not hours.isdigit(): print('You must supply an integer', file=sys.stderr) return 1 try: changelog = self.pypi.changelog(int(hours)) except XMLRPCFault as err_msg: print(err_msg, file=sys.stderr) print("Couldn't retrieve changelog", file=sys.stderr) return 1 last_pkg = '' for entry in changelog: pkg = entry[0] if pkg != last_pkg: print('{} {}\n\t{}'.format(entry[0], entry[1], entry[3])) last_pkg = pkg else: print('\t{}'.format(entry[3])) return 0 def show_pypi_releases(self): """Show PyPI releases for the last number of `hours` @returns: 0 = success or 1 if failed to retrieve from XML-RPC server """ try: hours = int(self.options.show_pypi_releases) except ValueError: print('You must supply an integer', file=sys.stderr) return 1 try: latest_releases = self.pypi.updated_releases(hours) except XMLRPCFault as err_msg: print(err_msg, file=sys.stderr) print("Couldn't retrieve latest releases.", file=sys.stderr) return 1 for release in latest_releases: print('{} {}'.format(release[0], release[1])) return 0 def show_download_links(self): """Query PyPI for pkg download URI for a packge. @returns: 0 """ # In case they specify version as 'dev' instead of using -T svn, # don't show three svn URI's if self.options.file_type == 'all' and self.version == 'dev': self.options.file_type = 'svn' if self.options.file_type == 'svn': version = 'dev' else: if self.version: version = self.version else: version = self.all_versions[0] if self.options.file_type == 'all': # Search for source, egg, and svn. self.print_download_uri(version, True) self.print_download_uri(version, False) self.print_download_uri('dev', True) else: if self.options.file_type == 'source': source = True else: source = False self.print_download_uri(version, source) return 0 def print_download_uri(self, version, source): """@param version: version number or 'dev' for svn. @type version: string @param source: download source or egg @type source: boolean @returns: None """ if version == 'dev': source = True # Use setuptools monkey-patch to grab url. url = get_download_uri(self.project_name, version, source, self.options.pypi_index) if url: print('{}'.format(url)) def fetch(self): """Download a package. @returns: 0 = success or 1 if failed download """ source = True directory = '.' if self.options.file_type == 'svn': svn_uri = get_download_uri(self.project_name, 'dev', True) if svn_uri: directory = self.project_name + '_svn' return self.fetch_svn(svn_uri, directory) else: print( 'No subversion repository found for {}'.format( self.project_name), file=sys.stderr) return 1 elif self.options.file_type == 'source': source = True elif self.options.file_type == 'egg': source = False uri = get_download_uri(self.project_name, self.version, source) if uri: return self.fetch_uri(directory, uri) else: print('No {} URI found for package: {}'.format( self.options.file_type, self.project_name)) return 1 def fetch_uri(self, directory, uri): """Use ``urllib.urlretrieve`` to download package to file in sandbox dir. @param directory: directory to download to @type directory: string @param uri: uri to download @type uri: string @returns: 0 = success or 1 for failed download """ filename = os.path.basename(urlparse(uri)[2]) if os.path.exists(filename): print('File exists: ' + filename, file=sys.stderr) return 1 try: downloaded_filename, headers = urlretrieve(uri, filename) except IOError as err_msg: print( 'Error downloading package {} from URL {}'.format( filename, uri), file=sys.stderr) print(str(err_msg), file=sys.stderr) return 1 if 'text/html' in headers: dfile = open(downloaded_filename) if re.search('404 Not Found', ''.join(dfile.readlines())): dfile.close() print("'404 Not Found' error", file=sys.stderr) return 1 dfile.close() return 0 def fetch_svn(self, svn_uri, directory): """Fetch subversion repository. @param svn_uri: subversion repository uri to check out @type svn_uri: string @param directory: directory to download to @type directory: string """ if not command_successful(['svn', '--version']): raise YolkException('Do you have subversion installed?') if os.path.exists(directory): raise YolkException( 'Checkout directory exists - {}'.format(directory)) try: os.mkdir(directory) except OSError as err_msg: raise YolkException('' + str(err_msg)) cwd = os.path.realpath(os.curdir) os.chdir(directory) status, _ = run_command(['svn', 'checkout', svn_uri]) os.chdir(cwd) def browse_website(self, browser=None): """Launch web browser at project's homepage. @param browser: name of web browser to use @type browser: string @returns: 0 if homepage found, 1 if no homepage found """ if len(self.all_versions): metadata = self.pypi.release_data(self.project_name, self.all_versions[0]) if 'home_page' in metadata: if browser == 'konqueror': browser = webbrowser.Konqueror() else: browser = webbrowser.get() browser.open(metadata['home_page'], 2) return 0 print('No homepage URL found', file=sys.stderr) return 1 def query_metadata_pypi(self): """Show pkg metadata queried from PyPI. @returns: 0 """ if self.version and self.version in self.all_versions: metadata = self.pypi.release_data(self.project_name, self.version) else: # Give highest version metadata = self.pypi.release_data(self.project_name, self.all_versions[0]) if metadata: if len(self.options.fields) == 1: try: print(metadata[self.options.fields[0]]) except KeyError: pass else: for key in metadata.keys(): if ( not self.options.fields or (self.options.fields and key.lower() in self.options.fields) ): print('{}: {}'.format(key, metadata[key])) return 0 def versions_available(self): """Query PyPI for a particular version or all versions of a package. @returns: 0 if version(s) found or 1 if none found """ if self.all_versions and self.version in self.all_versions: print_pkg_versions(self.project_name, [self.version]) elif not self.version and self.all_versions: print_pkg_versions(self.project_name, self.all_versions) else: if self.version: print( 'No package found for version {}'.format(self.version), file=sys.stderr) else: print( 'No package found for {}'.format(self.project_name), file=sys.stderr) return 1 return 0 def parse_search_spec(self, spec): """Parse search args and return spec dict for PyPI. * Owwww, my eyes!. Re-write this. @param spec: Cheese Shop package search spec e.g. name=Cheetah license=ZPL license=ZPL AND name=Cheetah @type spec: string @returns: tuple with spec and operator """ usage = """You can search PyPI by the following: name version author author_email maintainer maintainer_email home_page license summary description keywords platform download_url e.g. yolk -S name=Cheetah yolk -S name=yolk AND license=PSF """ if not spec: print(usage, file=sys.stderr) return (None, None) try: spec = (' ').join(spec) operator = 'AND' first = second = '' if ' AND ' in spec: (first, second) = spec.split('AND') elif ' OR ' in spec: (first, second) = spec.split('OR') operator = 'OR' else: first = spec (key1, term1) = first.split('=') key1 = key1.strip() if second: (key2, term2) = second.split('=') key2 = key2.strip() spec = {} spec[key1] = term1 if second: spec[key2] = term2 except: print(usage, file=sys.stderr) spec = operator = None return (spec, operator) def pypi_search(self): """Search PyPI by metadata keyword e.g. yolk -S name=yolk AND license=GPL @param spec: Cheese Shop search spec @type spec: list of strings spec examples: ["name=yolk"] ["license=GPL"] ["name=yolk", "AND", "license=GPL"] @returns: 0 on success or 1 if mal-formed search spec """ spec = self.pkg_spec # Add remaining cli arguments to options.pypi_search. search_arg = self.options.pypi_search spec.insert(0, search_arg.strip()) (spec, operator) = self.parse_search_spec(spec) if not spec: return 1 for pkg in self.pypi.search(spec, operator): if pkg['summary']: summary = pkg['summary'].encode('utf-8') else: summary = '' print("""{} ({}): {} """.format(pkg['name'].encode('utf-8'), pkg['version'], summary)) return 0 def show_entry_map(self): """Show entry map for a package. @param dist: package @param type: string @returns: 0 for success or 1 if error """ pprinter = pprint.PrettyPrinter() try: entry_map = pkg_resources.get_entry_map( self.options.show_entry_map) if entry_map: pprinter.pprint(entry_map) except pkg_resources.DistributionNotFound: print( 'Distribution not found: {}'.format( self.options.show_entry_map), file=sys.stderr) return 1 return 0 def show_entry_points(self): """Show entry points for a module. @returns: 0 for success or 1 if error """ found = False for entry_point in pkg_resources.iter_entry_points( self.options.show_entry_points): found = True try: plugin = entry_point.load() print(plugin.__module__) print(' {}'.format(entry_point)) if plugin.__doc__: print(plugin.__doc__) print() except ImportError: pass if not found: print( 'No entry points found for {}'.format( self.options.show_entry_points), file=sys.stderr) return 1 return 0 def yolk_version(self): """Show yolk's version.""" print('yolk {}'.format(VERSION)) def parse_pkg_ver(self, want_installed): """Return tuple with project_name and version from CLI args If the user gave the wrong case for the project name, this corrects it. @param want_installed: whether package we want is installed or not @type want_installed: boolean @returns: tuple(project_name, version, all_versions) """ all_versions = [] arg_str = self.pkg_spec if '==' not in arg_str: # No version specified. project_name = arg_str version = None else: (project_name, version) = arg_str.split('==') project_name = project_name.strip() version = version.strip() # Find proper case for package name. if want_installed: project_name = yolklib.case_sensitive_name(project_name) else: (project_name, all_versions) = self.pypi.query_versions_pypi( project_name) if not len(all_versions): msg = "I'm afraid we have no '{}' at ".format(project_name) msg += 'The Cheese Shop. A little Red Leicester, perhaps?' print(msg, file=sys.stderr) sys.exit(2) return (project_name, version, all_versions)
class Yolk(object): """ Main class for yolk """ def __init__(self): #PyPI project name with proper case self.project_name = "" #PyPI project version self.version = "" #List of all versions not hidden on PyPI self.all_versions = [] self.pkg_spec = [] self.options = None self.logger = logging.getLogger("yolk") #Squelch output from setuptools #Add future offenders to this list. shut_up = ['distutils.log'] sys.stdout = StdOut(sys.stdout, shut_up) sys.stderr = StdOut(sys.stderr, shut_up) self.pypi = None def get_plugin(self, method): """ Return plugin object if CLI option is activated and method exists @param method: name of plugin's method we're calling @type method: string @returns: list of plugins with `method` """ all_plugins = [] for entry_point in pkg_resources.iter_entry_points('yolk.plugins'): plugin_obj = entry_point.load() plugin = plugin_obj() plugin.configure(self.options, None) if plugin.enabled: if not hasattr(plugin, method): self.logger.warn("Error: plugin has no method: %s" % method) plugin = None else: all_plugins.append(plugin) return all_plugins def set_log_level(self): """ Set log level according to command-line options @returns: logger object """ if self.options.debug: self.logger.setLevel(logging.DEBUG) elif self.options.quiet: self.logger.setLevel(logging.ERROR) else: self.logger.setLevel(logging.INFO) self.logger.addHandler(logging.StreamHandler()) return self.logger def run(self): """ Perform actions based on CLI options @returns: status code """ opt_parser = setup_opt_parser() (self.options, remaining_args) = opt_parser.parse_args() logger = self.set_log_level() pkg_spec = validate_pypi_opts(opt_parser) if not pkg_spec: pkg_spec = remaining_args self.pkg_spec = pkg_spec if not self.options.pypi_search and (len(sys.argv) == 1 or\ len(remaining_args) > 2): opt_parser.print_help() return 2 #Options that depend on querying installed packages, not PyPI. #We find the proper case for package names if they are installed, #otherwise PyPI returns the correct case. if self.options.show_deps or self.options.show_all or \ self.options.show_active or self.options.show_non_active or \ (self.options.show_updates and pkg_spec): want_installed = True else: want_installed = False #show_updates may or may not have a pkg_spec if not want_installed or self.options.show_updates: self.pypi = CheeseShop(self.options.debug) #XXX: We should return 2 here if we couldn't create xmlrpc server if pkg_spec: (self.project_name, self.version, self.all_versions) = \ self.parse_pkg_ver(want_installed) if want_installed and not self.project_name: logger.error("%s is not installed." % pkg_spec[0]) return 1 #I could prefix all these with 'cmd_' and the methods also #and then iterate over the `options` dictionary keys... commands = [ 'show_deps', 'query_metadata_pypi', 'fetch', 'versions_available', 'show_updates', 'browse_website', 'show_download_links', 'pypi_search', 'show_pypi_changelog', 'show_pypi_releases', 'yolk_version', 'show_all', 'show_active', 'show_non_active', 'show_entry_map', 'show_entry_points' ] #Run first command it finds, and only the first command, then return #XXX: Check if more than one command was set in options and give error? for action in commands: if getattr(self.options, action): return getattr(self, action)() opt_parser.print_help() def show_active(self): """ Show installed active packages """ return self.show_distributions("active") def show_non_active(self): """ Show installed non-active packages """ return self.show_distributions("nonactive") def show_all(self): """ Show all installed packages """ return self.show_distributions("all") def show_updates(self): """ Check installed packages for available updates on PyPI @param project_name: optional package name to check; checks every installed pacakge if none specified @type project_name: string @returns: None """ dists = Distributions() if self.project_name: #Check for a single package pkg_list = [self.project_name] else: #Check for every installed package pkg_list = get_pkglist() found = None for pkg in pkg_list: for (dist, active) in dists.get_distributions( "all", pkg, dists.get_highest_installed(pkg)): (project_name, versions) = \ self.pypi.query_versions_pypi(dist.project_name) if versions: #PyPI returns them in chronological order, #but who knows if its guaranteed in the API? #Make sure we grab the highest version: newest = get_highest_version(versions) if newest != dist.version: #We may have newer than what PyPI knows about if pkg_resources.parse_version(dist.version) < \ pkg_resources.parse_version(newest): found = True print " %s %s (%s)" % (project_name, dist.version, newest) if not found and self.project_name: self.logger.info("You have the latest version installed.") elif not found: self.logger.info("No newer packages found at The Cheese Shop") return 0 def show_distributions(self, show): """ Show list of installed activated OR non-activated packages @param show: type of pkgs to show (all, active or nonactive) @type show: string @returns: None or 2 if error """ show_metadata = self.options.metadata #Search for any plugins with active CLI options with add_column() method plugins = self.get_plugin("add_column") #Some locations show false positive for 'development' packages: ignores = ["/UNIONFS", "/KNOPPIX.IMG"] #Check if we're in a workingenv #See http://cheeseshop.python.org/pypi/workingenv.py workingenv = os.environ.get('WORKING_ENV') if workingenv: ignores.append(workingenv) dists = Distributions() results = None for (dist, active) in dists.get_distributions(show, self.project_name, self.version): metadata = get_metadata(dist) for prefix in ignores: if dist.location.startswith(prefix): dist.location = dist.location.replace(prefix, "") #Case-insensitve search because of Windows if dist.location.lower().startswith(get_python_lib().lower()): develop = "" else: develop = dist.location if metadata: add_column_text = "" for my_plugin in plugins: #See if package is 'owned' by a package manager such as #portage, apt, rpm etc. #add_column_text += my_plugin.add_column(filename) + " " add_column_text += my_plugin.add_column(dist) + " " self.print_metadata(metadata, develop, active, add_column_text) else: print str(dist) + " has no metadata" results = True if not results and self.project_name: if self.version: pkg_spec = "%s==%s" % (self.project_name, self.version) else: pkg_spec = "%s" % self.project_name if show == "all": self.logger.error("There are no versions of %s installed." \ % pkg_spec) else: self.logger.error("There are no %s versions of %s installed." \ % \ (show, pkg_spec)) return 2 elif show == "all" and results and self.options.fields: print "Versions with '*' are non-active." print "Versions with '!' are deployed in development mode." def print_metadata(self, metadata, develop, active, installed_by): """ Print out formatted metadata @param metadata: package's metadata @type metadata: pkg_resources Distribution obj @param develop: path to pkg if its deployed in development mode @type develop: string @param active: show if package is activated or not @type active: boolean @param installed_by: Shows if pkg was installed by a package manager other than setuptools @type installed_by: string @returns: None """ show_metadata = self.options.metadata if self.options.fields: fields = self.options.fields.split(',') fields = map(str.strip, fields) else: fields = [] version = metadata['Version'] #When showing all packages, note which are not active: if active: if fields: active_status = "" else: active_status = "active" else: if fields: active_status = "*" else: active_status = "non-active" if develop: if fields: development_status = "! (%s)" % develop else: development_status = "development (%s)" % develop else: development_status = installed_by status = "%s %s" % (active_status, development_status) if fields: print '%s (%s)%s %s' % (metadata['Name'], version, active_status, development_status) else: # Need intelligent justification print metadata['Name'].ljust(15) + " - " + version.ljust(12) + \ " - " + status if fields: #Only show specific fields, using case-insensitive search fields = map(str.lower, fields) for field in metadata.keys(): if field.lower() in fields: print ' %s: %s' % (field, metadata[field]) print elif show_metadata: #Print all available metadata fields for field in metadata.keys(): if field != 'Name' and field != 'Summary': print ' %s: %s' % (field, metadata[field]) def show_deps(self): """ Show dependencies for package(s) @returns: 0 - sucess 1 - No dependency info supplied """ pkgs = pkg_resources.Environment() for pkg in pkgs[self.project_name]: if not self.version: print pkg.project_name, pkg.version i = len(pkg._dep_map.values()[0]) if i: while i: if not self.version or self.version and \ pkg.version == self.version: if self.version and i == len(pkg._dep_map.values()[0]): print pkg.project_name, pkg.version print " " + str(pkg._dep_map.values()[0][i - 1]) i -= 1 else: self.logger.info(\ "No dependency information was supplied with the package.") return 1 return 0 def show_pypi_changelog(self): """ Show detailed PyPI ChangeLog for the last `hours` @returns: 0 = sucess or 1 if failed to retrieve from XML-RPC server """ hours = self.options.show_pypi_changelog if not hours.isdigit(): self.logger.error("Error: You must supply an integer.") return 1 try: changelog = self.pypi.changelog(int(hours)) except XMLRPCFault, err_msg: self.logger.error(err_msg) self.logger.error("ERROR: Couldn't retrieve changelog.") return 1 last_pkg = '' for entry in changelog: pkg = entry[0] if pkg != last_pkg: print "%s %s\n\t%s" % (entry[0], entry[1], entry[3]) last_pkg = pkg else: print "\t%s" % entry[3] return 0
failed_msgs.append("%s %s" % (pkg_name, versions)) print "Testing %s... failed" % elem.text print "%s tests failed." % failed for msg in failed_msgs: print "\t%s" % msg print "%s tests skipped." % skipped for msg in skipped_msgs: print "\t%s" % msg def test_cli(pypi_xml): """Test the command-line tool""" print "Testing CLI" for event, elem in iterparse(pypi_xml): if elem.tag == "title": if not elem.text.startswith('Cheese Shop recent updates'): print "Testing %s..." % elem.text pkg_name, ver = get_pkg_ver(elem.text) if " " in pkg_name: print "Space in package name, skipping: %s" % pkg_name else: os.system("yolk -V '%s'" % pkg_name) os.system("yolk -D %s==%s" % (pkg_name, ver)) elem.clear() test_cli(urllib.urlopen(PYPI_URL)) PyPI = CheeseShop() test_api(urllib.urlopen(PYPI_URL))
class Yolk(object): """ Main class for yolk """ def __init__(self): #PyPI project name with proper case self.project_name = "" #PyPI project version self.version = "" #List of all versions not hidden on PyPI self.all_versions = [] self.pkg_spec = [] self.options = None self.logger = logging.getLogger("yolk") #Squelch output from setuptools #Add future offenders to this list. shut_up = ['distutils.log'] sys.stdout = StdOut(sys.stdout, shut_up) sys.stderr = StdOut(sys.stderr, shut_up) self.pypi = None def get_plugin(self, method): """ Return plugin object if CLI option is activated and method exists @param method: name of plugin's method we're calling @type method: string @returns: list of plugins with `method` """ all_plugins = [] for entry_point in pkg_resources.iter_entry_points('yolk.plugins'): plugin_obj = entry_point.load() plugin = plugin_obj() plugin.configure(self.options, None) if plugin.enabled: if not hasattr(plugin, method): self.logger.warn("Error: plugin has no method: %s" % method) plugin = None else: all_plugins.append(plugin) return all_plugins def set_log_level(self): """ Set log level according to command-line options @returns: logger object """ if self.options.debug: self.logger.setLevel(logging.DEBUG) elif self.options.quiet: self.logger.setLevel(logging.ERROR) else: self.logger.setLevel(logging.INFO) self.logger.addHandler(logging.StreamHandler()) return self.logger def run(self): """ Perform actions based on CLI options @returns: status code """ opt_parser = setup_opt_parser() (self.options, remaining_args) = opt_parser.parse_args() logger = self.set_log_level() pkg_spec = validate_pypi_opts(opt_parser) if not pkg_spec: pkg_spec = remaining_args self.pkg_spec = pkg_spec if not self.options.pypi_search and (len(sys.argv) == 1 or\ len(remaining_args) > 2): opt_parser.print_help() return 2 #Options that depend on querying installed packages, not PyPI. #We find the proper case for package names if they are installed, #otherwise PyPI returns the correct case. if self.options.show_deps or self.options.show_all or \ self.options.show_active or self.options.show_non_active or \ (self.options.show_updates and pkg_spec): want_installed = True else: want_installed = False #show_updates may or may not have a pkg_spec if not want_installed or self.options.show_updates: self.pypi = CheeseShop(self.options.debug) #XXX: We should return 2 here if we couldn't create xmlrpc server if pkg_spec: (self.project_name, self.version, self.all_versions) = \ self.parse_pkg_ver(want_installed) if want_installed and not self.project_name: logger.error("%s is not installed." % pkg_spec[0]) return 1 #I could prefix all these with 'cmd_' and the methods also #and then iterate over the `options` dictionary keys... commands = ['show_deps', 'query_metadata_pypi', 'fetch', 'versions_available', 'show_updates', 'browse_website', 'show_download_links', 'pypi_search', 'show_pypi_changelog', 'show_pypi_releases', 'yolk_version', 'show_all', 'show_active', 'show_non_active', 'show_entry_map', 'show_entry_points'] #Run first command it finds, and only the first command, then return #XXX: Check if more than one command was set in options and give error? for action in commands: if getattr(self.options, action): return getattr(self, action)() opt_parser.print_help() def show_active(self): """ Show installed active packages """ return self.show_distributions("active") def show_non_active(self): """ Show installed non-active packages """ return self.show_distributions("nonactive") def show_all(self): """ Show all installed packages """ return self.show_distributions("all") def show_updates(self): """ Check installed packages for available updates on PyPI @param project_name: optional package name to check; checks every installed pacakge if none specified @type project_name: string @returns: None """ dists = Distributions() if self.project_name: #Check for a single package pkg_list = [self.project_name] else: #Check for every installed package pkg_list = get_pkglist() found = None for pkg in pkg_list: for (dist, active) in dists.get_distributions("all", pkg, dists.get_highest_installed(pkg)): (project_name, versions) = \ self.pypi.query_versions_pypi(dist.project_name) if versions: #PyPI returns them in chronological order, #but who knows if its guaranteed in the API? #Make sure we grab the highest version: newest = get_highest_version(versions) if newest != dist.version: #We may have newer than what PyPI knows about if pkg_resources.parse_version(dist.version) < \ pkg_resources.parse_version(newest): found = True print " %s %s (%s)" % (project_name, dist.version, newest) if not found and self.project_name: self.logger.info("You have the latest version installed.") elif not found: self.logger.info("No newer packages found at The Cheese Shop") return 0 def show_distributions(self, show): """ Show list of installed activated OR non-activated packages @param show: type of pkgs to show (all, active or nonactive) @type show: string @returns: None or 2 if error """ show_metadata = self.options.metadata #Search for any plugins with active CLI options with add_column() method plugins = self.get_plugin("add_column") #Some locations show false positive for 'development' packages: ignores = ["/UNIONFS", "/KNOPPIX.IMG"] #Check if we're in a workingenv #See http://cheeseshop.python.org/pypi/workingenv.py workingenv = os.environ.get('WORKING_ENV') if workingenv: ignores.append(workingenv) dists = Distributions() results = None for (dist, active) in dists.get_distributions(show, self.project_name, self.version): metadata = get_metadata(dist) for prefix in ignores: if dist.location.startswith(prefix): dist.location = dist.location.replace(prefix, "") #Case-insensitve search because of Windows if dist.location.lower().startswith(get_python_lib().lower()): develop = "" else: develop = dist.location if metadata: add_column_text = "" for my_plugin in plugins: #See if package is 'owned' by a package manager such as #portage, apt, rpm etc. #add_column_text += my_plugin.add_column(filename) + " " add_column_text += my_plugin.add_column(dist) + " " self.print_metadata(metadata, develop, active, add_column_text) else: print str(dist) + " has no metadata" results = True if not results and self.project_name: if self.version: pkg_spec = "%s==%s" % (self.project_name, self.version) else: pkg_spec = "%s" % self.project_name if show == "all": self.logger.error("There are no versions of %s installed." \ % pkg_spec) else: self.logger.error("There are no %s versions of %s installed." \ % \ (show, pkg_spec)) return 2 elif show == "all" and results and self.options.fields: print "Versions with '*' are non-active." print "Versions with '!' are deployed in development mode." def print_metadata(self, metadata, develop, active, installed_by): """ Print out formatted metadata @param metadata: package's metadata @type metadata: pkg_resources Distribution obj @param develop: path to pkg if its deployed in development mode @type develop: string @param active: show if package is activated or not @type active: boolean @param installed_by: Shows if pkg was installed by a package manager other than setuptools @type installed_by: string @returns: None """ show_metadata = self.options.metadata if self.options.fields: fields = self.options.fields.split(',') fields = map(str.strip, fields) else: fields = [] version = metadata['Version'] #When showing all packages, note which are not active: if active: if fields: active_status = "" else: active_status = "active" else: if fields: active_status = "*" else: active_status = "non-active" if develop: if fields: development_status = "! (%s)" % develop else: development_status = "development (%s)" % develop else: development_status = installed_by status = "%s %s" % (active_status, development_status) if fields: print '%s (%s)%s %s' % (metadata['Name'], version, active_status, development_status) else: # Need intelligent justification print metadata['Name'].ljust(15) + " - " + version.ljust(12) + \ " - " + status if fields: #Only show specific fields, using case-insensitive search fields = map(str.lower, fields) for field in metadata.keys(): if field.lower() in fields: print ' %s: %s' % (field, metadata[field]) print elif show_metadata: #Print all available metadata fields for field in metadata.keys(): if field != 'Name' and field != 'Summary': print ' %s: %s' % (field, metadata[field]) def show_deps(self): """ Show dependencies for package(s) @returns: 0 - sucess 1 - No dependency info supplied """ pkgs = pkg_resources.Environment() for pkg in pkgs[self.project_name]: if not self.version: print pkg.project_name, pkg.version i = len(pkg._dep_map.values()[0]) if i: while i: if not self.version or self.version and \ pkg.version == self.version: if self.version and i == len(pkg._dep_map.values()[0]): print pkg.project_name, pkg.version print " " + str(pkg._dep_map.values()[0][i - 1]) i -= 1 else: self.logger.info(\ "No dependency information was supplied with the package.") return 1 return 0 def show_pypi_changelog(self): """ Show detailed PyPI ChangeLog for the last `hours` @returns: 0 = sucess or 1 if failed to retrieve from XML-RPC server """ hours = self.options.show_pypi_changelog if not hours.isdigit(): self.logger.error("Error: You must supply an integer.") return 1 try: changelog = self.pypi.changelog(int(hours)) except XMLRPCFault, err_msg: self.logger.error(err_msg) self.logger.error("ERROR: Couldn't retrieve changelog.") return 1 last_pkg = '' for entry in changelog: pkg = entry[0] if pkg != last_pkg: print "%s %s\n\t%s" % (entry[0], entry[1], entry[3]) last_pkg = pkg else: print "\t%s" % entry[3] return 0
ARG_PARSER.add_argument( "-e", "--extra", help="extra extends file") ARG_PARSER.add_argument( "--no-venv", action="store_true", help="no virtualenv") ARG_PARSER.add_argument( "--no-buildout", action="store_true", help="no pip install zc.buildout") ARG_PARSER.add_argument( "--unified-only", action="store_true", help="get unified cache & quit") BUILDOUT_CFG = """\ [buildout] extends = %s """ CFG_PARSER = configparser.SafeConfigParser() PYPI = CheeseShop() EXTENDS = "https://raw.github.com/plock/pins/master/plone-4-3" UNIFIEDINSTALLER_DIR = "Plone-4.3.3-UnifiedInstaller" UNIFIEDINSTALLER_URL = "https://launchpad.net/plone/4.3/4.3.3/+download/" UNIFIEDINSTALLER_URL += "Plone-4.3.3-UnifiedInstaller.tgz" SEARCH_OPER = 'AND' SEARCH_SPEC = {'description': 'plone', 'keyword': 'plone', 'summary': 'plone'}
def run(self): """Perform actions based on CLI options. @returns: status code """ parser = setup_parser() try: import argcomplete argcomplete.autocomplete(parser) except ImportError: pass self.options = parser.parse_args() pkg_spec = validate_pypi_opts(parser) if not pkg_spec: pkg_spec = self.options.pkg_spec self.pkg_spec = pkg_spec if self.options.fields: self.options.fields = [s.strip().lower() for s in self.options.fields.split(',')] else: self.options.fields = [] if ( not self.options.pypi_search and len(sys.argv) == 1 ): parser.print_help() return 2 # Options that depend on querying installed packages, not PyPI. # We find the proper case for package names if they are installed, # otherwise PyPI returns the correct case. if ( self.options.show_deps or self.options.show_all or self.options.show_active or self.options.show_non_active or (self.options.show_updates and pkg_spec) or self.options.upgrade ): want_installed = True else: want_installed = False # show_updates may or may not have a pkg_spec if ( not want_installed or self.options.show_updates or self.options.upgrade ): self.pypi = CheeseShop(self.options.debug) # XXX: We should return 2 here if we couldn't create xmlrpc server if pkg_spec: (self.project_name, self.version, self.all_versions) = self.parse_pkg_ver(want_installed) if want_installed and not self.project_name: print('{} is not installed'.format(pkg_spec), file=sys.stderr) return 1 # I could prefix all these with 'cmd_' and the methods also # and then iterate over the `options` dictionary keys... commands = ['show_deps', 'query_metadata_pypi', 'fetch', 'versions_available', 'show_updates', 'upgrade', 'browse_website', 'show_download_links', 'pypi_search', 'show_pypi_changelog', 'show_pypi_releases', 'yolk_version', 'show_all', 'show_active', 'show_non_active', 'show_entry_map', 'show_entry_points'] # Run first command it finds, and only the first command, then return # XXX: Check if more than one command was set in options and give # error? for action in commands: if getattr(self.options, action): return getattr(self, action)() parser.print_help()
argparser.add_argument( "-u", "--use", action="store_true", help="use existing buildout.cfg") argparser.add_argument( "-w", "--write", action="store_true", dest="write_only", help="write buildout.cfg and exit") argparser.add_argument( "--no-cache", action="store_true", dest="no_unified", help="do not download unified installer cache") argparser.add_argument( "--no-buildout", action="store_true", help="do not install buildout") argparser.add_argument( "--no-virtualenv", action="store_true", help="do not create virtualenv") argparser.add_argument( "--cache", action="store_true", dest="unified_only", help="download unified installer cache and exit") cfgparser = configparser.SafeConfigParser() pypi = CheeseShop() query = { 'description': 'plone', 'keyword': 'plone', 'summary': 'plone' }
def check_latest(): """ Problems: * we use several third-party packages that are not included in a normal plone release. We want to keep those up to date * we keep all packages pinned to avoid getting unstable releases and we need to know when newer releases are available The tool should do the following: * watch the egg repos we are using (pypi, eea eggrepo) and report when new versions of packages are available, especially for the third-party packages * parse the buildout files (including on-the-web cfgs) and report of conflicts between versions * this should be implemented as a buildout tool * this tool should be run in automatic by Jenkins and report when new versions are available """ # we use the .installed.cfg file to try to find the longest line # of __buildout_signature__, which contains the eggs that we need. # this is (maybe) highly specific to EEA. v = {} with open('.installed.cfg', 'r') as f: lines = f.readlines() longest = "" for line in lines: if line.startswith("__buildout_signature__") \ and len(line) > len(longest): longest = line eggs = longest.split('=')[1].strip().split(' ') for egg in eggs: spec = egg.split('-') name = spec[0] version = spec[1] v[name.strip()] = version.strip() skipped = [] if os.path.exists('.skipped_packages'): with open(".skipped_packages") as f: skipped = [x.strip() for x in f.readlines() if x.strip()] picked_versions = v # repos = [EEAEggRepo(), CheeseShop(), ] #order is important repos = [ CheeseShop(), ] # order is important flag = 0 report = [] for name, v in picked_versions.items(): if name in skipped: continue print "Checking new version for", name for pypi in repos: new_name, versions = pypi.query_versions_pypi(name) if versions: break if versions: latest = versions[0] else: print "Could not find any version for this package" continue if latest != v: print "Package %s - %s" % (name, v), " has a new version: ", latest report.append((name, v, latest)) flag = 1 report.sort(lambda x, y: cmp(x[0], y[0])) print print "New versions report" print "===================" for l in report: name, old, new = l space1 = (40 - len(name)) * ' ' space2 = (20 - len(old)) * ' ' print name + space1 + old + space2 + new sys.exit(flag)
def get_lastest_version_number(package_name): pkg, all_versions = CheeseShop().query_versions_pypi(package_name) if len(all_versions): return all_versions[0] return None
class Yolk(object): """Main class for yolk.""" def __init__(self): # PyPI project name with proper case self.project_name = '' # PyPI project version self.version = '' # List of all versions not hidden on PyPI self.all_versions = [] self.pkg_spec = None self.options = None # Squelch output from setuptools # Add future offenders to this list. shut_up = ['distutils.log'] sys.stdout = StdOut(sys.stdout, shut_up) sys.stderr = StdOut(sys.stderr, shut_up) self.pypi = None def get_plugin(self, method): """Return plugin object if CLI option is activated and method exists. @param method: name of plugin's method we're calling @type method: string @returns: list of plugins with `method` """ all_plugins = [] for entry_point in pkg_resources.iter_entry_points('yolk.plugins'): plugin_obj = entry_point.load() plugin = plugin_obj() plugin.configure(self.options, None) if plugin.enabled: if not hasattr(plugin, method): plugin = None else: all_plugins.append(plugin) return all_plugins def run(self): """Perform actions based on CLI options. @returns: status code """ parser = setup_parser() try: import argcomplete argcomplete.autocomplete(parser) except ImportError: pass self.options = parser.parse_args() pkg_spec = validate_pypi_opts(parser) if not pkg_spec: pkg_spec = self.options.pkg_spec self.pkg_spec = pkg_spec if self.options.fields: self.options.fields = [ s.strip().lower() for s in self.options.fields.split(',') ] else: self.options.fields = [] if (not self.options.pypi_search and len(sys.argv) == 1): parser.print_help() return 2 # Options that depend on querying installed packages, not PyPI. # We find the proper case for package names if they are installed, # otherwise PyPI returns the correct case. if (self.options.show_deps or self.options.show_all or self.options.show_active or self.options.show_non_active or (self.options.show_updates and pkg_spec) or self.options.upgrade): want_installed = True else: want_installed = False # show_updates may or may not have a pkg_spec if (not want_installed or self.options.show_updates or self.options.upgrade): self.pypi = CheeseShop(self.options.debug) # XXX: We should return 2 here if we couldn't create xmlrpc server if pkg_spec: (self.project_name, self.version, self.all_versions) = self.parse_pkg_ver(want_installed) if want_installed and not self.project_name: print(u'{} is not installed'.format(pkg_spec), file=sys.stderr) return 1 # I could prefix all these with 'cmd_' and the methods also # and then iterate over the `options` dictionary keys... commands = [ 'show_deps', 'query_metadata_pypi', 'fetch', 'versions_available', 'show_updates', 'upgrade', 'browse_website', 'show_download_links', 'pypi_search', 'show_pypi_changelog', 'show_pypi_releases', 'yolk_version', 'show_all', 'show_active', 'show_non_active', 'show_entry_map', 'show_entry_points' ] # Run first command it finds, and only the first command, then return # XXX: Check if more than one command was set in options and give # error? for action in commands: if getattr(self.options, action): return getattr(self, action)() parser.print_help() def show_active(self): """Show installed active packages.""" return self.show_distributions('active') def show_non_active(self): """Show installed non-active packages.""" return self.show_distributions('nonactive') def show_all(self): """Show all installed packages.""" return self.show_distributions('all') def show_updates(self): """Check installed packages for available updates on PyPI. @param project_name: optional package name to check; checks every installed package if none specified @type project_name: string @returns: None """ if self.project_name: pkg_list = [self.project_name] else: pkg_list = get_pkglist() for (project_name, version, newest) in _updates(pkg_list, self.pypi, user_installs_only=self.options.user): print(u'{} {} ({})'.format(project_name, version, newest)) return 0 def upgrade(self): """Check installed packages for available updates on PyPI and upgrade. @param project_name: optional package name to check; checks every installed package if none specified @type project_name: string @returns: None """ if self.project_name: pkg_list = [self.project_name] else: pkg_list = get_pkglist() names = [ values[0] for values in _updates( pkg_list, self.pypi, user_installs_only=self.options.user) ] if names: subprocess.call( [sys.executable, '-m', 'pip', 'install', '--upgrade'] + (['--user'] if self.options.user else []) + names) return 0 def show_distributions(self, show): """Show list of installed activated OR non-activated packages. @param show: type of pkgs to show (all, active or nonactive) @type show: string @returns: None or 2 if error """ # Search for any plugins with active CLI options with add_column() # method. plugins = self.get_plugin('add_column') # Some locations show false positive for 'development' packages: ignores = ['/UNIONFS', '/KNOPPIX.IMG'] # See http://cheeseshop.python.org/pypi/workingenv.py for details. workingenv = os.environ.get('WORKING_ENV') if workingenv: ignores.append(workingenv) results = None for (dist, active) in yolklib.get_distributions(show, self.project_name, self.version): metadata = get_metadata(dist) for prefix in ignores: if dist.location.startswith(prefix): dist.location = dist.location.replace(prefix, '') # Case-insensitive search because of Windows. if dist.location.lower().startswith(get_python_lib().lower()): develop = '' else: develop = dist.location if metadata: add_column_text = '' for my_plugin in plugins: # See if package is 'owned' by a package manager such as # portage, apt, rpm etc. add_column_text += my_plugin.add_column(dist) + ' ' self.print_metadata(metadata, develop, active, add_column_text) else: print(str(dist) + ' has no metadata') results = True if not results and self.project_name: if self.version: pkg_spec = '{}=={}'.format(self.project_name, self.version) else: pkg_spec = self.project_name if show == 'all': print( u'There are no versions of {} installed'.format(pkg_spec), file=sys.stderr) else: print(u'There are no {} versions of {} installed'.format( show, pkg_spec), file=sys.stderr) return 2 elif show == 'all' and results and self.options.fields: print("Versions with '*' are non-active.") print("Versions with '!' are deployed in development mode.") def print_metadata(self, metadata, develop, active, installed_by): """Print out formatted metadata. @param metadata: package's metadata @type metadata: pkg_resources Distribution obj @param develop: path to pkg if its deployed in development mode @type develop: string @param active: show if package is activated or not @type active: boolean @param installed_by: Shows if pkg was installed by a package manager other than setuptools @type installed_by: string @returns: None """ show_metadata = self.options.metadata version = metadata['Version'] # When showing all packages, note which are not active: if active: if self.options.fields: active_status = '' else: active_status = 'active' else: if self.options.fields: active_status = '*' else: active_status = 'non-active' if develop: if self.options.fields: development_status = '! ({})'.format(develop) else: development_status = 'development ({})'.format(develop) else: development_status = installed_by status = '{} {}'.format(active_status, development_status) if self.options.fields: print('{} ({}){} {}'.format(metadata['Name'], version, active_status, development_status)) else: # Need intelligent justification. print(metadata['Name'].ljust(15) + ' - ' + version.ljust(12) + ' - ' + status) if self.options.fields: for field in metadata.keys(): if field.lower() in self.options.fields: print(u' {}: {}'.format(field, metadata[field])) print() elif show_metadata: for field in metadata.keys(): if field != 'Name' and field != 'Summary': print(u' {}: {}'.format(field, metadata[field])) def show_deps(self): """Show dependencies for package(s) @returns: 0 - success 1 - No dependency info supplied """ pkgs = pkg_resources.Environment() for pkg in pkgs[self.project_name]: if not self.version: print(pkg.project_name, pkg.version) i = len(list(pkg._dep_map.values())[0]) if i: while i: if (not self.version or self.version and pkg.version == self.version): if self.version and i == len( list(pkg._dep_map.values())[0]): print(pkg.project_name, pkg.version) print(u' ' + str(list(pkg._dep_map.values())[0][i - 1])) i -= 1 else: return 1 return 0 def show_pypi_changelog(self): """Show detailed PyPI ChangeLog for the last `hours` @returns: 0 = success or 1 if failed to retrieve from XML-RPC server """ hours = self.options.show_pypi_changelog if not hours.isdigit(): print('You must supply an integer', file=sys.stderr) return 1 try: changelog = self.pypi.changelog(int(hours)) except XMLRPCFault as err_msg: print(err_msg, file=sys.stderr) print("Couldn't retrieve changelog", file=sys.stderr) return 1 last_pkg = '' for entry in changelog: pkg = entry[0] if pkg != last_pkg: print(u'{} {}\n\t{}'.format(entry[0], entry[1], entry[3])) last_pkg = pkg else: print(u'\t{}'.format(entry[3])) return 0 def show_pypi_releases(self): """Show PyPI releases for the last number of `hours` @returns: 0 = success or 1 if failed to retrieve from XML-RPC server """ try: hours = int(self.options.show_pypi_releases) except ValueError: print('You must supply an integer', file=sys.stderr) return 1 try: latest_releases = self.pypi.updated_releases(hours) except XMLRPCFault as err_msg: print(err_msg, file=sys.stderr) print("Couldn't retrieve latest releases.", file=sys.stderr) return 1 for release in latest_releases: print(u'{} {}'.format(release[0], release[1])) return 0 def show_download_links(self): """Query PyPI for pkg download URI for a packge. @returns: 0 """ # In case they specify version as 'dev' instead of using -T svn, # don't show three svn URI's if self.options.file_type == 'all' and self.version == 'dev': self.options.file_type = 'svn' if self.options.file_type == 'svn': version = 'dev' else: if self.version: version = self.version else: version = self.all_versions[0] if self.options.file_type == 'all': # Search for source, egg, and svn. self.print_download_uri(version, True) self.print_download_uri(version, False) self.print_download_uri('dev', True) else: if self.options.file_type == 'source': source = True else: source = False self.print_download_uri(version, source) return 0 def print_download_uri(self, version, source): """@param version: version number or 'dev' for svn. @type version: string @param source: download source or egg @type source: boolean @returns: None """ if version == 'dev': source = True # Use setuptools monkey-patch to grab url. url = get_download_uri(self.project_name, version, source, self.options.pypi_index) if url: print(u'{}'.format(url)) def fetch(self): """Download a package. @returns: 0 = success or 1 if failed download """ source = True directory = '.' if self.options.file_type == 'svn': svn_uri = get_download_uri(self.project_name, 'dev', True) if svn_uri: directory = self.project_name + '_svn' return self.fetch_svn(svn_uri, directory) else: print('No subversion repository found for {}'.format( self.project_name), file=sys.stderr) return 1 elif self.options.file_type == 'source': source = True elif self.options.file_type == 'egg': source = False uri = get_download_uri(self.project_name, self.version, source) if uri: return self.fetch_uri(directory, uri) else: print(u'No {} URI found for package: {}'.format( self.options.file_type, self.project_name)) return 1 def fetch_uri(self, directory, uri): """Use ``urllib.urlretrieve`` to download package to file in sandbox dir. @param directory: directory to download to @type directory: string @param uri: uri to download @type uri: string @returns: 0 = success or 1 for failed download """ filename = os.path.basename(urlparse(uri)[2]) if os.path.exists(filename): print(u'File exists: ' + filename, file=sys.stderr) return 1 try: downloaded_filename, headers = urlretrieve(uri, filename) except IOError as err_msg: print('Error downloading package {} from URL {}'.format( filename, uri), file=sys.stderr) print(str(err_msg), file=sys.stderr) return 1 if 'text/html' in headers: dfile = open(downloaded_filename) if re.search('404 Not Found', ''.join(dfile.readlines())): dfile.close() print("'404 Not Found' error", file=sys.stderr) return 1 dfile.close() return 0 def fetch_svn(self, svn_uri, directory): """Fetch subversion repository. @param svn_uri: subversion repository uri to check out @type svn_uri: string @param directory: directory to download to @type directory: string """ if not command_successful(['svn', '--version']): raise YolkException('Do you have subversion installed?') if os.path.exists(directory): raise YolkException( 'Checkout directory exists - {}'.format(directory)) try: os.mkdir(directory) except OSError as err_msg: raise YolkException('' + str(err_msg)) cwd = os.path.realpath(os.curdir) os.chdir(directory) status, _ = run_command(['svn', 'checkout', svn_uri]) os.chdir(cwd) def browse_website(self, browser=None): """Launch web browser at project's homepage. @param browser: name of web browser to use @type browser: string @returns: 0 if homepage found, 1 if no homepage found """ if len(self.all_versions): metadata = self.pypi.release_data(self.project_name, self.all_versions[0]) if 'home_page' in metadata: if browser == 'konqueror': browser = webbrowser.Konqueror() else: browser = webbrowser.get() browser.open(metadata['home_page'], 2) return 0 print('No homepage URL found', file=sys.stderr) return 1 def query_metadata_pypi(self): """Show pkg metadata queried from PyPI. @returns: 0 """ if self.version and self.version in self.all_versions: metadata = self.pypi.release_data(self.project_name, self.version) else: # Give highest version metadata = self.pypi.release_data(self.project_name, self.all_versions[0]) if metadata: if len(self.options.fields) == 1: try: print(metadata[self.options.fields[0]]) except KeyError: pass else: for key in metadata.keys(): if (not self.options.fields or (self.options.fields and key.lower() in self.options.fields)): print(u'{}: {}'.format(key, metadata[key])) return 0 def versions_available(self): """Query PyPI for a particular version or all versions of a package. @returns: 0 if version(s) found or 1 if none found """ if self.all_versions and self.version in self.all_versions: print_pkg_versions(self.project_name, [self.version]) elif not self.version and self.all_versions: print_pkg_versions(self.project_name, self.all_versions) else: if self.version: print('No package found for version {}'.format(self.version), file=sys.stderr) else: print('No package found for {}'.format(self.project_name), file=sys.stderr) return 1 return 0 def parse_search_spec(self, spec): """Parse search args and return spec dict for PyPI. * Owwww, my eyes!. Re-write this. @param spec: Cheese Shop package search spec e.g. name=Cheetah license=ZPL license=ZPL AND name=Cheetah @type spec: string @returns: tuple with spec and operator """ usage = """You can search PyPI by the following: name version author author_email maintainer maintainer_email home_page license summary description keywords platform download_url e.g. yolk -S name=Cheetah yolk -S name=yolk AND license=PSF """ if not spec: print(usage, file=sys.stderr) return (None, None) try: spec = (' ').join(spec) operator = 'AND' first = second = '' if ' AND ' in spec: (first, second) = spec.split('AND') elif ' OR ' in spec: (first, second) = spec.split('OR') operator = 'OR' else: first = spec (key1, term1) = first.split('=') key1 = key1.strip() if second: (key2, term2) = second.split('=') key2 = key2.strip() spec = {} spec[key1] = term1 if second: spec[key2] = term2 except: print(usage, file=sys.stderr) spec = operator = None return (spec, operator) def pypi_search(self): """Search PyPI by metadata keyword e.g. yolk -S name=yolk AND license=GPL @param spec: Cheese Shop search spec @type spec: list of strings spec examples: ["name=yolk"] ["license=GPL"] ["name=yolk", "AND", "license=GPL"] @returns: 0 on success or 1 if mal-formed search spec """ spec = self.pkg_spec # Add remaining cli arguments to options.pypi_search. search_arg = self.options.pypi_search spec.insert(0, search_arg.strip()) (spec, operator) = self.parse_search_spec(spec) if not spec: return 1 for pkg in self.pypi.search(spec, operator): if pkg['summary']: summary = pkg['summary'].encode('utf-8') else: summary = '' print("""{} ({}): {} """.format(pkg['name'].encode('utf-8'), pkg['version'], summary)) return 0 def show_entry_map(self): """Show entry map for a package. @param dist: package @param type: string @returns: 0 for success or 1 if error """ pprinter = pprint.PrettyPrinter() try: entry_map = pkg_resources.get_entry_map( self.options.show_entry_map) if entry_map: pprinter.pprint(entry_map) except pkg_resources.DistributionNotFound: print('Distribution not found: {}'.format( self.options.show_entry_map), file=sys.stderr) return 1 return 0 def show_entry_points(self): """Show entry points for a module. @returns: 0 for success or 1 if error """ found = False for entry_point in pkg_resources.iter_entry_points( self.options.show_entry_points): found = True try: plugin = entry_point.load() print(plugin.__module__) print(u' {}'.format(entry_point)) if plugin.__doc__: print(plugin.__doc__) print() except ImportError: pass if not found: print('No entry points found for {}'.format( self.options.show_entry_points), file=sys.stderr) return 1 return 0 def yolk_version(self): """Show yolk's version.""" print(u'yolk {}'.format(VERSION)) def parse_pkg_ver(self, want_installed): """Return tuple with project_name and version from CLI args If the user gave the wrong case for the project name, this corrects it. @param want_installed: whether package we want is installed or not @type want_installed: boolean @returns: tuple(project_name, version, all_versions) """ all_versions = [] arg_str = self.pkg_spec if '==' not in arg_str: # No version specified. project_name = arg_str version = None else: (project_name, version) = arg_str.split('==') project_name = project_name.strip() version = version.strip() # Find proper case for package name. if want_installed: project_name = yolklib.case_sensitive_name(project_name) else: (project_name, all_versions) = self.pypi.query_versions_pypi(project_name) if not len(all_versions): msg = "I'm afraid we have no '{}' at ".format(project_name) msg += 'The Cheese Shop. A little Red Leicester, perhaps?' print(msg, file=sys.stderr) sys.exit(2) return (project_name, version, all_versions)