def test_latest_vulndb(self):
dists = Distributions()
pkg = 'vulndb'
found = None
pypi = CheeseShop(False)
all_dists = dists.get_distributions('all', pkg,
dists.get_highest_installed(pkg))
for dist, active in all_dists:
project_name, versions = pypi.query_versions_pypi(
dist.project_name)
if versions:
# PyPI returns them in chronological order,
# but who knows if its guaranteed in the API?
# Make sure we grab the highest version:
newest = get_highest_version(versions)
if newest != dist.version:
# We may have newer than what PyPI knows about
if pkg_resources.parse_version(dist.version) < \
pkg_resources.parse_version(newest):
found = True
if found:
self.assertTrue(False, MESSAGE)
def get_fresh_updates(package_name="", version=""):
userpath = expanduser("~")
now = datetime.now()
# Do we have a cache ?
if isfile(userpath + "/.qyolk"):
f = open(userpath + "/.qyolk", "r")
cache = cPickle.load(f)
check_time = now - timedelta(hours=24)
if cache[0] > check_time:
# fresh cache, use it
return cache[1]
# No cache, get updates and create the cache
ret = []
pypi = CheeseShop()
dists = Distributions()
for pkg in get_pkglist():
for (dist, active) in dists.get_distributions(
"all", pkg, dists.get_highest_installed(pkg)):
(project_name,
versions) = pypi.query_versions_pypi(dist.project_name)
if versions:
newest = get_highest_version(versions)
if newest != dist.version:
if pkg_resources.parse_version(
dist.version) < pkg_resources.parse_version(
newest):
ret.append([project_name, dist.version, newest])
f = open(userpath + "/.qyolk", "w")
cPickle.dump([now, ret], f)
return ret
def get_fresh_updates(package_name="", version=""):
userpath = expanduser("~")
now = datetime.now()
# Do we have a cache ?
if isfile(userpath + "/.qyolk"):
f = open(userpath + "/.qyolk", "r")
cache = cPickle.load(f)
check_time = now - timedelta(hours=24)
if cache[0] > check_time:
# fresh cache, use it
return cache[1]
# No cache, get updates and create the cache
ret = []
pypi = CheeseShop()
dists = Distributions()
for pkg in get_pkglist():
for (dist, active) in dists.get_distributions("all", pkg, dists.get_highest_installed(pkg)):
(project_name, versions) = pypi.query_versions_pypi(dist.project_name)
if versions:
newest = get_highest_version(versions)
if newest != dist.version:
if pkg_resources.parse_version(dist.version) < pkg_resources.parse_version(newest):
ret.append([project_name, dist.version, newest])
f = open(userpath + "/.qyolk", "w")
cPickle.dump([now, ret], f)
return ret
class TestPyPi(BaseTestCase):
""""""
def setUp(self):
self.pypi = CheeseShop()
self.all_packages = []
for package in self.pypi.list_packages():
(pn, vers) = self.pypi.query_versions_pypi(package)
for version in vers:
try:
url = self.pypi.get_download_urls(pn, version)[0]
except IndexError:
pass
# TODO: log how many packages do not have URL
else:
self.all_packages.append((pn, version))
# TODO: cache entries with cPickle
#def test_get_vars_against_pypi(self):
#for package_name, version in self.all_packages:
#try:
#d = Enamer.get_vars(url, pn, version)
#except:
#pass
## TODO: maybe assert some of dictionary stuff?
#self.fail('Fail!')
def test_get_vars_against_pypi(self):
for package_name, version in self.all_packages:
#try:
main(['create', package_name, version])
#except:
#pass
# TODO: maybe assert some of dictionary stuff?
self.fail('Fail!')
def sync(self):
""""""
pypi = CheeseShop()
for package in pypi.list_packages():
(pn, vers) = pypi.query_versions_pypi(package)
for version in vers:
# TODO: parse_* will not return anything for correct atoms
atom = Enamer.construct_atom(
Enamer.parse_pn(pn)[0], self.config.category,
Enamer.parse_pv(version[0]))
# we skip existing ebuilds
if PortageUtils.ebuild_exists(atom):
continue
try:
url = pypi.get_download_urls(pn, version)[0]
# TODO: use setuptools way also
except IndexError:
log.warn('Skipping %s, no download url', atom)
else:
try:
self.config.configs['argparse']['uri'] = url
self.config.configs['argparse']['up_pn'] = pn
self.config.configs['argparse']['up_pv'] = version
gpypi = GPyPI(pn, version, self.config)
gpypi.create_ebuilds()
except KeyboardInterrupt:
raise
except:
log.exception(
'Unexpected error occured during ebuild creation:')
def sync(self):
""""""
pypi = CheeseShop()
for package in pypi.list_packages():
(pn, vers) = pypi.query_versions_pypi(package)
for version in vers:
# TODO: parse_* will not return anything for correct atoms
atom = Enamer.construct_atom(Enamer.parse_pn(pn)[0], self.config.category, Enamer.parse_pv(version[0]))
# we skip existing ebuilds
if PortageUtils.ebuild_exists(atom):
continue
try:
url = pypi.get_download_urls(pn, version)[0]
# TODO: use setuptools way also
except IndexError:
log.warn('Skipping %s, no download url', atom)
else:
try:
self.config.configs['argparse']['uri'] = url
self.config.configs['argparse']['up_pn'] = pn
self.config.configs['argparse']['up_pv'] = version
gpypi = GPyPI(pn, version, self.config)
gpypi.create_ebuilds()
except KeyboardInterrupt:
raise
except:
log.exception('Unexpected error occured during ebuild creation:')
def test_latest_vulndb(self):
dists = Distributions()
pkg = 'vulndb'
found = None
pypi = CheeseShop(False)
all_dists = dists.get_distributions('all', pkg,
dists.get_highest_installed(pkg))
for dist, active in all_dists:
project_name, versions = pypi.query_versions_pypi(dist.project_name)
if versions:
# PyPI returns them in chronological order,
# but who knows if its guaranteed in the API?
# Make sure we grab the highest version:
newest = get_highest_version(versions)
if newest != dist.version:
#We may have newer than what PyPI knows about
if pkg_resources.parse_version(dist.version) < \
pkg_resources.parse_version(newest):
found = True
if found:
self.assertTrue(False, MESSAGE)
def run(self):
"""
Perform actions based on CLI options
@returns: status code
"""
opt_parser = setup_opt_parser()
(self.options, remaining_args) = opt_parser.parse_args()
logger = self.set_log_level()
pkg_spec = validate_pypi_opts(opt_parser)
if not pkg_spec:
pkg_spec = remaining_args
self.pkg_spec = pkg_spec
if not self.options.pypi_search and (len(sys.argv) == 1 or\
len(remaining_args) > 2):
opt_parser.print_help()
return 2
#Options that depend on querying installed packages, not PyPI.
#We find the proper case for package names if they are installed,
#otherwise PyPI returns the correct case.
if self.options.show_deps or self.options.show_all or \
self.options.show_active or self.options.show_non_active or \
(self.options.show_updates and pkg_spec):
want_installed = True
else:
want_installed = False
#show_updates may or may not have a pkg_spec
if not want_installed or self.options.show_updates:
self.pypi = CheeseShop(self.options.debug)
#XXX: We should return 2 here if we couldn't create xmlrpc server
if pkg_spec:
(self.project_name, self.version, self.all_versions) = \
self.parse_pkg_ver(want_installed)
if want_installed and not self.project_name:
logger.error("%s is not installed." % pkg_spec[0])
return 1
#I could prefix all these with 'cmd_' and the methods also
#and then iterate over the `options` dictionary keys...
commands = [
'show_deps', 'query_metadata_pypi', 'fetch', 'versions_available',
'show_updates', 'browse_website', 'show_download_links',
'pypi_search', 'show_pypi_changelog', 'show_pypi_releases',
'yolk_version', 'show_all', 'show_active', 'show_non_active',
'show_entry_map', 'show_entry_points'
]
#Run first command it finds, and only the first command, then return
#XXX: Check if more than one command was set in options and give error?
for action in commands:
if getattr(self.options, action):
return getattr(self, action)()
opt_parser.print_help()
def setUp(self):
self.pypi = CheeseShop()
self.all_packages = []
for package in self.pypi.list_packages():
(pn, vers) = self.pypi.query_versions_pypi(package)
for version in vers:
try:
url = self.pypi.get_download_urls(pn, version)[0]
except IndexError:
pass
# TODO: log how many packages do not have URL
else:
self.all_packages.append((pn, version))
def get_latest_version_number(package_name):
"""Receives a package name and
returns the latest version its available in from pip"""
pkg, all_versions = CheeseShop().query_versions_pypi(package_name)
if len(all_versions):
return all_versions[0]
return None
def get_fresh_updates(package_name="", version=""):
ret = []
pypi = CheeseShop()
for pkg in get_pkglist():
for (dist, active) in get_distributions("all", pkg,
get_highest_installed(pkg)):
(project_name,
versions) = pypi.query_versions_pypi(dist.project_name)
if versions:
newest = get_highest_version(versions)
if newest != dist.version:
if pkg_resources.parse_version(
dist.version) < pkg_resources.parse_version(
newest):
ret.append([project_name, dist.version, newest])
return ret
def run(self):
"""
Perform actions based on CLI options
@returns: status code
"""
opt_parser = setup_opt_parser()
(self.options, remaining_args) = opt_parser.parse_args()
logger = self.set_log_level()
pkg_spec = validate_pypi_opts(opt_parser)
if not pkg_spec:
pkg_spec = remaining_args
self.pkg_spec = pkg_spec
if not self.options.pypi_search and (len(sys.argv) == 1 or\
len(remaining_args) > 2):
opt_parser.print_help()
return 2
#Options that depend on querying installed packages, not PyPI.
#We find the proper case for package names if they are installed,
#otherwise PyPI returns the correct case.
if self.options.show_deps or self.options.show_all or \
self.options.show_active or self.options.show_non_active or \
(self.options.show_updates and pkg_spec):
want_installed = True
else:
want_installed = False
#show_updates may or may not have a pkg_spec
if not want_installed or self.options.show_updates:
self.pypi = CheeseShop(self.options.debug)
#XXX: We should return 2 here if we couldn't create xmlrpc server
if pkg_spec:
(self.project_name, self.version, self.all_versions) = \
self.parse_pkg_ver(want_installed)
if want_installed and not self.project_name:
logger.error("%s is not installed." % pkg_spec[0])
return 1
#I could prefix all these with 'cmd_' and the methods also
#and then iterate over the `options` dictionary keys...
commands = ['show_deps', 'query_metadata_pypi', 'fetch',
'versions_available', 'show_updates', 'browse_website',
'show_download_links', 'pypi_search', 'show_pypi_changelog',
'show_pypi_releases', 'yolk_version', 'show_all',
'show_active', 'show_non_active', 'show_entry_map',
'show_entry_points']
#Run first command it finds, and only the first command, then return
#XXX: Check if more than one command was set in options and give error?
for action in commands:
if getattr(self.options, action):
return getattr(self, action)()
opt_parser.print_help()
def run(self):
"""Perform actions based on CLI options.
@returns: status code
"""
parser = setup_parser()
try:
import argcomplete
argcomplete.autocomplete(parser)
except ImportError:
pass
self.options = parser.parse_args()
pkg_spec = validate_pypi_opts(parser)
if not pkg_spec:
pkg_spec = self.options.pkg_spec
self.pkg_spec = pkg_spec
if self.options.fields:
self.options.fields = [
s.strip().lower() for s in self.options.fields.split(',')
]
else:
self.options.fields = []
if (not self.options.pypi_search and len(sys.argv) == 1):
parser.print_help()
return 2
# Options that depend on querying installed packages, not PyPI.
# We find the proper case for package names if they are installed,
# otherwise PyPI returns the correct case.
if (self.options.show_deps or self.options.show_all
or self.options.show_active or self.options.show_non_active
or (self.options.show_updates and pkg_spec)
or self.options.upgrade):
want_installed = True
else:
want_installed = False
# show_updates may or may not have a pkg_spec
if (not want_installed or self.options.show_updates
or self.options.upgrade):
self.pypi = CheeseShop(self.options.debug)
# XXX: We should return 2 here if we couldn't create xmlrpc server
if pkg_spec:
(self.project_name, self.version,
self.all_versions) = self.parse_pkg_ver(want_installed)
if want_installed and not self.project_name:
print(u'{} is not installed'.format(pkg_spec), file=sys.stderr)
return 1
# I could prefix all these with 'cmd_' and the methods also
# and then iterate over the `options` dictionary keys...
commands = [
'show_deps', 'query_metadata_pypi', 'fetch', 'versions_available',
'show_updates', 'upgrade', 'browse_website', 'show_download_links',
'pypi_search', 'show_pypi_changelog', 'show_pypi_releases',
'yolk_version', 'show_all', 'show_active', 'show_non_active',
'show_entry_map', 'show_entry_points'
]
# Run first command it finds, and only the first command, then return
# XXX: Check if more than one command was set in options and give
# error?
for action in commands:
if getattr(self.options, action):
return getattr(self, action)()
parser.print_help()
def __init__(self, package_name, version, options):
self.package_name = package_name
self.version = version
self.options = options
self.tree = [(package_name, version)]
self.pypi = CheeseShop()
class GPyPI(object):
"""
Main class for GPyPi interface
:param package_name: case-insensitive package name
:type package_name: string
:param version: package version
:type version: string
:param options: command-line options
:type options: ArgParse options
"""
def __init__(self, package_name, version, options):
self.package_name = package_name
self.version = version
self.options = options
self.tree = [(package_name, version)]
self.pypi = CheeseShop()
def create_ebuilds(self):
"""
Create ebuild for given package_name and any ebuilds for dependencies
if needed. If no version is given we use the highest available.
"""
while len(self.tree):
(project_name, version) = self.tree.pop(0)
self.package_name = project_name
self.version = version
requires = self.do_ebuild()
if requires:
for req in requires:
if self.options.no_deps:
pass
else:
self.handle_dependencies(req.project_name)
# TODO: disable some options after first ebuild is created
#self.options.overwrite = False
#self.options.category = None
def handle_dependencies(self, project_name):
"""Add dependency if not already in self.tree"""
pkgs = []
if len(self.tree):
for deps in self.tree:
pkgs.append(deps[0])
if project_name not in pkgs:
# TODO: document that we can not query pypi with version spec or use distutils2
# for dependencies
self.tree.append((project_name, None))
log.info("Dependency needed: %s" % project_name)
def url_from_pypi(self):
"""
Query PyPI to find a package's URI
:returns: source URL string
"""
try:
return self.pypi.get_download_urls(self.package_name, self.version, pkg_type="source")[0]
except IndexError:
return None
def url_from_setuptools(self):
"""
Use setuptools to find a package's URI
:returns: source URL string or None
"""
#if self.options.subversion:
# src_uri = get_download_uri(self.package_name, "dev", "source")
#else:
return get_download_uri(self.package_name, self.version, "source", self.options.index_url)
def find_uri(self, method="all"):
"""
Returns download URI for package
If no package version was given it returns highest available
Setuptools should find anything xml-rpc can and more.
:param method: download method can be 'xml-rpc', 'setuptools', or 'all'
:type method: string
:returns download_url string
"""
download_url = None
if method == "all" or method == "xml-rpc":
download_url = self.url_from_pypi()
if (method == "all" or method == "setuptools") and not download_url:
#Sometimes setuptools can find a package URI if PyPI doesn't have it
download_url = self.url_from_setuptools()
# TODO: configuratior
log.debug("Package URI: %s " % download_url)
return download_url
def do_ebuild(self):
"""
Get SRC_URI using PyPI and attempt to create ebuild
:returns: tuple with exit code and pkg_resources requirement
"""
#Get proper case for project name:
(self.package_name, versions) = self.pypi.query_versions_pypi(self.package_name)
if not versions:
log.error("No package %s on PyPi." % self.package_name)
return
if self.version and (self.version not in versions):
log.error("No package %s for version %s on PyPi." % (self.package_name, self.version))
return
else:
self.version = get_highest_version(versions)
# TODO: self.options.uri only for first ebuild
# TODO: make find_uri method configurable
download_url = self.find_uri()
log.info('Generating ebuild: %s %s', self.package_name, self.version)
log.debug('URI from PyPi: %s', download_url)
self.options.configs['argparse']['uri'] = download_url
self.options.configs['argparse']['up_pn'] = self.package_name
self.options.configs['argparse']['up_pv'] = self.version
ebuild = Ebuild(self.options)
ebuild.set_metadata(self.query_metadata())
if self.options.command == 'echo':
ebuild.print_formatted()
else:
ebuild.create()
return ebuild.requires
def query_metadata(self):
"""
Get package metadata from PyPI
:returns: metadata text
"""
if self.version:
return self.pypi.release_data(self.package_name, self.version)
else:
(pn, vers) = self.pypi.query_versions_pypi(self.package_name)
return self.pypi.release_data(self.package_name, get_highest_version(vers))
class Yolk(object):
"""Main class for yolk."""
def __init__(self):
# PyPI project name with proper case
self.project_name = ''
# PyPI project version
self.version = ''
# List of all versions not hidden on PyPI
self.all_versions = []
self.pkg_spec = None
self.options = None
# Squelch output from setuptools
# Add future offenders to this list.
shut_up = ['distutils.log']
sys.stdout = StdOut(sys.stdout, shut_up)
sys.stderr = StdOut(sys.stderr, shut_up)
self.pypi = None
def get_plugin(self, method):
"""Return plugin object if CLI option is activated and method exists.
@param method: name of plugin's method we're calling
@type method: string
@returns: list of plugins with `method`
"""
all_plugins = []
for entry_point in pkg_resources.iter_entry_points('yolk.plugins'):
plugin_obj = entry_point.load()
plugin = plugin_obj()
plugin.configure(self.options, None)
if plugin.enabled:
if not hasattr(plugin, method):
plugin = None
else:
all_plugins.append(plugin)
return all_plugins
def run(self):
"""Perform actions based on CLI options.
@returns: status code
"""
parser = setup_parser()
try:
import argcomplete
argcomplete.autocomplete(parser)
except ImportError:
pass
self.options = parser.parse_args()
pkg_spec = validate_pypi_opts(parser)
if not pkg_spec:
pkg_spec = self.options.pkg_spec
self.pkg_spec = pkg_spec
if self.options.fields:
self.options.fields = [s.strip().lower()
for s in self.options.fields.split(',')]
else:
self.options.fields = []
if (
not self.options.pypi_search and
len(sys.argv) == 1
):
parser.print_help()
return 2
# Options that depend on querying installed packages, not PyPI.
# We find the proper case for package names if they are installed,
# otherwise PyPI returns the correct case.
if (
self.options.show_deps or
self.options.show_all or
self.options.show_active or
self.options.show_non_active or
(self.options.show_updates and pkg_spec) or
self.options.upgrade
):
want_installed = True
else:
want_installed = False
# show_updates may or may not have a pkg_spec
if (
not want_installed or
self.options.show_updates or
self.options.upgrade
):
self.pypi = CheeseShop(self.options.debug)
# XXX: We should return 2 here if we couldn't create xmlrpc server
if pkg_spec:
(self.project_name,
self.version,
self.all_versions) = self.parse_pkg_ver(want_installed)
if want_installed and not self.project_name:
print('{} is not installed'.format(pkg_spec),
file=sys.stderr)
return 1
# I could prefix all these with 'cmd_' and the methods also
# and then iterate over the `options` dictionary keys...
commands = ['show_deps', 'query_metadata_pypi', 'fetch',
'versions_available', 'show_updates', 'upgrade',
'browse_website',
'show_download_links', 'pypi_search',
'show_pypi_changelog', 'show_pypi_releases',
'yolk_version', 'show_all',
'show_active', 'show_non_active', 'show_entry_map',
'show_entry_points']
# Run first command it finds, and only the first command, then return
# XXX: Check if more than one command was set in options and give
# error?
for action in commands:
if getattr(self.options, action):
return getattr(self, action)()
parser.print_help()
def show_active(self):
"""Show installed active packages."""
return self.show_distributions('active')
def show_non_active(self):
"""Show installed non-active packages."""
return self.show_distributions('nonactive')
def show_all(self):
"""Show all installed packages."""
return self.show_distributions('all')
def show_updates(self):
"""Check installed packages for available updates on PyPI.
@param project_name: optional package name to check; checks every
installed package if none specified
@type project_name: string
@returns: None
"""
if self.project_name:
pkg_list = [self.project_name]
else:
pkg_list = get_pkglist()
for (project_name, version, newest) in _updates(
pkg_list,
self.pypi,
user_installs_only=self.options.user):
print('{} {} ({})'.format(project_name,
version,
newest))
return 0
def upgrade(self):
"""Check installed packages for available updates on PyPI and upgrade.
@param project_name: optional package name to check; checks every
installed package if none specified
@type project_name: string
@returns: None
"""
if self.project_name:
pkg_list = [self.project_name]
else:
pkg_list = get_pkglist()
names = [values[0]
for values in _updates(pkg_list,
self.pypi,
user_installs_only=self.options.user)]
if names:
subprocess.call(
[sys.executable, '-m', 'pip', 'install', '--upgrade'] +
(['--user'] if self.options.user else []) +
names)
return 0
def show_distributions(self, show):
"""Show list of installed activated OR non-activated packages.
@param show: type of pkgs to show (all, active or nonactive)
@type show: string
@returns: None or 2 if error
"""
# Search for any plugins with active CLI options with add_column()
# method.
plugins = self.get_plugin('add_column')
# Some locations show false positive for 'development' packages:
ignores = ['/UNIONFS', '/KNOPPIX.IMG']
# See http://cheeseshop.python.org/pypi/workingenv.py for details.
workingenv = os.environ.get('WORKING_ENV')
if workingenv:
ignores.append(workingenv)
results = None
for (dist, active) in yolklib.get_distributions(show,
self.project_name,
self.version):
metadata = get_metadata(dist)
for prefix in ignores:
if dist.location.startswith(prefix):
dist.location = dist.location.replace(prefix, '')
# Case-insensitive search because of Windows.
if dist.location.lower().startswith(get_python_lib().lower()):
develop = ''
else:
develop = dist.location
if metadata:
add_column_text = ''
for my_plugin in plugins:
# See if package is 'owned' by a package manager such as
# portage, apt, rpm etc.
add_column_text += my_plugin.add_column(dist) + ' '
self.print_metadata(metadata, develop, active, add_column_text)
else:
print(str(dist) + ' has no metadata')
results = True
if not results and self.project_name:
if self.version:
pkg_spec = '{}=={}'.format(self.project_name, self.version)
else:
pkg_spec = self.project_name
if show == 'all':
print('There are no versions of {} installed'.format(pkg_spec),
file=sys.stderr)
else:
print(
'There are no {} versions of {} installed'.format(
show, pkg_spec),
file=sys.stderr)
return 2
elif show == 'all' and results and self.options.fields:
print("Versions with '*' are non-active.")
print("Versions with '!' are deployed in development mode.")
def print_metadata(self, metadata, develop, active, installed_by):
"""Print out formatted metadata.
@param metadata: package's metadata
@type metadata: pkg_resources Distribution obj
@param develop: path to pkg if its deployed in development mode
@type develop: string
@param active: show if package is activated or not
@type active: boolean
@param installed_by: Shows if pkg was installed by a package manager
other than setuptools
@type installed_by: string
@returns: None
"""
show_metadata = self.options.metadata
version = metadata['Version']
# When showing all packages, note which are not active:
if active:
if self.options.fields:
active_status = ''
else:
active_status = 'active'
else:
if self.options.fields:
active_status = '*'
else:
active_status = 'non-active'
if develop:
if self.options.fields:
development_status = '! ({})'.format(develop)
else:
development_status = 'development ({})'.format(develop)
else:
development_status = installed_by
status = '{} {}'.format(active_status, development_status)
if self.options.fields:
print(
'{} ({}){} {}'.format(metadata['Name'], version, active_status,
development_status))
else:
# Need intelligent justification.
print(metadata['Name'].ljust(15) + ' - ' + version.ljust(12) +
' - ' + status)
if self.options.fields:
for field in metadata.keys():
if field.lower() in self.options.fields:
print(' {}: {}'.format(field, metadata[field]))
print()
elif show_metadata:
for field in metadata.keys():
if field != 'Name' and field != 'Summary':
print(' {}: {}'.format(field, metadata[field]))
def show_deps(self):
"""Show dependencies for package(s)
@returns: 0 - success 1 - No dependency info supplied
"""
pkgs = pkg_resources.Environment()
for pkg in pkgs[self.project_name]:
if not self.version:
print(pkg.project_name, pkg.version)
i = len(list(pkg._dep_map.values())[0])
if i:
while i:
if (
not self.version or
self.version and
pkg.version == self.version
):
if self.version and i == len(list(
pkg._dep_map.values())[0]):
print(pkg.project_name, pkg.version)
print(' ' + str(list(
pkg._dep_map.values())[0][i - 1]))
i -= 1
else:
return 1
return 0
def show_pypi_changelog(self):
"""Show detailed PyPI ChangeLog for the last `hours`
@returns: 0 = success or 1 if failed to retrieve from XML-RPC server
"""
hours = self.options.show_pypi_changelog
if not hours.isdigit():
print('You must supply an integer',
file=sys.stderr)
return 1
try:
changelog = self.pypi.changelog(int(hours))
except XMLRPCFault as err_msg:
print(err_msg, file=sys.stderr)
print("Couldn't retrieve changelog", file=sys.stderr)
return 1
last_pkg = ''
for entry in changelog:
pkg = entry[0]
if pkg != last_pkg:
print('{} {}\n\t{}'.format(entry[0], entry[1], entry[3]))
last_pkg = pkg
else:
print('\t{}'.format(entry[3]))
return 0
def show_pypi_releases(self):
"""Show PyPI releases for the last number of `hours`
@returns: 0 = success or 1 if failed to retrieve from XML-RPC server
"""
try:
hours = int(self.options.show_pypi_releases)
except ValueError:
print('You must supply an integer', file=sys.stderr)
return 1
try:
latest_releases = self.pypi.updated_releases(hours)
except XMLRPCFault as err_msg:
print(err_msg, file=sys.stderr)
print("Couldn't retrieve latest releases.", file=sys.stderr)
return 1
for release in latest_releases:
print('{} {}'.format(release[0], release[1]))
return 0
def show_download_links(self):
"""Query PyPI for pkg download URI for a packge.
@returns: 0
"""
# In case they specify version as 'dev' instead of using -T svn,
# don't show three svn URI's
if self.options.file_type == 'all' and self.version == 'dev':
self.options.file_type = 'svn'
if self.options.file_type == 'svn':
version = 'dev'
else:
if self.version:
version = self.version
else:
version = self.all_versions[0]
if self.options.file_type == 'all':
# Search for source, egg, and svn.
self.print_download_uri(version, True)
self.print_download_uri(version, False)
self.print_download_uri('dev', True)
else:
if self.options.file_type == 'source':
source = True
else:
source = False
self.print_download_uri(version, source)
return 0
def print_download_uri(self, version, source):
"""@param version: version number or 'dev' for svn.
@type version: string
@param source: download source or egg
@type source: boolean
@returns: None
"""
if version == 'dev':
source = True
# Use setuptools monkey-patch to grab url.
url = get_download_uri(self.project_name, version, source,
self.options.pypi_index)
if url:
print('{}'.format(url))
def fetch(self):
"""Download a package.
@returns: 0 = success or 1 if failed download
"""
source = True
directory = '.'
if self.options.file_type == 'svn':
svn_uri = get_download_uri(self.project_name,
'dev', True)
if svn_uri:
directory = self.project_name + '_svn'
return self.fetch_svn(svn_uri, directory)
else:
print(
'No subversion repository found for {}'.format(
self.project_name),
file=sys.stderr)
return 1
elif self.options.file_type == 'source':
source = True
elif self.options.file_type == 'egg':
source = False
uri = get_download_uri(self.project_name, self.version, source)
if uri:
return self.fetch_uri(directory, uri)
else:
print('No {} URI found for package: {}'.format(
self.options.file_type, self.project_name))
return 1
def fetch_uri(self, directory, uri):
"""Use ``urllib.urlretrieve`` to download package to file in sandbox
dir.
@param directory: directory to download to
@type directory: string
@param uri: uri to download
@type uri: string
@returns: 0 = success or 1 for failed download
"""
filename = os.path.basename(urlparse(uri)[2])
if os.path.exists(filename):
print('File exists: ' + filename, file=sys.stderr)
return 1
try:
downloaded_filename, headers = urlretrieve(uri, filename)
except IOError as err_msg:
print(
'Error downloading package {} from URL {}'.format(
filename, uri),
file=sys.stderr)
print(str(err_msg), file=sys.stderr)
return 1
if 'text/html' in headers:
dfile = open(downloaded_filename)
if re.search('404 Not Found', ''.join(dfile.readlines())):
dfile.close()
print("'404 Not Found' error", file=sys.stderr)
return 1
dfile.close()
return 0
def fetch_svn(self, svn_uri, directory):
"""Fetch subversion repository.
@param svn_uri: subversion repository uri to check out
@type svn_uri: string
@param directory: directory to download to
@type directory: string
"""
if not command_successful(['svn', '--version']):
raise YolkException('Do you have subversion installed?')
if os.path.exists(directory):
raise YolkException(
'Checkout directory exists - {}'.format(directory))
try:
os.mkdir(directory)
except OSError as err_msg:
raise YolkException('' + str(err_msg))
cwd = os.path.realpath(os.curdir)
os.chdir(directory)
status, _ = run_command(['svn', 'checkout', svn_uri])
os.chdir(cwd)
def browse_website(self, browser=None):
"""Launch web browser at project's homepage.
@param browser: name of web browser to use
@type browser: string
@returns: 0 if homepage found, 1 if no homepage found
"""
if len(self.all_versions):
metadata = self.pypi.release_data(self.project_name,
self.all_versions[0])
if 'home_page' in metadata:
if browser == 'konqueror':
browser = webbrowser.Konqueror()
else:
browser = webbrowser.get()
browser.open(metadata['home_page'], 2)
return 0
print('No homepage URL found', file=sys.stderr)
return 1
def query_metadata_pypi(self):
"""Show pkg metadata queried from PyPI.
@returns: 0
"""
if self.version and self.version in self.all_versions:
metadata = self.pypi.release_data(self.project_name, self.version)
else:
# Give highest version
metadata = self.pypi.release_data(self.project_name,
self.all_versions[0])
if metadata:
if len(self.options.fields) == 1:
try:
print(metadata[self.options.fields[0]])
except KeyError:
pass
else:
for key in metadata.keys():
if (
not self.options.fields or
(self.options.fields and
key.lower() in self.options.fields)
):
print('{}: {}'.format(key, metadata[key]))
return 0
def versions_available(self):
"""Query PyPI for a particular version or all versions of a package.
@returns: 0 if version(s) found or 1 if none found
"""
if self.all_versions and self.version in self.all_versions:
print_pkg_versions(self.project_name, [self.version])
elif not self.version and self.all_versions:
print_pkg_versions(self.project_name, self.all_versions)
else:
if self.version:
print(
'No package found for version {}'.format(self.version),
file=sys.stderr)
else:
print(
'No package found for {}'.format(self.project_name),
file=sys.stderr)
return 1
return 0
def parse_search_spec(self, spec):
"""Parse search args and return spec dict for PyPI.
* Owwww, my eyes!. Re-write this.
@param spec: Cheese Shop package search spec
e.g.
name=Cheetah
license=ZPL
license=ZPL AND name=Cheetah
@type spec: string
@returns: tuple with spec and operator
"""
usage = """You can search PyPI by the following:
name
version
author
author_email
maintainer
maintainer_email
home_page
license
summary
description
keywords
platform
download_url
e.g. yolk -S name=Cheetah
yolk -S name=yolk AND license=PSF
"""
if not spec:
print(usage, file=sys.stderr)
return (None, None)
try:
spec = (' ').join(spec)
operator = 'AND'
first = second = ''
if ' AND ' in spec:
(first, second) = spec.split('AND')
elif ' OR ' in spec:
(first, second) = spec.split('OR')
operator = 'OR'
else:
first = spec
(key1, term1) = first.split('=')
key1 = key1.strip()
if second:
(key2, term2) = second.split('=')
key2 = key2.strip()
spec = {}
spec[key1] = term1
if second:
spec[key2] = term2
except:
print(usage, file=sys.stderr)
spec = operator = None
return (spec, operator)
def pypi_search(self):
"""Search PyPI by metadata keyword e.g.
yolk -S name=yolk AND license=GPL
@param spec: Cheese Shop search spec
@type spec: list of strings
spec examples:
["name=yolk"]
["license=GPL"]
["name=yolk", "AND", "license=GPL"]
@returns: 0 on success or 1 if mal-formed search spec
"""
spec = self.pkg_spec
# Add remaining cli arguments to options.pypi_search.
search_arg = self.options.pypi_search
spec.insert(0, search_arg.strip())
(spec, operator) = self.parse_search_spec(spec)
if not spec:
return 1
for pkg in self.pypi.search(spec, operator):
if pkg['summary']:
summary = pkg['summary'].encode('utf-8')
else:
summary = ''
print("""{} ({}):
{}
""".format(pkg['name'].encode('utf-8'), pkg['version'],
summary))
return 0
def show_entry_map(self):
"""Show entry map for a package.
@param dist: package
@param type: string
@returns: 0 for success or 1 if error
"""
pprinter = pprint.PrettyPrinter()
try:
entry_map = pkg_resources.get_entry_map(
self.options.show_entry_map)
if entry_map:
pprinter.pprint(entry_map)
except pkg_resources.DistributionNotFound:
print(
'Distribution not found: {}'.format(
self.options.show_entry_map),
file=sys.stderr)
return 1
return 0
def show_entry_points(self):
"""Show entry points for a module.
@returns: 0 for success or 1 if error
"""
found = False
for entry_point in pkg_resources.iter_entry_points(
self.options.show_entry_points):
found = True
try:
plugin = entry_point.load()
print(plugin.__module__)
print(' {}'.format(entry_point))
if plugin.__doc__:
print(plugin.__doc__)
print()
except ImportError:
pass
if not found:
print(
'No entry points found for {}'.format(
self.options.show_entry_points),
file=sys.stderr)
return 1
return 0
def yolk_version(self):
"""Show yolk's version."""
print('yolk {}'.format(VERSION))
def parse_pkg_ver(self, want_installed):
"""Return tuple with project_name and version from CLI args If the user
gave the wrong case for the project name, this corrects it.
@param want_installed: whether package we want is installed or not
@type want_installed: boolean
@returns: tuple(project_name, version, all_versions)
"""
all_versions = []
arg_str = self.pkg_spec
if '==' not in arg_str:
# No version specified.
project_name = arg_str
version = None
else:
(project_name, version) = arg_str.split('==')
project_name = project_name.strip()
version = version.strip()
# Find proper case for package name.
if want_installed:
project_name = yolklib.case_sensitive_name(project_name)
else:
(project_name, all_versions) = self.pypi.query_versions_pypi(
project_name)
if not len(all_versions):
msg = "I'm afraid we have no '{}' at ".format(project_name)
msg += 'The Cheese Shop. A little Red Leicester, perhaps?'
print(msg, file=sys.stderr)
sys.exit(2)
return (project_name, version, all_versions)
def __init__(self, package_name, version, options):
self.package_name = package_name
self.version = version
self.options = options
self.tree = [(package_name, version)]
self.pypi = CheeseShop()
class GPyPI(object):
"""
Main class for GPyPi interface
:param package_name: case-insensitive package name
:type package_name: string
:param version: package version
:type version: string
:param options: command-line options
:type options: ArgParse options
"""
def __init__(self, package_name, version, options):
self.package_name = package_name
self.version = version
self.options = options
self.tree = [(package_name, version)]
self.pypi = CheeseShop()
def create_ebuilds(self):
"""
Create ebuild for given package_name and any ebuilds for dependencies
if needed. If no version is given we use the highest available.
"""
while len(self.tree):
(project_name, version) = self.tree.pop(0)
self.package_name = project_name
self.version = version
requires = self.do_ebuild()
if requires:
for req in requires:
if self.options.no_deps:
pass
else:
self.handle_dependencies(req.project_name)
# TODO: disable some options after first ebuild is created
#self.options.overwrite = False
#self.options.category = None
def handle_dependencies(self, project_name):
"""Add dependency if not already in self.tree"""
pkgs = []
if len(self.tree):
for deps in self.tree:
pkgs.append(deps[0])
if project_name not in pkgs:
# TODO: document that we can not query pypi with version spec or use distutils2
# for dependencies
self.tree.append((project_name, None))
log.info("Dependency needed: %s" % project_name)
def url_from_pypi(self):
"""
Query PyPI to find a package's URI
:returns: source URL string
"""
try:
return self.pypi.get_download_urls(self.package_name,
self.version,
pkg_type="source")[0]
except IndexError:
return None
def url_from_setuptools(self):
"""
Use setuptools to find a package's URI
:returns: source URL string or None
"""
#if self.options.subversion:
# src_uri = get_download_uri(self.package_name, "dev", "source")
#else:
return get_download_uri(self.package_name, self.version, "source",
self.options.index_url)
def find_uri(self, method="all"):
"""
Returns download URI for package
If no package version was given it returns highest available
Setuptools should find anything xml-rpc can and more.
:param method: download method can be 'xml-rpc', 'setuptools', or 'all'
:type method: string
:returns download_url string
"""
download_url = None
if method == "all" or method == "xml-rpc":
download_url = self.url_from_pypi()
if (method == "all" or method == "setuptools") and not download_url:
#Sometimes setuptools can find a package URI if PyPI doesn't have it
download_url = self.url_from_setuptools()
# TODO: configuratior
log.debug("Package URI: %s " % download_url)
return download_url
def do_ebuild(self):
"""
Get SRC_URI using PyPI and attempt to create ebuild
:returns: tuple with exit code and pkg_resources requirement
"""
#Get proper case for project name:
(self.package_name,
versions) = self.pypi.query_versions_pypi(self.package_name)
if not versions:
log.error("No package %s on PyPi." % self.package_name)
return
if self.version and (self.version not in versions):
log.error("No package %s for version %s on PyPi." %
(self.package_name, self.version))
return
else:
self.version = get_highest_version(versions)
# TODO: self.options.uri only for first ebuild
# TODO: make find_uri method configurable
download_url = self.find_uri()
log.info('Generating ebuild: %s %s', self.package_name, self.version)
log.debug('URI from PyPi: %s', download_url)
self.options.configs['argparse']['uri'] = download_url
self.options.configs['argparse']['up_pn'] = self.package_name
self.options.configs['argparse']['up_pv'] = self.version
ebuild = Ebuild(self.options)
ebuild.set_metadata(self.query_metadata())
if self.options.command == 'echo':
ebuild.print_formatted()
else:
ebuild.create()
return ebuild.requires
def query_metadata(self):
"""
Get package metadata from PyPI
:returns: metadata text
"""
if self.version:
return self.pypi.release_data(self.package_name, self.version)
else:
(pn, vers) = self.pypi.query_versions_pypi(self.package_name)
return self.pypi.release_data(self.package_name,
get_highest_version(vers))
class Yolk(object):
"""
Main class for yolk
"""
def __init__(self):
#PyPI project name with proper case
self.project_name = ""
#PyPI project version
self.version = ""
#List of all versions not hidden on PyPI
self.all_versions = []
self.pkg_spec = []
self.options = None
self.logger = logging.getLogger("yolk")
#Squelch output from setuptools
#Add future offenders to this list.
shut_up = ['distutils.log']
sys.stdout = StdOut(sys.stdout, shut_up)
sys.stderr = StdOut(sys.stderr, shut_up)
self.pypi = None
def get_plugin(self, method):
"""
Return plugin object if CLI option is activated and method exists
@param method: name of plugin's method we're calling
@type method: string
@returns: list of plugins with `method`
"""
all_plugins = []
for entry_point in pkg_resources.iter_entry_points('yolk.plugins'):
plugin_obj = entry_point.load()
plugin = plugin_obj()
plugin.configure(self.options, None)
if plugin.enabled:
if not hasattr(plugin, method):
self.logger.warn("Error: plugin has no method: %s" %
method)
plugin = None
else:
all_plugins.append(plugin)
return all_plugins
def set_log_level(self):
"""
Set log level according to command-line options
@returns: logger object
"""
if self.options.debug:
self.logger.setLevel(logging.DEBUG)
elif self.options.quiet:
self.logger.setLevel(logging.ERROR)
else:
self.logger.setLevel(logging.INFO)
self.logger.addHandler(logging.StreamHandler())
return self.logger
def run(self):
"""
Perform actions based on CLI options
@returns: status code
"""
opt_parser = setup_opt_parser()
(self.options, remaining_args) = opt_parser.parse_args()
logger = self.set_log_level()
pkg_spec = validate_pypi_opts(opt_parser)
if not pkg_spec:
pkg_spec = remaining_args
self.pkg_spec = pkg_spec
if not self.options.pypi_search and (len(sys.argv) == 1 or\
len(remaining_args) > 2):
opt_parser.print_help()
return 2
#Options that depend on querying installed packages, not PyPI.
#We find the proper case for package names if they are installed,
#otherwise PyPI returns the correct case.
if self.options.show_deps or self.options.show_all or \
self.options.show_active or self.options.show_non_active or \
(self.options.show_updates and pkg_spec):
want_installed = True
else:
want_installed = False
#show_updates may or may not have a pkg_spec
if not want_installed or self.options.show_updates:
self.pypi = CheeseShop(self.options.debug)
#XXX: We should return 2 here if we couldn't create xmlrpc server
if pkg_spec:
(self.project_name, self.version, self.all_versions) = \
self.parse_pkg_ver(want_installed)
if want_installed and not self.project_name:
logger.error("%s is not installed." % pkg_spec[0])
return 1
#I could prefix all these with 'cmd_' and the methods also
#and then iterate over the `options` dictionary keys...
commands = [
'show_deps', 'query_metadata_pypi', 'fetch', 'versions_available',
'show_updates', 'browse_website', 'show_download_links',
'pypi_search', 'show_pypi_changelog', 'show_pypi_releases',
'yolk_version', 'show_all', 'show_active', 'show_non_active',
'show_entry_map', 'show_entry_points'
]
#Run first command it finds, and only the first command, then return
#XXX: Check if more than one command was set in options and give error?
for action in commands:
if getattr(self.options, action):
return getattr(self, action)()
opt_parser.print_help()
def show_active(self):
"""
Show installed active packages
"""
return self.show_distributions("active")
def show_non_active(self):
"""
Show installed non-active packages
"""
return self.show_distributions("nonactive")
def show_all(self):
"""
Show all installed packages
"""
return self.show_distributions("all")
def show_updates(self):
"""
Check installed packages for available updates on PyPI
@param project_name: optional package name to check; checks every
installed pacakge if none specified
@type project_name: string
@returns: None
"""
dists = Distributions()
if self.project_name:
#Check for a single package
pkg_list = [self.project_name]
else:
#Check for every installed package
pkg_list = get_pkglist()
found = None
for pkg in pkg_list:
for (dist, active) in dists.get_distributions(
"all", pkg, dists.get_highest_installed(pkg)):
(project_name, versions) = \
self.pypi.query_versions_pypi(dist.project_name)
if versions:
#PyPI returns them in chronological order,
#but who knows if its guaranteed in the API?
#Make sure we grab the highest version:
newest = get_highest_version(versions)
if newest != dist.version:
#We may have newer than what PyPI knows about
if pkg_resources.parse_version(dist.version) < \
pkg_resources.parse_version(newest):
found = True
print " %s %s (%s)" % (project_name, dist.version,
newest)
if not found and self.project_name:
self.logger.info("You have the latest version installed.")
elif not found:
self.logger.info("No newer packages found at The Cheese Shop")
return 0
def show_distributions(self, show):
"""
Show list of installed activated OR non-activated packages
@param show: type of pkgs to show (all, active or nonactive)
@type show: string
@returns: None or 2 if error
"""
show_metadata = self.options.metadata
#Search for any plugins with active CLI options with add_column() method
plugins = self.get_plugin("add_column")
#Some locations show false positive for 'development' packages:
ignores = ["/UNIONFS", "/KNOPPIX.IMG"]
#Check if we're in a workingenv
#See http://cheeseshop.python.org/pypi/workingenv.py
workingenv = os.environ.get('WORKING_ENV')
if workingenv:
ignores.append(workingenv)
dists = Distributions()
results = None
for (dist, active) in dists.get_distributions(show, self.project_name,
self.version):
metadata = get_metadata(dist)
for prefix in ignores:
if dist.location.startswith(prefix):
dist.location = dist.location.replace(prefix, "")
#Case-insensitve search because of Windows
if dist.location.lower().startswith(get_python_lib().lower()):
develop = ""
else:
develop = dist.location
if metadata:
add_column_text = ""
for my_plugin in plugins:
#See if package is 'owned' by a package manager such as
#portage, apt, rpm etc.
#add_column_text += my_plugin.add_column(filename) + " "
add_column_text += my_plugin.add_column(dist) + " "
self.print_metadata(metadata, develop, active, add_column_text)
else:
print str(dist) + " has no metadata"
results = True
if not results and self.project_name:
if self.version:
pkg_spec = "%s==%s" % (self.project_name, self.version)
else:
pkg_spec = "%s" % self.project_name
if show == "all":
self.logger.error("There are no versions of %s installed." \
% pkg_spec)
else:
self.logger.error("There are no %s versions of %s installed." \
% \
(show, pkg_spec))
return 2
elif show == "all" and results and self.options.fields:
print "Versions with '*' are non-active."
print "Versions with '!' are deployed in development mode."
def print_metadata(self, metadata, develop, active, installed_by):
"""
Print out formatted metadata
@param metadata: package's metadata
@type metadata: pkg_resources Distribution obj
@param develop: path to pkg if its deployed in development mode
@type develop: string
@param active: show if package is activated or not
@type active: boolean
@param installed_by: Shows if pkg was installed by a package manager other
than setuptools
@type installed_by: string
@returns: None
"""
show_metadata = self.options.metadata
if self.options.fields:
fields = self.options.fields.split(',')
fields = map(str.strip, fields)
else:
fields = []
version = metadata['Version']
#When showing all packages, note which are not active:
if active:
if fields:
active_status = ""
else:
active_status = "active"
else:
if fields:
active_status = "*"
else:
active_status = "non-active"
if develop:
if fields:
development_status = "! (%s)" % develop
else:
development_status = "development (%s)" % develop
else:
development_status = installed_by
status = "%s %s" % (active_status, development_status)
if fields:
print '%s (%s)%s %s' % (metadata['Name'], version, active_status,
development_status)
else:
# Need intelligent justification
print metadata['Name'].ljust(15) + " - " + version.ljust(12) + \
" - " + status
if fields:
#Only show specific fields, using case-insensitive search
fields = map(str.lower, fields)
for field in metadata.keys():
if field.lower() in fields:
print ' %s: %s' % (field, metadata[field])
print
elif show_metadata:
#Print all available metadata fields
for field in metadata.keys():
if field != 'Name' and field != 'Summary':
print ' %s: %s' % (field, metadata[field])
def show_deps(self):
"""
Show dependencies for package(s)
@returns: 0 - sucess 1 - No dependency info supplied
"""
pkgs = pkg_resources.Environment()
for pkg in pkgs[self.project_name]:
if not self.version:
print pkg.project_name, pkg.version
i = len(pkg._dep_map.values()[0])
if i:
while i:
if not self.version or self.version and \
pkg.version == self.version:
if self.version and i == len(pkg._dep_map.values()[0]):
print pkg.project_name, pkg.version
print " " + str(pkg._dep_map.values()[0][i - 1])
i -= 1
else:
self.logger.info(\
"No dependency information was supplied with the package.")
return 1
return 0
def show_pypi_changelog(self):
"""
Show detailed PyPI ChangeLog for the last `hours`
@returns: 0 = sucess or 1 if failed to retrieve from XML-RPC server
"""
hours = self.options.show_pypi_changelog
if not hours.isdigit():
self.logger.error("Error: You must supply an integer.")
return 1
try:
changelog = self.pypi.changelog(int(hours))
except XMLRPCFault, err_msg:
self.logger.error(err_msg)
self.logger.error("ERROR: Couldn't retrieve changelog.")
return 1
last_pkg = ''
for entry in changelog:
pkg = entry[0]
if pkg != last_pkg:
print "%s %s\n\t%s" % (entry[0], entry[1], entry[3])
last_pkg = pkg
else:
print "\t%s" % entry[3]
return 0
failed_msgs.append("%s %s" % (pkg_name, versions))
print "Testing %s... failed" % elem.text
print "%s tests failed." % failed
for msg in failed_msgs:
print "\t%s" % msg
print "%s tests skipped." % skipped
for msg in skipped_msgs:
print "\t%s" % msg
def test_cli(pypi_xml):
"""Test the command-line tool"""
print "Testing CLI"
for event, elem in iterparse(pypi_xml):
if elem.tag == "title":
if not elem.text.startswith('Cheese Shop recent updates'):
print "Testing %s..." % elem.text
pkg_name, ver = get_pkg_ver(elem.text)
if " " in pkg_name:
print "Space in package name, skipping: %s" % pkg_name
else:
os.system("yolk -V '%s'" % pkg_name)
os.system("yolk -D %s==%s" % (pkg_name, ver))
elem.clear()
test_cli(urllib.urlopen(PYPI_URL))
PyPI = CheeseShop()
test_api(urllib.urlopen(PYPI_URL))
class Yolk(object):
"""
Main class for yolk
"""
def __init__(self):
#PyPI project name with proper case
self.project_name = ""
#PyPI project version
self.version = ""
#List of all versions not hidden on PyPI
self.all_versions = []
self.pkg_spec = []
self.options = None
self.logger = logging.getLogger("yolk")
#Squelch output from setuptools
#Add future offenders to this list.
shut_up = ['distutils.log']
sys.stdout = StdOut(sys.stdout, shut_up)
sys.stderr = StdOut(sys.stderr, shut_up)
self.pypi = None
def get_plugin(self, method):
"""
Return plugin object if CLI option is activated and method exists
@param method: name of plugin's method we're calling
@type method: string
@returns: list of plugins with `method`
"""
all_plugins = []
for entry_point in pkg_resources.iter_entry_points('yolk.plugins'):
plugin_obj = entry_point.load()
plugin = plugin_obj()
plugin.configure(self.options, None)
if plugin.enabled:
if not hasattr(plugin, method):
self.logger.warn("Error: plugin has no method: %s" % method)
plugin = None
else:
all_plugins.append(plugin)
return all_plugins
def set_log_level(self):
"""
Set log level according to command-line options
@returns: logger object
"""
if self.options.debug:
self.logger.setLevel(logging.DEBUG)
elif self.options.quiet:
self.logger.setLevel(logging.ERROR)
else:
self.logger.setLevel(logging.INFO)
self.logger.addHandler(logging.StreamHandler())
return self.logger
def run(self):
"""
Perform actions based on CLI options
@returns: status code
"""
opt_parser = setup_opt_parser()
(self.options, remaining_args) = opt_parser.parse_args()
logger = self.set_log_level()
pkg_spec = validate_pypi_opts(opt_parser)
if not pkg_spec:
pkg_spec = remaining_args
self.pkg_spec = pkg_spec
if not self.options.pypi_search and (len(sys.argv) == 1 or\
len(remaining_args) > 2):
opt_parser.print_help()
return 2
#Options that depend on querying installed packages, not PyPI.
#We find the proper case for package names if they are installed,
#otherwise PyPI returns the correct case.
if self.options.show_deps or self.options.show_all or \
self.options.show_active or self.options.show_non_active or \
(self.options.show_updates and pkg_spec):
want_installed = True
else:
want_installed = False
#show_updates may or may not have a pkg_spec
if not want_installed or self.options.show_updates:
self.pypi = CheeseShop(self.options.debug)
#XXX: We should return 2 here if we couldn't create xmlrpc server
if pkg_spec:
(self.project_name, self.version, self.all_versions) = \
self.parse_pkg_ver(want_installed)
if want_installed and not self.project_name:
logger.error("%s is not installed." % pkg_spec[0])
return 1
#I could prefix all these with 'cmd_' and the methods also
#and then iterate over the `options` dictionary keys...
commands = ['show_deps', 'query_metadata_pypi', 'fetch',
'versions_available', 'show_updates', 'browse_website',
'show_download_links', 'pypi_search', 'show_pypi_changelog',
'show_pypi_releases', 'yolk_version', 'show_all',
'show_active', 'show_non_active', 'show_entry_map',
'show_entry_points']
#Run first command it finds, and only the first command, then return
#XXX: Check if more than one command was set in options and give error?
for action in commands:
if getattr(self.options, action):
return getattr(self, action)()
opt_parser.print_help()
def show_active(self):
"""
Show installed active packages
"""
return self.show_distributions("active")
def show_non_active(self):
"""
Show installed non-active packages
"""
return self.show_distributions("nonactive")
def show_all(self):
"""
Show all installed packages
"""
return self.show_distributions("all")
def show_updates(self):
"""
Check installed packages for available updates on PyPI
@param project_name: optional package name to check; checks every
installed pacakge if none specified
@type project_name: string
@returns: None
"""
dists = Distributions()
if self.project_name:
#Check for a single package
pkg_list = [self.project_name]
else:
#Check for every installed package
pkg_list = get_pkglist()
found = None
for pkg in pkg_list:
for (dist, active) in dists.get_distributions("all", pkg,
dists.get_highest_installed(pkg)):
(project_name, versions) = \
self.pypi.query_versions_pypi(dist.project_name)
if versions:
#PyPI returns them in chronological order,
#but who knows if its guaranteed in the API?
#Make sure we grab the highest version:
newest = get_highest_version(versions)
if newest != dist.version:
#We may have newer than what PyPI knows about
if pkg_resources.parse_version(dist.version) < \
pkg_resources.parse_version(newest):
found = True
print " %s %s (%s)" % (project_name, dist.version,
newest)
if not found and self.project_name:
self.logger.info("You have the latest version installed.")
elif not found:
self.logger.info("No newer packages found at The Cheese Shop")
return 0
def show_distributions(self, show):
"""
Show list of installed activated OR non-activated packages
@param show: type of pkgs to show (all, active or nonactive)
@type show: string
@returns: None or 2 if error
"""
show_metadata = self.options.metadata
#Search for any plugins with active CLI options with add_column() method
plugins = self.get_plugin("add_column")
#Some locations show false positive for 'development' packages:
ignores = ["/UNIONFS", "/KNOPPIX.IMG"]
#Check if we're in a workingenv
#See http://cheeseshop.python.org/pypi/workingenv.py
workingenv = os.environ.get('WORKING_ENV')
if workingenv:
ignores.append(workingenv)
dists = Distributions()
results = None
for (dist, active) in dists.get_distributions(show, self.project_name,
self.version):
metadata = get_metadata(dist)
for prefix in ignores:
if dist.location.startswith(prefix):
dist.location = dist.location.replace(prefix, "")
#Case-insensitve search because of Windows
if dist.location.lower().startswith(get_python_lib().lower()):
develop = ""
else:
develop = dist.location
if metadata:
add_column_text = ""
for my_plugin in plugins:
#See if package is 'owned' by a package manager such as
#portage, apt, rpm etc.
#add_column_text += my_plugin.add_column(filename) + " "
add_column_text += my_plugin.add_column(dist) + " "
self.print_metadata(metadata, develop, active, add_column_text)
else:
print str(dist) + " has no metadata"
results = True
if not results and self.project_name:
if self.version:
pkg_spec = "%s==%s" % (self.project_name, self.version)
else:
pkg_spec = "%s" % self.project_name
if show == "all":
self.logger.error("There are no versions of %s installed." \
% pkg_spec)
else:
self.logger.error("There are no %s versions of %s installed." \
% \
(show, pkg_spec))
return 2
elif show == "all" and results and self.options.fields:
print "Versions with '*' are non-active."
print "Versions with '!' are deployed in development mode."
def print_metadata(self, metadata, develop, active, installed_by):
"""
Print out formatted metadata
@param metadata: package's metadata
@type metadata: pkg_resources Distribution obj
@param develop: path to pkg if its deployed in development mode
@type develop: string
@param active: show if package is activated or not
@type active: boolean
@param installed_by: Shows if pkg was installed by a package manager other
than setuptools
@type installed_by: string
@returns: None
"""
show_metadata = self.options.metadata
if self.options.fields:
fields = self.options.fields.split(',')
fields = map(str.strip, fields)
else:
fields = []
version = metadata['Version']
#When showing all packages, note which are not active:
if active:
if fields:
active_status = ""
else:
active_status = "active"
else:
if fields:
active_status = "*"
else:
active_status = "non-active"
if develop:
if fields:
development_status = "! (%s)" % develop
else:
development_status = "development (%s)" % develop
else:
development_status = installed_by
status = "%s %s" % (active_status, development_status)
if fields:
print '%s (%s)%s %s' % (metadata['Name'], version, active_status,
development_status)
else:
# Need intelligent justification
print metadata['Name'].ljust(15) + " - " + version.ljust(12) + \
" - " + status
if fields:
#Only show specific fields, using case-insensitive search
fields = map(str.lower, fields)
for field in metadata.keys():
if field.lower() in fields:
print ' %s: %s' % (field, metadata[field])
print
elif show_metadata:
#Print all available metadata fields
for field in metadata.keys():
if field != 'Name' and field != 'Summary':
print ' %s: %s' % (field, metadata[field])
def show_deps(self):
"""
Show dependencies for package(s)
@returns: 0 - sucess 1 - No dependency info supplied
"""
pkgs = pkg_resources.Environment()
for pkg in pkgs[self.project_name]:
if not self.version:
print pkg.project_name, pkg.version
i = len(pkg._dep_map.values()[0])
if i:
while i:
if not self.version or self.version and \
pkg.version == self.version:
if self.version and i == len(pkg._dep_map.values()[0]):
print pkg.project_name, pkg.version
print " " + str(pkg._dep_map.values()[0][i - 1])
i -= 1
else:
self.logger.info(\
"No dependency information was supplied with the package.")
return 1
return 0
def show_pypi_changelog(self):
"""
Show detailed PyPI ChangeLog for the last `hours`
@returns: 0 = sucess or 1 if failed to retrieve from XML-RPC server
"""
hours = self.options.show_pypi_changelog
if not hours.isdigit():
self.logger.error("Error: You must supply an integer.")
return 1
try:
changelog = self.pypi.changelog(int(hours))
except XMLRPCFault, err_msg:
self.logger.error(err_msg)
self.logger.error("ERROR: Couldn't retrieve changelog.")
return 1
last_pkg = ''
for entry in changelog:
pkg = entry[0]
if pkg != last_pkg:
print "%s %s\n\t%s" % (entry[0], entry[1], entry[3])
last_pkg = pkg
else:
print "\t%s" % entry[3]
return 0
ARG_PARSER.add_argument(
"-e", "--extra", help="extra extends file")
ARG_PARSER.add_argument(
"--no-venv", action="store_true", help="no virtualenv")
ARG_PARSER.add_argument(
"--no-buildout", action="store_true", help="no pip install zc.buildout")
ARG_PARSER.add_argument(
"--unified-only", action="store_true", help="get unified cache & quit")
BUILDOUT_CFG = """\
[buildout]
extends =
%s
"""
CFG_PARSER = configparser.SafeConfigParser()
PYPI = CheeseShop()
EXTENDS = "https://raw.github.com/plock/pins/master/plone-4-3"
UNIFIEDINSTALLER_DIR = "Plone-4.3.3-UnifiedInstaller"
UNIFIEDINSTALLER_URL = "https://launchpad.net/plone/4.3/4.3.3/+download/"
UNIFIEDINSTALLER_URL += "Plone-4.3.3-UnifiedInstaller.tgz"
SEARCH_OPER = 'AND'
SEARCH_SPEC = {'description': 'plone', 'keyword': 'plone', 'summary': 'plone'}
def run(self):
"""Perform actions based on CLI options.
@returns: status code
"""
parser = setup_parser()
try:
import argcomplete
argcomplete.autocomplete(parser)
except ImportError:
pass
self.options = parser.parse_args()
pkg_spec = validate_pypi_opts(parser)
if not pkg_spec:
pkg_spec = self.options.pkg_spec
self.pkg_spec = pkg_spec
if self.options.fields:
self.options.fields = [s.strip().lower()
for s in self.options.fields.split(',')]
else:
self.options.fields = []
if (
not self.options.pypi_search and
len(sys.argv) == 1
):
parser.print_help()
return 2
# Options that depend on querying installed packages, not PyPI.
# We find the proper case for package names if they are installed,
# otherwise PyPI returns the correct case.
if (
self.options.show_deps or
self.options.show_all or
self.options.show_active or
self.options.show_non_active or
(self.options.show_updates and pkg_spec) or
self.options.upgrade
):
want_installed = True
else:
want_installed = False
# show_updates may or may not have a pkg_spec
if (
not want_installed or
self.options.show_updates or
self.options.upgrade
):
self.pypi = CheeseShop(self.options.debug)
# XXX: We should return 2 here if we couldn't create xmlrpc server
if pkg_spec:
(self.project_name,
self.version,
self.all_versions) = self.parse_pkg_ver(want_installed)
if want_installed and not self.project_name:
print('{} is not installed'.format(pkg_spec),
file=sys.stderr)
return 1
# I could prefix all these with 'cmd_' and the methods also
# and then iterate over the `options` dictionary keys...
commands = ['show_deps', 'query_metadata_pypi', 'fetch',
'versions_available', 'show_updates', 'upgrade',
'browse_website',
'show_download_links', 'pypi_search',
'show_pypi_changelog', 'show_pypi_releases',
'yolk_version', 'show_all',
'show_active', 'show_non_active', 'show_entry_map',
'show_entry_points']
# Run first command it finds, and only the first command, then return
# XXX: Check if more than one command was set in options and give
# error?
for action in commands:
if getattr(self.options, action):
return getattr(self, action)()
parser.print_help()
argparser.add_argument(
"-u", "--use", action="store_true",
help="use existing buildout.cfg")
argparser.add_argument(
"-w", "--write", action="store_true", dest="write_only",
help="write buildout.cfg and exit")
argparser.add_argument(
"--no-cache", action="store_true", dest="no_unified",
help="do not download unified installer cache")
argparser.add_argument(
"--no-buildout", action="store_true", help="do not install buildout")
argparser.add_argument(
"--no-virtualenv", action="store_true", help="do not create virtualenv")
argparser.add_argument(
"--cache", action="store_true", dest="unified_only",
help="download unified installer cache and exit")
cfgparser = configparser.SafeConfigParser()
pypi = CheeseShop()
query = {
'description': 'plone',
'keyword': 'plone',
'summary': 'plone'
}
def check_latest():
"""
Problems:
* we use several third-party packages that are not included in a
normal plone release. We want to keep those up to date
* we keep all packages pinned to avoid getting unstable releases
and we need to know when newer releases are available
The tool should do the following:
* watch the egg repos we are using (pypi, eea eggrepo) and report
when new versions of packages are available, especially for the
third-party packages
* parse the buildout files (including on-the-web cfgs) and report
of conflicts between versions
* this should be implemented as a buildout tool
* this tool should be run in automatic by Jenkins and report when
new versions are available
"""
# we use the .installed.cfg file to try to find the longest line
# of __buildout_signature__, which contains the eggs that we need.
# this is (maybe) highly specific to EEA.
v = {}
with open('.installed.cfg', 'r') as f:
lines = f.readlines()
longest = ""
for line in lines:
if line.startswith("__buildout_signature__") \
and len(line) > len(longest):
longest = line
eggs = longest.split('=')[1].strip().split(' ')
for egg in eggs:
spec = egg.split('-')
name = spec[0]
version = spec[1]
v[name.strip()] = version.strip()
skipped = []
if os.path.exists('.skipped_packages'):
with open(".skipped_packages") as f:
skipped = [x.strip() for x in f.readlines() if x.strip()]
picked_versions = v
# repos = [EEAEggRepo(), CheeseShop(), ] #order is important
repos = [
CheeseShop(),
] # order is important
flag = 0
report = []
for name, v in picked_versions.items():
if name in skipped:
continue
print "Checking new version for", name
for pypi in repos:
new_name, versions = pypi.query_versions_pypi(name)
if versions:
break
if versions:
latest = versions[0]
else:
print "Could not find any version for this package"
continue
if latest != v:
print "Package %s - %s" % (name, v), " has a new version: ", latest
report.append((name, v, latest))
flag = 1
report.sort(lambda x, y: cmp(x[0], y[0]))
print
print "New versions report"
print "==================="
for l in report:
name, old, new = l
space1 = (40 - len(name)) * ' '
space2 = (20 - len(old)) * ' '
print name + space1 + old + space2 + new
sys.exit(flag)
def get_lastest_version_number(package_name):
pkg, all_versions = CheeseShop().query_versions_pypi(package_name)
if len(all_versions):
return all_versions[0]
return None
class Yolk(object):
"""Main class for yolk."""
def __init__(self):
# PyPI project name with proper case
self.project_name = ''
# PyPI project version
self.version = ''
# List of all versions not hidden on PyPI
self.all_versions = []
self.pkg_spec = None
self.options = None
# Squelch output from setuptools
# Add future offenders to this list.
shut_up = ['distutils.log']
sys.stdout = StdOut(sys.stdout, shut_up)
sys.stderr = StdOut(sys.stderr, shut_up)
self.pypi = None
def get_plugin(self, method):
"""Return plugin object if CLI option is activated and method exists.
@param method: name of plugin's method we're calling
@type method: string
@returns: list of plugins with `method`
"""
all_plugins = []
for entry_point in pkg_resources.iter_entry_points('yolk.plugins'):
plugin_obj = entry_point.load()
plugin = plugin_obj()
plugin.configure(self.options, None)
if plugin.enabled:
if not hasattr(plugin, method):
plugin = None
else:
all_plugins.append(plugin)
return all_plugins
def run(self):
"""Perform actions based on CLI options.
@returns: status code
"""
parser = setup_parser()
try:
import argcomplete
argcomplete.autocomplete(parser)
except ImportError:
pass
self.options = parser.parse_args()
pkg_spec = validate_pypi_opts(parser)
if not pkg_spec:
pkg_spec = self.options.pkg_spec
self.pkg_spec = pkg_spec
if self.options.fields:
self.options.fields = [
s.strip().lower() for s in self.options.fields.split(',')
]
else:
self.options.fields = []
if (not self.options.pypi_search and len(sys.argv) == 1):
parser.print_help()
return 2
# Options that depend on querying installed packages, not PyPI.
# We find the proper case for package names if they are installed,
# otherwise PyPI returns the correct case.
if (self.options.show_deps or self.options.show_all
or self.options.show_active or self.options.show_non_active
or (self.options.show_updates and pkg_spec)
or self.options.upgrade):
want_installed = True
else:
want_installed = False
# show_updates may or may not have a pkg_spec
if (not want_installed or self.options.show_updates
or self.options.upgrade):
self.pypi = CheeseShop(self.options.debug)
# XXX: We should return 2 here if we couldn't create xmlrpc server
if pkg_spec:
(self.project_name, self.version,
self.all_versions) = self.parse_pkg_ver(want_installed)
if want_installed and not self.project_name:
print(u'{} is not installed'.format(pkg_spec), file=sys.stderr)
return 1
# I could prefix all these with 'cmd_' and the methods also
# and then iterate over the `options` dictionary keys...
commands = [
'show_deps', 'query_metadata_pypi', 'fetch', 'versions_available',
'show_updates', 'upgrade', 'browse_website', 'show_download_links',
'pypi_search', 'show_pypi_changelog', 'show_pypi_releases',
'yolk_version', 'show_all', 'show_active', 'show_non_active',
'show_entry_map', 'show_entry_points'
]
# Run first command it finds, and only the first command, then return
# XXX: Check if more than one command was set in options and give
# error?
for action in commands:
if getattr(self.options, action):
return getattr(self, action)()
parser.print_help()
def show_active(self):
"""Show installed active packages."""
return self.show_distributions('active')
def show_non_active(self):
"""Show installed non-active packages."""
return self.show_distributions('nonactive')
def show_all(self):
"""Show all installed packages."""
return self.show_distributions('all')
def show_updates(self):
"""Check installed packages for available updates on PyPI.
@param project_name: optional package name to check; checks every
installed package if none specified
@type project_name: string
@returns: None
"""
if self.project_name:
pkg_list = [self.project_name]
else:
pkg_list = get_pkglist()
for (project_name, version,
newest) in _updates(pkg_list,
self.pypi,
user_installs_only=self.options.user):
print(u'{} {} ({})'.format(project_name, version, newest))
return 0
def upgrade(self):
"""Check installed packages for available updates on PyPI and upgrade.
@param project_name: optional package name to check; checks every
installed package if none specified
@type project_name: string
@returns: None
"""
if self.project_name:
pkg_list = [self.project_name]
else:
pkg_list = get_pkglist()
names = [
values[0] for values in _updates(
pkg_list, self.pypi, user_installs_only=self.options.user)
]
if names:
subprocess.call(
[sys.executable, '-m', 'pip', 'install', '--upgrade'] +
(['--user'] if self.options.user else []) + names)
return 0
def show_distributions(self, show):
"""Show list of installed activated OR non-activated packages.
@param show: type of pkgs to show (all, active or nonactive)
@type show: string
@returns: None or 2 if error
"""
# Search for any plugins with active CLI options with add_column()
# method.
plugins = self.get_plugin('add_column')
# Some locations show false positive for 'development' packages:
ignores = ['/UNIONFS', '/KNOPPIX.IMG']
# See http://cheeseshop.python.org/pypi/workingenv.py for details.
workingenv = os.environ.get('WORKING_ENV')
if workingenv:
ignores.append(workingenv)
results = None
for (dist,
active) in yolklib.get_distributions(show, self.project_name,
self.version):
metadata = get_metadata(dist)
for prefix in ignores:
if dist.location.startswith(prefix):
dist.location = dist.location.replace(prefix, '')
# Case-insensitive search because of Windows.
if dist.location.lower().startswith(get_python_lib().lower()):
develop = ''
else:
develop = dist.location
if metadata:
add_column_text = ''
for my_plugin in plugins:
# See if package is 'owned' by a package manager such as
# portage, apt, rpm etc.
add_column_text += my_plugin.add_column(dist) + ' '
self.print_metadata(metadata, develop, active, add_column_text)
else:
print(str(dist) + ' has no metadata')
results = True
if not results and self.project_name:
if self.version:
pkg_spec = '{}=={}'.format(self.project_name, self.version)
else:
pkg_spec = self.project_name
if show == 'all':
print(
u'There are no versions of {} installed'.format(pkg_spec),
file=sys.stderr)
else:
print(u'There are no {} versions of {} installed'.format(
show, pkg_spec),
file=sys.stderr)
return 2
elif show == 'all' and results and self.options.fields:
print("Versions with '*' are non-active.")
print("Versions with '!' are deployed in development mode.")
def print_metadata(self, metadata, develop, active, installed_by):
"""Print out formatted metadata.
@param metadata: package's metadata
@type metadata: pkg_resources Distribution obj
@param develop: path to pkg if its deployed in development mode
@type develop: string
@param active: show if package is activated or not
@type active: boolean
@param installed_by: Shows if pkg was installed by a package manager
other than setuptools
@type installed_by: string
@returns: None
"""
show_metadata = self.options.metadata
version = metadata['Version']
# When showing all packages, note which are not active:
if active:
if self.options.fields:
active_status = ''
else:
active_status = 'active'
else:
if self.options.fields:
active_status = '*'
else:
active_status = 'non-active'
if develop:
if self.options.fields:
development_status = '! ({})'.format(develop)
else:
development_status = 'development ({})'.format(develop)
else:
development_status = installed_by
status = '{} {}'.format(active_status, development_status)
if self.options.fields:
print('{} ({}){} {}'.format(metadata['Name'], version,
active_status, development_status))
else:
# Need intelligent justification.
print(metadata['Name'].ljust(15) + ' - ' + version.ljust(12) +
' - ' + status)
if self.options.fields:
for field in metadata.keys():
if field.lower() in self.options.fields:
print(u' {}: {}'.format(field, metadata[field]))
print()
elif show_metadata:
for field in metadata.keys():
if field != 'Name' and field != 'Summary':
print(u' {}: {}'.format(field, metadata[field]))
def show_deps(self):
"""Show dependencies for package(s)
@returns: 0 - success 1 - No dependency info supplied
"""
pkgs = pkg_resources.Environment()
for pkg in pkgs[self.project_name]:
if not self.version:
print(pkg.project_name, pkg.version)
i = len(list(pkg._dep_map.values())[0])
if i:
while i:
if (not self.version
or self.version and pkg.version == self.version):
if self.version and i == len(
list(pkg._dep_map.values())[0]):
print(pkg.project_name, pkg.version)
print(u' ' +
str(list(pkg._dep_map.values())[0][i - 1]))
i -= 1
else:
return 1
return 0
def show_pypi_changelog(self):
"""Show detailed PyPI ChangeLog for the last `hours`
@returns: 0 = success or 1 if failed to retrieve from XML-RPC server
"""
hours = self.options.show_pypi_changelog
if not hours.isdigit():
print('You must supply an integer', file=sys.stderr)
return 1
try:
changelog = self.pypi.changelog(int(hours))
except XMLRPCFault as err_msg:
print(err_msg, file=sys.stderr)
print("Couldn't retrieve changelog", file=sys.stderr)
return 1
last_pkg = ''
for entry in changelog:
pkg = entry[0]
if pkg != last_pkg:
print(u'{} {}\n\t{}'.format(entry[0], entry[1], entry[3]))
last_pkg = pkg
else:
print(u'\t{}'.format(entry[3]))
return 0
def show_pypi_releases(self):
"""Show PyPI releases for the last number of `hours`
@returns: 0 = success or 1 if failed to retrieve from XML-RPC server
"""
try:
hours = int(self.options.show_pypi_releases)
except ValueError:
print('You must supply an integer', file=sys.stderr)
return 1
try:
latest_releases = self.pypi.updated_releases(hours)
except XMLRPCFault as err_msg:
print(err_msg, file=sys.stderr)
print("Couldn't retrieve latest releases.", file=sys.stderr)
return 1
for release in latest_releases:
print(u'{} {}'.format(release[0], release[1]))
return 0
def show_download_links(self):
"""Query PyPI for pkg download URI for a packge.
@returns: 0
"""
# In case they specify version as 'dev' instead of using -T svn,
# don't show three svn URI's
if self.options.file_type == 'all' and self.version == 'dev':
self.options.file_type = 'svn'
if self.options.file_type == 'svn':
version = 'dev'
else:
if self.version:
version = self.version
else:
version = self.all_versions[0]
if self.options.file_type == 'all':
# Search for source, egg, and svn.
self.print_download_uri(version, True)
self.print_download_uri(version, False)
self.print_download_uri('dev', True)
else:
if self.options.file_type == 'source':
source = True
else:
source = False
self.print_download_uri(version, source)
return 0
def print_download_uri(self, version, source):
"""@param version: version number or 'dev' for svn.
@type version: string
@param source: download source or egg
@type source: boolean
@returns: None
"""
if version == 'dev':
source = True
# Use setuptools monkey-patch to grab url.
url = get_download_uri(self.project_name, version, source,
self.options.pypi_index)
if url:
print(u'{}'.format(url))
def fetch(self):
"""Download a package.
@returns: 0 = success or 1 if failed download
"""
source = True
directory = '.'
if self.options.file_type == 'svn':
svn_uri = get_download_uri(self.project_name, 'dev', True)
if svn_uri:
directory = self.project_name + '_svn'
return self.fetch_svn(svn_uri, directory)
else:
print('No subversion repository found for {}'.format(
self.project_name),
file=sys.stderr)
return 1
elif self.options.file_type == 'source':
source = True
elif self.options.file_type == 'egg':
source = False
uri = get_download_uri(self.project_name, self.version, source)
if uri:
return self.fetch_uri(directory, uri)
else:
print(u'No {} URI found for package: {}'.format(
self.options.file_type, self.project_name))
return 1
def fetch_uri(self, directory, uri):
"""Use ``urllib.urlretrieve`` to download package to file in sandbox
dir.
@param directory: directory to download to
@type directory: string
@param uri: uri to download
@type uri: string
@returns: 0 = success or 1 for failed download
"""
filename = os.path.basename(urlparse(uri)[2])
if os.path.exists(filename):
print(u'File exists: ' + filename, file=sys.stderr)
return 1
try:
downloaded_filename, headers = urlretrieve(uri, filename)
except IOError as err_msg:
print('Error downloading package {} from URL {}'.format(
filename, uri),
file=sys.stderr)
print(str(err_msg), file=sys.stderr)
return 1
if 'text/html' in headers:
dfile = open(downloaded_filename)
if re.search('404 Not Found', ''.join(dfile.readlines())):
dfile.close()
print("'404 Not Found' error", file=sys.stderr)
return 1
dfile.close()
return 0
def fetch_svn(self, svn_uri, directory):
"""Fetch subversion repository.
@param svn_uri: subversion repository uri to check out
@type svn_uri: string
@param directory: directory to download to
@type directory: string
"""
if not command_successful(['svn', '--version']):
raise YolkException('Do you have subversion installed?')
if os.path.exists(directory):
raise YolkException(
'Checkout directory exists - {}'.format(directory))
try:
os.mkdir(directory)
except OSError as err_msg:
raise YolkException('' + str(err_msg))
cwd = os.path.realpath(os.curdir)
os.chdir(directory)
status, _ = run_command(['svn', 'checkout', svn_uri])
os.chdir(cwd)
def browse_website(self, browser=None):
"""Launch web browser at project's homepage.
@param browser: name of web browser to use
@type browser: string
@returns: 0 if homepage found, 1 if no homepage found
"""
if len(self.all_versions):
metadata = self.pypi.release_data(self.project_name,
self.all_versions[0])
if 'home_page' in metadata:
if browser == 'konqueror':
browser = webbrowser.Konqueror()
else:
browser = webbrowser.get()
browser.open(metadata['home_page'], 2)
return 0
print('No homepage URL found', file=sys.stderr)
return 1
def query_metadata_pypi(self):
"""Show pkg metadata queried from PyPI.
@returns: 0
"""
if self.version and self.version in self.all_versions:
metadata = self.pypi.release_data(self.project_name, self.version)
else:
# Give highest version
metadata = self.pypi.release_data(self.project_name,
self.all_versions[0])
if metadata:
if len(self.options.fields) == 1:
try:
print(metadata[self.options.fields[0]])
except KeyError:
pass
else:
for key in metadata.keys():
if (not self.options.fields
or (self.options.fields
and key.lower() in self.options.fields)):
print(u'{}: {}'.format(key, metadata[key]))
return 0
def versions_available(self):
"""Query PyPI for a particular version or all versions of a package.
@returns: 0 if version(s) found or 1 if none found
"""
if self.all_versions and self.version in self.all_versions:
print_pkg_versions(self.project_name, [self.version])
elif not self.version and self.all_versions:
print_pkg_versions(self.project_name, self.all_versions)
else:
if self.version:
print('No package found for version {}'.format(self.version),
file=sys.stderr)
else:
print('No package found for {}'.format(self.project_name),
file=sys.stderr)
return 1
return 0
def parse_search_spec(self, spec):
"""Parse search args and return spec dict for PyPI.
* Owwww, my eyes!. Re-write this.
@param spec: Cheese Shop package search spec
e.g.
name=Cheetah
license=ZPL
license=ZPL AND name=Cheetah
@type spec: string
@returns: tuple with spec and operator
"""
usage = """You can search PyPI by the following:
name
version
author
author_email
maintainer
maintainer_email
home_page
license
summary
description
keywords
platform
download_url
e.g. yolk -S name=Cheetah
yolk -S name=yolk AND license=PSF
"""
if not spec:
print(usage, file=sys.stderr)
return (None, None)
try:
spec = (' ').join(spec)
operator = 'AND'
first = second = ''
if ' AND ' in spec:
(first, second) = spec.split('AND')
elif ' OR ' in spec:
(first, second) = spec.split('OR')
operator = 'OR'
else:
first = spec
(key1, term1) = first.split('=')
key1 = key1.strip()
if second:
(key2, term2) = second.split('=')
key2 = key2.strip()
spec = {}
spec[key1] = term1
if second:
spec[key2] = term2
except:
print(usage, file=sys.stderr)
spec = operator = None
return (spec, operator)
def pypi_search(self):
"""Search PyPI by metadata keyword e.g.
yolk -S name=yolk AND license=GPL
@param spec: Cheese Shop search spec
@type spec: list of strings
spec examples:
["name=yolk"]
["license=GPL"]
["name=yolk", "AND", "license=GPL"]
@returns: 0 on success or 1 if mal-formed search spec
"""
spec = self.pkg_spec
# Add remaining cli arguments to options.pypi_search.
search_arg = self.options.pypi_search
spec.insert(0, search_arg.strip())
(spec, operator) = self.parse_search_spec(spec)
if not spec:
return 1
for pkg in self.pypi.search(spec, operator):
if pkg['summary']:
summary = pkg['summary'].encode('utf-8')
else:
summary = ''
print("""{} ({}):
{}
""".format(pkg['name'].encode('utf-8'), pkg['version'], summary))
return 0
def show_entry_map(self):
"""Show entry map for a package.
@param dist: package
@param type: string
@returns: 0 for success or 1 if error
"""
pprinter = pprint.PrettyPrinter()
try:
entry_map = pkg_resources.get_entry_map(
self.options.show_entry_map)
if entry_map:
pprinter.pprint(entry_map)
except pkg_resources.DistributionNotFound:
print('Distribution not found: {}'.format(
self.options.show_entry_map),
file=sys.stderr)
return 1
return 0
def show_entry_points(self):
"""Show entry points for a module.
@returns: 0 for success or 1 if error
"""
found = False
for entry_point in pkg_resources.iter_entry_points(
self.options.show_entry_points):
found = True
try:
plugin = entry_point.load()
print(plugin.__module__)
print(u' {}'.format(entry_point))
if plugin.__doc__:
print(plugin.__doc__)
print()
except ImportError:
pass
if not found:
print('No entry points found for {}'.format(
self.options.show_entry_points),
file=sys.stderr)
return 1
return 0
def yolk_version(self):
"""Show yolk's version."""
print(u'yolk {}'.format(VERSION))
def parse_pkg_ver(self, want_installed):
"""Return tuple with project_name and version from CLI args If the user
gave the wrong case for the project name, this corrects it.
@param want_installed: whether package we want is installed or not
@type want_installed: boolean
@returns: tuple(project_name, version, all_versions)
"""
all_versions = []
arg_str = self.pkg_spec
if '==' not in arg_str:
# No version specified.
project_name = arg_str
version = None
else:
(project_name, version) = arg_str.split('==')
project_name = project_name.strip()
version = version.strip()
# Find proper case for package name.
if want_installed:
project_name = yolklib.case_sensitive_name(project_name)
else:
(project_name,
all_versions) = self.pypi.query_versions_pypi(project_name)
if not len(all_versions):
msg = "I'm afraid we have no '{}' at ".format(project_name)
msg += 'The Cheese Shop. A little Red Leicester, perhaps?'
print(msg, file=sys.stderr)
sys.exit(2)
return (project_name, version, all_versions)
