def read_pkg_requires(path):
"""Read a package requires.txt
"""
try:
data = open(os.path.join(path, 'requires.txt'), 'r')
except IOError:
return Requirements(), {}
lines = []
requires = []
extras = {}
current = None
for line in data.readlines():
line = line.strip()
if not line or line[0] in '#;':
continue
if line[0] == '[' and line[-1] == ']':
# New extra
if current is None:
requires = lines
else:
extras[current] = Requirements.parse(lines)
lines = []
current = line[1:-1]
else:
lines.append(line)
# Store last extra
if current is None:
requires = lines
else:
extras[current] = Requirements.parse(lines)
return Requirements.parse(requires), extras
def test_add_fail(self):
"""Test adding two incompatible set of requirements together
"""
reqs = Requirements.parse(
['zeam.form.base',
'zeam.test>=2.1'])
other_reqs = Requirements.parse(
['zeam.form.ztk[test] >=1.0b1',
'zeam.test <=1.0'])
self.assertRaises(IncompatibleVersion, operator.add, reqs, other_reqs)
def load(self):
egginfo = self.configuration['egginfo']
self.distribution.name = egginfo['name'].as_text()
self.distribution.version = Version.parse(egginfo['version'].as_text())
self.distribution.summary = egginfo.get('summary', '').as_text()
self.distribution.author = egginfo.get('author', '').as_text()
self.distribution.author_email = egginfo.get(
'author_email', '').as_text()
self.distribution.license = egginfo.get('license', '').as_text()
self.distribution.classifiers = egginfo.get('classifier', '').as_list()
self.distribution.format = None
self.distribution.pyversion = None
self.distribution.platform = None
self.distribution.requirements = Requirements.parse(
egginfo.get('requires', '').as_list())
self.distribution.extras = {}
# Source path of the extension
path = os.path.join(self.path, egginfo.get('source', '.').as_text())
if not os.path.isdir(path):
raise PackageError(path, 'Invalid source path "%s"' % path)
self.distribution.path = os.path.abspath(path)
self.distribution.package_path = self.path
# Entry points
self.distribution.entry_points = {}
entry_points = egginfo.get('entry_points', None)
if entry_points is not None:
for category_name in entry_points.as_list():
info = self.configuration['entry_points:' + category_name]
self.distribution.entry_points[category_name] = info.as_dict()
return self.distribution
def __init__(self, options, installed_options=None):
self.versions = dict(map(lambda v: (v.key, v),
map(KnownVersion, options.values())))
self.name = options.name.split(':', 1)[1]
self.used = set()
self.missing = Requirements()
self.picked = {}
self.options = options
self.installed_options = installed_options
self._uptodate = None
self._activated = False
def add_recipe_packages(self, names):
"""Install a list of packages required for the parts to
execute. Packages are enabled in the current Python
environment so that the parts can use them directly.
"""
# This must be used only to install recipe and recipe dependency.
requirements = Requirements.parse(names)
install_set = self._installer(
requirements,
strategy=self.strategy)
for requirement in requirements:
install_set.get(requirement.key).activate()
def __init__(self, options, working_set, directory=None):
__status__ = u"Configuring package installer."
self.interpretor = working_set.interpretor
self.working_set = working_set
self.kgs = options.utilities.kgs.get(options)
self.sources = options.utilities.sources
self.query = None
self._to_install = Requirements()
self._verify_being_installed = Requirements()
self._being_installed = Requirements()
self._lock = threading.RLock()
self._wait = threading.Condition(threading.RLock())
self._worker_count = options.get_with_default(
'install_workers', 'setup', '5').as_int()
self._options = options
self._first_done = False
self._error = None
if directory is None:
directory = options.get_with_default(
'lib_directory', 'setup').as_text()
self._directory = directory
def load(self, distribution, interpretor):
distribution.package_path = self.path
# Read extracted configuration
config = Configuration.read_lines(self.source.splitlines, self.path)
setuptool_config = config['setuptools']
distribution.version = Version.parse(
setuptool_config['version'].as_text())
# Look for requirements
if 'install_requires' in setuptool_config:
distribution.requirements = Requirements.parse(
setuptool_config['install_requires'].as_list())
if 'extras_require' in setuptool_config:
extra_config = config[setuptool_config['extras_require'].as_text()]
for extra, extra_requirements in extra_config.items():
distribution.extras[extra] = Requirements.parse(
extra_requirements.as_list())
# Look for source directory
if 'package_dir' in setuptool_config:
package_config = config[
setuptool_config['package_dir'].as_text()]
if '_' in package_config:
prefix = package_config['_'].as_text()
distribution.path = os.path.join(self.path, prefix)
if 'description' in setuptool_config:
distribution.description = setuptool_config['description'].as_text()
if 'license' in setuptool_config:
distribution.license = setuptool_config['license'].as_text()
if 'author' in setuptool_config:
distribution.author = setuptool_config['author'].as_text()
if 'autor_email' in setuptool_config:
distribution.author_email = \
setuptool_config['author_email'].as_text()
if 'ext_modules' in setuptool_config:
libraries = self.extensions(prefix, setuptool_config)
create_autotools(distribution, prefix, libraries)
distribution.extensions = libraries
return distribution
def test_add(self):
"""Test adding two sets of requirements together
"""
reqs = Requirements.parse(
['zeam.form.base',
'zeam.test>=2.1',
'zope.testing<=3.7dev'])
self.assertRaises(ValueError, operator.add, reqs, 42)
self.assertRaises(ValueError, operator.add, reqs, "zeam >= 1.1")
other_reqs = Requirements.parse(
['zeam.form.ztk[test] >=1.0b1',
'zeam.test <=4.0, !=3.0'])
result_reqs = reqs + other_reqs
self.assertEqual(
str(result_reqs).split('\n'),
['zeam.form.base',
'zeam.form.ztk[test]>=1.0b1',
'zeam.test>=2.1,!=3.0,<=4.0',
'zope.testing<=3.7dev'])
def test_parse(self):
"""Test requirements parsing and printing
"""
reqs = Requirements.parse([])
self.assertEqual(len(reqs), 0)
self.assertEqual(len(reqs.requirements), 0)
reqs = Requirements.parse('test.software')
self.assertEqual(len(reqs), 1)
self.assertEqual(len(reqs.requirements), 1)
self.assertEqual(str(reqs), 'test.software')
reqs = Requirements.parse(
['zeam.form.base',
'zeam.test>=2.1',
'zope.testing<=3.7dev'])
self.assertEqual(len(reqs), 3)
self.assertEqual(len(reqs.requirements), 3)
self.assertEqual(
str(reqs).split('\n'),
['zeam.form.base',
'zeam.test>=2.1',
'zope.testing<=3.7dev'])
def __init__(self, options, status):
super(Package, self).__init__(options, status)
self.isolation = options.get(
'isolation', 'on').as_bool()
self.directory = options.get(
'lib_directory',
'${setup:lib_directory}').as_directory()
self.bin_directory = options.get(
'bin_directory',
'${setup:bin_directory}').as_directory()
requirements = []
self.extra_sets = []
if 'packages' in options:
for requirement in options['packages'].as_list():
match = INSTALLED_SET.match(requirement)
if match:
self.extra_sets.append(match.group('name'))
else:
requirements.append(requirement)
else:
requirements = [get_package_name(options).as_text()]
self.requirements = Requirements.parse(requirements)
if self.extra_sets:
status.depends.update(self.extra_sets)
if self.requirements:
# Run the recipe is installer settings changed.
status.enable(is_installer_changed(options))
self.wanted_scripts = None
if 'scripts' in options:
self.wanted_scripts = options['scripts'].as_list()
self.extra_args = []
if 'extra_args' in options:
self.extra_args = options['extra_args'].as_list()
self.extra_paths = []
if 'extra_paths' in options:
self.extra_paths = options['extra_paths'].as_list()
self.working_set = None
class PackageInstaller(object):
"""Package installer: install new package in a working set from
sources.
"""
def __init__(self, options, working_set, directory=None):
__status__ = u"Configuring package installer."
self.interpretor = working_set.interpretor
self.working_set = working_set
self.kgs = options.utilities.kgs.get(options)
self.sources = options.utilities.sources
self.query = None
self._to_install = Requirements()
self._verify_being_installed = Requirements()
self._being_installed = Requirements()
self._lock = threading.RLock()
self._wait = threading.Condition(threading.RLock())
self._worker_count = options.get_with_default(
'install_workers', 'setup', '5').as_int()
self._options = options
self._first_done = False
self._error = None
if directory is None:
directory = options.get_with_default(
'lib_directory', 'setup').as_text()
self._directory = directory
def _wakeup_workers(self):
# Wake up some workers to work.
count_to_wake_up = max(
self._worker_count - 1, len(self._to_install))
self._wait.acquire()
for count in range(count_to_wake_up):
self._wait.notify()
self._wait.release()
def _verify_extra_install(self, requirement):
# Verify is some extra need installation
release = self.working_set[requirement]
for extra in requirement.extras:
if extra not in release.extras:
raise PackageError(
u'Require missing extra requirements "%s" in "%s"' % (
extra, release))
self._register_install(release.extras[extra])
def _register_install(self, requirements):
# Mark requirements to be installed.
for requirement in requirements:
if self.kgs is not None:
requirement = self.kgs.upgrade(requirement)
if requirement in self.working_set:
if requirement.extras:
self._verify_extra_install(requirement)
logger.debug(
u'Skip already installed dependency %s', requirement)
continue
if requirement in self._being_installed:
if requirement.extras:
self._verify_being_installed.append(requirement)
logger.debug(
u'Skip already being installed dependency %s',
requirement)
continue
logger.debug(
u'Need to install dependency %s', requirement)
self._to_install.append(requirement)
def wait_for_requirements(self):
"""Called by a worker to wait for requirement to arrive.
"""
logger.debug(u'Wait for dependencies')
self._wait.acquire()
self._wait.wait()
self._wait.release()
def mark_failed(self, error):
"""Called by a worked to report an error.
"""
self._lock.acquire()
try:
logger.debug(u'Failure')
self._error = error
logs.report(fatal=False)
self._wait.acquire()
self._wait.notifyAll()
self._wait.release()
finally:
self._lock.release()
def get_requirement(self):
"""Called by a worker to get a new requirement to install.
"""
self._lock.acquire()
try:
if self._error is not None:
return INSTALLATION_DONE
if not self._to_install:
if not self._being_installed:
if not self._first_done:
# Wake up other waiting worker
self._wait.acquire()
self._wait.notifyAll()
self._wait.release()
self._first_done = True
return INSTALLATION_DONE
return None
requirement = self._to_install.pop()
assert requirement not in self._being_installed
self._being_installed.append(requirement)
logger.info(u'Installing %s', requirement)
return requirement
finally:
self._lock.release()
def mark_installed(self, requirement, package):
"""Called by a worker to mark that the given requirement have
been installed, using the given package.
"""
self._lock.acquire()
try:
self.working_set.add(package)
if self.kgs is not None:
self.kgs.report_picked(requirement, package.version)
if requirement in self._verify_being_installed:
extra_requirement = self._verify_being_installed[requirement]
logger.debug(
u'Verify pending extra for %s', extra_requirement)
worker_need_wake_up = len(self._to_install) == 0
self._verify_extra_install(extra_requirement)
if worker_need_wake_up:
self._wakeup_workers()
self._verify_being_installed.remove(extra_requirement)
self._being_installed.remove(requirement)
logger.debug(u'Mark %s as installed', requirement)
finally:
self._lock.release()
def install_dependencies(self, requirements):
self._lock.acquire()
try:
worker_need_wake_up = len(self._to_install) == 0
self._register_install(requirements)
if worker_need_wake_up:
self._wakeup_workers()
finally:
self._lock.release()
def __call__(self, requirements, strategy=STRATEGY_UPDATE, directory=None):
"""Called by the user to trigger the installation of the given
list of requirements.
"""
__status__ = u"Installing %r." % (requirements)
if directory is None:
directory = self._directory
self._error = None
self._first_done = False
self._register_install(requirements)
if self._to_install:
self.query = self.sources(
self.interpretor,
os.path.abspath(directory))
workers = []
for count in range(self._worker_count):
worker = PackageInstallerWorker(self, count, strategy)
worker.start()
workers.append(worker)
for worker in workers:
worker.join()
if self._error is not None:
raise self._error
return self.working_set
class KnownGoodVersionSet(object):
"""Represent a Known Good Version set.
"""
def __init__(self, options, installed_options=None):
self.versions = dict(map(lambda v: (v.key, v),
map(KnownVersion, options.values())))
self.name = options.name.split(':', 1)[1]
self.used = set()
self.missing = Requirements()
self.picked = {}
self.options = options
self.installed_options = installed_options
self._uptodate = None
self._activated = False
def get(self, requirement, default=None):
if self._activated:
__status__ = u"Looking version for %s in Known Good Set %s." % (
str(requirement), self.name)
if requirement.key in self.versions:
self.used.add(requirement.key)
return Version.parse(self.versions[requirement.key].version)
return default
def is_activated(self):
return self._activated
def is_uptodate(self):
if self._uptodate is None:
if self.installed_options is None:
self._uptodate = True
else:
self._uptodate = (self.options == self.installed_options)
return self._uptodate
def activate(self):
# This means the KGS is activated in order to be used.
self._activated = True
def report_missing(self, requirement):
self.missing.append(requirement)
def report_picked(self, requirement, version):
picked = self.picked.setdefault(requirement, [])
picked.append(version)
def log_usage(self):
if self.missing:
for requirement in self.missing:
msg = u"Missing requirement for '%s' in '%s'"
args = (requirement, self.name)
picked = self.picked.get(requirement)
if picked:
msg += u", picked '%s'"
args += (','.join(map(str, picked)),)
msg += u"."
logger.warn(msg, *args)
unused = sorted(set(self.versions.keys()) - self.used)
if unused:
for name in unused:
logger.info(
u"Unused requirement '%s' in '%s'.",
self.versions[name].name, self.name)
def __init__(self, *args):
__status__ = u"Initializing fake source."
super(FakeSource, self).__init__(*args)
self.requirements = Requirements.parse(
self.options.get('packages', '').as_list())