class PackageInstaller(object):
"""Package installer: install new package in a working set from
sources.
"""
def __init__(self, options, working_set, directory=None):
__status__ = u"Configuring package installer."
self.interpretor = working_set.interpretor
self.working_set = working_set
self.kgs = options.utilities.kgs.get(options)
self.sources = options.utilities.sources
self.query = None
self._to_install = Requirements()
self._verify_being_installed = Requirements()
self._being_installed = Requirements()
self._lock = threading.RLock()
self._wait = threading.Condition(threading.RLock())
self._worker_count = options.get_with_default(
'install_workers', 'setup', '5').as_int()
self._options = options
self._first_done = False
self._error = None
if directory is None:
directory = options.get_with_default(
'lib_directory', 'setup').as_text()
self._directory = directory
def _wakeup_workers(self):
# Wake up some workers to work.
count_to_wake_up = max(
self._worker_count - 1, len(self._to_install))
self._wait.acquire()
for count in range(count_to_wake_up):
self._wait.notify()
self._wait.release()
def _verify_extra_install(self, requirement):
# Verify is some extra need installation
release = self.working_set[requirement]
for extra in requirement.extras:
if extra not in release.extras:
raise PackageError(
u'Require missing extra requirements "%s" in "%s"' % (
extra, release))
self._register_install(release.extras[extra])
def _register_install(self, requirements):
# Mark requirements to be installed.
for requirement in requirements:
if self.kgs is not None:
requirement = self.kgs.upgrade(requirement)
if requirement in self.working_set:
if requirement.extras:
self._verify_extra_install(requirement)
logger.debug(
u'Skip already installed dependency %s', requirement)
continue
if requirement in self._being_installed:
if requirement.extras:
self._verify_being_installed.append(requirement)
logger.debug(
u'Skip already being installed dependency %s',
requirement)
continue
logger.debug(
u'Need to install dependency %s', requirement)
self._to_install.append(requirement)
def wait_for_requirements(self):
"""Called by a worker to wait for requirement to arrive.
"""
logger.debug(u'Wait for dependencies')
self._wait.acquire()
self._wait.wait()
self._wait.release()
def mark_failed(self, error):
"""Called by a worked to report an error.
"""
self._lock.acquire()
try:
logger.debug(u'Failure')
self._error = error
logs.report(fatal=False)
self._wait.acquire()
self._wait.notifyAll()
self._wait.release()
finally:
self._lock.release()
def get_requirement(self):
"""Called by a worker to get a new requirement to install.
"""
self._lock.acquire()
try:
if self._error is not None:
return INSTALLATION_DONE
if not self._to_install:
if not self._being_installed:
if not self._first_done:
# Wake up other waiting worker
self._wait.acquire()
self._wait.notifyAll()
self._wait.release()
self._first_done = True
return INSTALLATION_DONE
return None
requirement = self._to_install.pop()
assert requirement not in self._being_installed
self._being_installed.append(requirement)
logger.info(u'Installing %s', requirement)
return requirement
finally:
self._lock.release()
def mark_installed(self, requirement, package):
"""Called by a worker to mark that the given requirement have
been installed, using the given package.
"""
self._lock.acquire()
try:
self.working_set.add(package)
if self.kgs is not None:
self.kgs.report_picked(requirement, package.version)
if requirement in self._verify_being_installed:
extra_requirement = self._verify_being_installed[requirement]
logger.debug(
u'Verify pending extra for %s', extra_requirement)
worker_need_wake_up = len(self._to_install) == 0
self._verify_extra_install(extra_requirement)
if worker_need_wake_up:
self._wakeup_workers()
self._verify_being_installed.remove(extra_requirement)
self._being_installed.remove(requirement)
logger.debug(u'Mark %s as installed', requirement)
finally:
self._lock.release()
def install_dependencies(self, requirements):
self._lock.acquire()
try:
worker_need_wake_up = len(self._to_install) == 0
self._register_install(requirements)
if worker_need_wake_up:
self._wakeup_workers()
finally:
self._lock.release()
def __call__(self, requirements, strategy=STRATEGY_UPDATE, directory=None):
"""Called by the user to trigger the installation of the given
list of requirements.
"""
__status__ = u"Installing %r." % (requirements)
if directory is None:
directory = self._directory
self._error = None
self._first_done = False
self._register_install(requirements)
if self._to_install:
self.query = self.sources(
self.interpretor,
os.path.abspath(directory))
workers = []
for count in range(self._worker_count):
worker = PackageInstallerWorker(self, count, strategy)
worker.start()
workers.append(worker)
for worker in workers:
worker.join()
if self._error is not None:
raise self._error
return self.working_set
class KnownGoodVersionSet(object):
"""Represent a Known Good Version set.
"""
def __init__(self, options, installed_options=None):
self.versions = dict(map(lambda v: (v.key, v),
map(KnownVersion, options.values())))
self.name = options.name.split(':', 1)[1]
self.used = set()
self.missing = Requirements()
self.picked = {}
self.options = options
self.installed_options = installed_options
self._uptodate = None
self._activated = False
def get(self, requirement, default=None):
if self._activated:
__status__ = u"Looking version for %s in Known Good Set %s." % (
str(requirement), self.name)
if requirement.key in self.versions:
self.used.add(requirement.key)
return Version.parse(self.versions[requirement.key].version)
return default
def is_activated(self):
return self._activated
def is_uptodate(self):
if self._uptodate is None:
if self.installed_options is None:
self._uptodate = True
else:
self._uptodate = (self.options == self.installed_options)
return self._uptodate
def activate(self):
# This means the KGS is activated in order to be used.
self._activated = True
def report_missing(self, requirement):
self.missing.append(requirement)
def report_picked(self, requirement, version):
picked = self.picked.setdefault(requirement, [])
picked.append(version)
def log_usage(self):
if self.missing:
for requirement in self.missing:
msg = u"Missing requirement for '%s' in '%s'"
args = (requirement, self.name)
picked = self.picked.get(requirement)
if picked:
msg += u", picked '%s'"
args += (','.join(map(str, picked)),)
msg += u"."
logger.warn(msg, *args)
unused = sorted(set(self.versions.keys()) - self.used)
if unused:
for name in unused:
logger.info(
u"Unused requirement '%s' in '%s'.",
self.versions[name].name, self.name)
