class PackageInstaller(object): """Package installer: install new package in a working set from sources. """ def __init__(self, options, working_set, directory=None): __status__ = u"Configuring package installer." self.interpretor = working_set.interpretor self.working_set = working_set self.kgs = options.utilities.kgs.get(options) self.sources = options.utilities.sources self.query = None self._to_install = Requirements() self._verify_being_installed = Requirements() self._being_installed = Requirements() self._lock = threading.RLock() self._wait = threading.Condition(threading.RLock()) self._worker_count = options.get_with_default( 'install_workers', 'setup', '5').as_int() self._options = options self._first_done = False self._error = None if directory is None: directory = options.get_with_default( 'lib_directory', 'setup').as_text() self._directory = directory def _wakeup_workers(self): # Wake up some workers to work. count_to_wake_up = max( self._worker_count - 1, len(self._to_install)) self._wait.acquire() for count in range(count_to_wake_up): self._wait.notify() self._wait.release() def _verify_extra_install(self, requirement): # Verify is some extra need installation release = self.working_set[requirement] for extra in requirement.extras: if extra not in release.extras: raise PackageError( u'Require missing extra requirements "%s" in "%s"' % ( extra, release)) self._register_install(release.extras[extra]) def _register_install(self, requirements): # Mark requirements to be installed. for requirement in requirements: if self.kgs is not None: requirement = self.kgs.upgrade(requirement) if requirement in self.working_set: if requirement.extras: self._verify_extra_install(requirement) logger.debug( u'Skip already installed dependency %s', requirement) continue if requirement in self._being_installed: if requirement.extras: self._verify_being_installed.append(requirement) logger.debug( u'Skip already being installed dependency %s', requirement) continue logger.debug( u'Need to install dependency %s', requirement) self._to_install.append(requirement) def wait_for_requirements(self): """Called by a worker to wait for requirement to arrive. """ logger.debug(u'Wait for dependencies') self._wait.acquire() self._wait.wait() self._wait.release() def mark_failed(self, error): """Called by a worked to report an error. """ self._lock.acquire() try: logger.debug(u'Failure') self._error = error logs.report(fatal=False) self._wait.acquire() self._wait.notifyAll() self._wait.release() finally: self._lock.release() def get_requirement(self): """Called by a worker to get a new requirement to install. """ self._lock.acquire() try: if self._error is not None: return INSTALLATION_DONE if not self._to_install: if not self._being_installed: if not self._first_done: # Wake up other waiting worker self._wait.acquire() self._wait.notifyAll() self._wait.release() self._first_done = True return INSTALLATION_DONE return None requirement = self._to_install.pop() assert requirement not in self._being_installed self._being_installed.append(requirement) logger.info(u'Installing %s', requirement) return requirement finally: self._lock.release() def mark_installed(self, requirement, package): """Called by a worker to mark that the given requirement have been installed, using the given package. """ self._lock.acquire() try: self.working_set.add(package) if self.kgs is not None: self.kgs.report_picked(requirement, package.version) if requirement in self._verify_being_installed: extra_requirement = self._verify_being_installed[requirement] logger.debug( u'Verify pending extra for %s', extra_requirement) worker_need_wake_up = len(self._to_install) == 0 self._verify_extra_install(extra_requirement) if worker_need_wake_up: self._wakeup_workers() self._verify_being_installed.remove(extra_requirement) self._being_installed.remove(requirement) logger.debug(u'Mark %s as installed', requirement) finally: self._lock.release() def install_dependencies(self, requirements): self._lock.acquire() try: worker_need_wake_up = len(self._to_install) == 0 self._register_install(requirements) if worker_need_wake_up: self._wakeup_workers() finally: self._lock.release() def __call__(self, requirements, strategy=STRATEGY_UPDATE, directory=None): """Called by the user to trigger the installation of the given list of requirements. """ __status__ = u"Installing %r." % (requirements) if directory is None: directory = self._directory self._error = None self._first_done = False self._register_install(requirements) if self._to_install: self.query = self.sources( self.interpretor, os.path.abspath(directory)) workers = [] for count in range(self._worker_count): worker = PackageInstallerWorker(self, count, strategy) worker.start() workers.append(worker) for worker in workers: worker.join() if self._error is not None: raise self._error return self.working_set
class KnownGoodVersionSet(object): """Represent a Known Good Version set. """ def __init__(self, options, installed_options=None): self.versions = dict(map(lambda v: (v.key, v), map(KnownVersion, options.values()))) self.name = options.name.split(':', 1)[1] self.used = set() self.missing = Requirements() self.picked = {} self.options = options self.installed_options = installed_options self._uptodate = None self._activated = False def get(self, requirement, default=None): if self._activated: __status__ = u"Looking version for %s in Known Good Set %s." % ( str(requirement), self.name) if requirement.key in self.versions: self.used.add(requirement.key) return Version.parse(self.versions[requirement.key].version) return default def is_activated(self): return self._activated def is_uptodate(self): if self._uptodate is None: if self.installed_options is None: self._uptodate = True else: self._uptodate = (self.options == self.installed_options) return self._uptodate def activate(self): # This means the KGS is activated in order to be used. self._activated = True def report_missing(self, requirement): self.missing.append(requirement) def report_picked(self, requirement, version): picked = self.picked.setdefault(requirement, []) picked.append(version) def log_usage(self): if self.missing: for requirement in self.missing: msg = u"Missing requirement for '%s' in '%s'" args = (requirement, self.name) picked = self.picked.get(requirement) if picked: msg += u", picked '%s'" args += (','.join(map(str, picked)),) msg += u"." logger.warn(msg, *args) unused = sorted(set(self.versions.keys()) - self.used) if unused: for name in unused: logger.info( u"Unused requirement '%s' in '%s'.", self.versions[name].name, self.name)