class PackageInstaller(object): """Package installer: install new package in a working set from sources. """ def __init__(self, options, working_set, directory=None): __status__ = u"Configuring package installer." self.interpretor = working_set.interpretor self.working_set = working_set self.kgs = options.utilities.kgs.get(options) self.sources = options.utilities.sources self.query = None self._to_install = Requirements() self._verify_being_installed = Requirements() self._being_installed = Requirements() self._lock = threading.RLock() self._wait = threading.Condition(threading.RLock()) self._worker_count = options.get_with_default( 'install_workers', 'setup', '5').as_int() self._options = options self._first_done = False self._error = None if directory is None: directory = options.get_with_default( 'lib_directory', 'setup').as_text() self._directory = directory def _wakeup_workers(self): # Wake up some workers to work. count_to_wake_up = max( self._worker_count - 1, len(self._to_install)) self._wait.acquire() for count in range(count_to_wake_up): self._wait.notify() self._wait.release() def _verify_extra_install(self, requirement): # Verify is some extra need installation release = self.working_set[requirement] for extra in requirement.extras: if extra not in release.extras: raise PackageError( u'Require missing extra requirements "%s" in "%s"' % ( extra, release)) self._register_install(release.extras[extra]) def _register_install(self, requirements): # Mark requirements to be installed. for requirement in requirements: if self.kgs is not None: requirement = self.kgs.upgrade(requirement) if requirement in self.working_set: if requirement.extras: self._verify_extra_install(requirement) logger.debug( u'Skip already installed dependency %s', requirement) continue if requirement in self._being_installed: if requirement.extras: self._verify_being_installed.append(requirement) logger.debug( u'Skip already being installed dependency %s', requirement) continue logger.debug( u'Need to install dependency %s', requirement) self._to_install.append(requirement) def wait_for_requirements(self): """Called by a worker to wait for requirement to arrive. """ logger.debug(u'Wait for dependencies') self._wait.acquire() self._wait.wait() self._wait.release() def mark_failed(self, error): """Called by a worked to report an error. """ self._lock.acquire() try: logger.debug(u'Failure') self._error = error logs.report(fatal=False) self._wait.acquire() self._wait.notifyAll() self._wait.release() finally: self._lock.release() def get_requirement(self): """Called by a worker to get a new requirement to install. """ self._lock.acquire() try: if self._error is not None: return INSTALLATION_DONE if not self._to_install: if not self._being_installed: if not self._first_done: # Wake up other waiting worker self._wait.acquire() self._wait.notifyAll() self._wait.release() self._first_done = True return INSTALLATION_DONE return None requirement = self._to_install.pop() assert requirement not in self._being_installed self._being_installed.append(requirement) logger.info(u'Installing %s', requirement) return requirement finally: self._lock.release() def mark_installed(self, requirement, package): """Called by a worker to mark that the given requirement have been installed, using the given package. """ self._lock.acquire() try: self.working_set.add(package) if self.kgs is not None: self.kgs.report_picked(requirement, package.version) if requirement in self._verify_being_installed: extra_requirement = self._verify_being_installed[requirement] logger.debug( u'Verify pending extra for %s', extra_requirement) worker_need_wake_up = len(self._to_install) == 0 self._verify_extra_install(extra_requirement) if worker_need_wake_up: self._wakeup_workers() self._verify_being_installed.remove(extra_requirement) self._being_installed.remove(requirement) logger.debug(u'Mark %s as installed', requirement) finally: self._lock.release() def install_dependencies(self, requirements): self._lock.acquire() try: worker_need_wake_up = len(self._to_install) == 0 self._register_install(requirements) if worker_need_wake_up: self._wakeup_workers() finally: self._lock.release() def __call__(self, requirements, strategy=STRATEGY_UPDATE, directory=None): """Called by the user to trigger the installation of the given list of requirements. """ __status__ = u"Installing %r." % (requirements) if directory is None: directory = self._directory self._error = None self._first_done = False self._register_install(requirements) if self._to_install: self.query = self.sources( self.interpretor, os.path.abspath(directory)) workers = [] for count in range(self._worker_count): worker = PackageInstallerWorker(self, count, strategy) worker.start() workers.append(worker) for worker in workers: worker.join() if self._error is not None: raise self._error return self.working_set