class Executor(object): def __init__(self, env, pool, config, io, parallel=True): self._env = env self._io = io self._dry_run = False self._enabled = True self._verbose = False self._authenticator = Authenticator(config, self._io) self._chef = Chef(config, self._env) self._chooser = Chooser(pool, self._env) if parallel and not (PY2 and WINDOWS): # This should be directly handled by ThreadPoolExecutor # however, on some systems the number of CPUs cannot be determined # (it raises a NotImplementedError), so, in this case, we assume # that the system only has one CPU. try: self._max_workers = cpu_count() + 4 except NotImplementedError: self._max_workers = 5 else: self._max_workers = 1 self._executor = ThreadPoolExecutor(max_workers=self._max_workers) self._total_operations = 0 self._executed_operations = 0 self._executed = {"install": 0, "update": 0, "uninstall": 0} self._skipped = {"install": 0, "update": 0, "uninstall": 0} self._sections = OrderedDict() self._lock = threading.Lock() self._shutdown = False @property def installations_count(self): # type: () -> int return self._executed["install"] @property def updates_count(self): # type: () -> int return self._executed["update"] @property def removals_count(self): # type: () -> int return self._executed["uninstall"] def supports_fancy_output(self): # type: () -> bool return self._io.supports_ansi() and not self._dry_run def disable(self): self._enabled = False return self def dry_run(self, dry_run=True): self._dry_run = dry_run return self def verbose(self, verbose=True): self._verbose = verbose return self def execute(self, operations): # type: (Operation) -> int self._total_operations = len(operations) for job_type in self._executed: self._executed[job_type] = 0 self._skipped[job_type] = 0 if operations and (self._enabled or self._dry_run): self._display_summary(operations) # We group operations by priority groups = itertools.groupby(operations, key=lambda o: -o.priority) self._sections = OrderedDict() for _, group in groups: tasks = [] serial_operations = [] for operation in group: if self._shutdown: break # Some operations are unsafe, we must execute them serially in a group # https://github.com/python-poetry/poetry/issues/3086 # https://github.com/python-poetry/poetry/issues/2658 # # We need to explicitly check source type here, see: # https://github.com/python-poetry/poetry-core/pull/98 is_parallel_unsafe = operation.job_type == "uninstall" or ( operation.package.develop and operation.package.source_type in {"directory", "git"}) if not operation.skipped and is_parallel_unsafe: serial_operations.append(operation) continue tasks.append( self._executor.submit(self._execute_operation, operation)) try: wait(tasks) for operation in serial_operations: wait([ self._executor.submit(self._execute_operation, operation) ]) except KeyboardInterrupt: self._shutdown = True if self._shutdown: # Cancelling further tasks from being executed [task.cancel() for task in tasks] self._executor.shutdown(wait=True) break return 1 if self._shutdown else 0 def _write(self, operation, line): if not self.supports_fancy_output( ) or not self._should_write_operation(operation): return if self._io.is_debug(): with self._lock: section = self._sections[id(operation)] section.write_line(line) return with self._lock: section = self._sections[id(operation)] section.output.clear() section.write(line) def _execute_operation(self, operation): try: if self.supports_fancy_output(): if id(operation) not in self._sections: if self._should_write_operation(operation): with self._lock: self._sections[id(operation)] = self._io.section() self._sections[id(operation)].write_line( " <fg=blue;options=bold>•</> {message}: <fg=blue>Pending...</>" .format(message=self.get_operation_message( operation), ), ) else: if self._should_write_operation(operation): if not operation.skipped: self._io.write_line( " <fg=blue;options=bold>•</> {message}".format( message=self.get_operation_message( operation), ), ) else: self._io.write_line( " <fg=default;options=bold,dark>•</> {message}: " "<fg=default;options=bold,dark>Skipped</> " "<fg=default;options=dark>for the following reason:</> " "<fg=default;options=bold,dark>{reason}</>".format( message=self.get_operation_message(operation), reason=operation.skip_reason, )) try: result = self._do_execute_operation(operation) except EnvCommandError as e: if e.e.returncode == -2: result = -2 else: raise # If we have a result of -2 it means a KeyboardInterrupt # in the any python subprocess, so we raise a KeyboardInterrupt # error to be picked up by the error handler. if result == -2: raise KeyboardInterrupt except Exception as e: try: from clikit.ui.components.exception_trace import ExceptionTrace if not self.supports_fancy_output(): io = self._io else: message = " <error>•</error> {message}: <error>Failed</error>".format( message=self.get_operation_message(operation, error=True), ) self._write(operation, message) io = self._sections.get(id(operation), self._io) with self._lock: trace = ExceptionTrace(e) trace.render(io) io.write_line("") finally: with self._lock: self._shutdown = True except KeyboardInterrupt: try: message = " <warning>•</warning> {message}: <warning>Cancelled</warning>".format( message=self.get_operation_message(operation, warning=True), ) if not self.supports_fancy_output(): self._io.write_line(message) else: self._write(operation, message) finally: with self._lock: self._shutdown = True def _do_execute_operation(self, operation): method = operation.job_type operation_message = self.get_operation_message(operation) if operation.skipped: if self.supports_fancy_output(): self._write( operation, " <fg=default;options=bold,dark>•</> {message}: " "<fg=default;options=bold,dark>Skipped</> " "<fg=default;options=dark>for the following reason:</> " "<fg=default;options=bold,dark>{reason}</>".format( message=operation_message, reason=operation.skip_reason, ), ) self._skipped[operation.job_type] += 1 return 0 if not self._enabled or self._dry_run: self._io.write_line( " <fg=blue;options=bold>•</> {message}".format( message=operation_message, )) return 0 result = getattr(self, "_execute_{}".format(method))(operation) if result != 0: return result message = " <fg=green;options=bold>•</> {message}".format( message=self.get_operation_message(operation, done=True), ) self._write(operation, message) self._increment_operations_count(operation, True) return result def _increment_operations_count(self, operation, executed): with self._lock: if executed: self._executed_operations += 1 self._executed[operation.job_type] += 1 else: self._skipped[operation.job_type] += 1 def run_pip(self, *args, **kwargs): # type: (...) -> int try: self._env.run_pip(*args, **kwargs) except EnvCommandError as e: output = decode(e.e.output) if ("KeyboardInterrupt" in output or "ERROR: Operation cancelled by user" in output): return -2 raise return 0 def get_operation_message(self, operation, done=False, error=False, warning=False): base_tag = "fg=default" operation_color = "c2" source_operation_color = "c2" package_color = "c1" if error: operation_color = "error" elif warning: operation_color = "warning" elif done: operation_color = "success" if operation.skipped: base_tag = "fg=default;options=dark" operation_color += "_dark" source_operation_color += "_dark" package_color += "_dark" if operation.job_type == "install": return "<{}>Installing <{}>{}</{}> (<{}>{}</>)</>".format( base_tag, package_color, operation.package.name, package_color, operation_color, operation.package.full_pretty_version, ) if operation.job_type == "uninstall": return "<{}>Removing <{}>{}</{}> (<{}>{}</>)</>".format( base_tag, package_color, operation.package.name, package_color, operation_color, operation.package.full_pretty_version, ) if operation.job_type == "update": return "<{}>Updating <{}>{}</{}> (<{}>{}</{}> -> <{}>{}</>)</>".format( base_tag, package_color, operation.initial_package.name, package_color, source_operation_color, operation.initial_package.full_pretty_version, source_operation_color, operation_color, operation.target_package.full_pretty_version, ) return "" def _display_summary(self, operations): installs = 0 updates = 0 uninstalls = 0 skipped = 0 for op in operations: if op.skipped: skipped += 1 continue if op.job_type == "install": installs += 1 elif op.job_type == "update": updates += 1 elif op.job_type == "uninstall": uninstalls += 1 if not installs and not updates and not uninstalls and not self._verbose: self._io.write_line("") self._io.write_line("No dependencies to install or update") return self._io.write_line("") self._io.write_line("<b>Package operations</b>: " "<info>{}</> install{}, " "<info>{}</> update{}, " "<info>{}</> removal{}" "{}".format( installs, "" if installs == 1 else "s", updates, "" if updates == 1 else "s", uninstalls, "" if uninstalls == 1 else "s", ", <info>{}</> skipped".format(skipped) if skipped and self._verbose else "", )) self._io.write_line("") def _execute_install(self, operation): # type: (Install) -> None return self._install(operation) def _execute_update(self, operation): # type: (Update) -> None return self._update(operation) def _execute_uninstall(self, operation): # type: (Uninstall) -> None message = " <fg=blue;options=bold>•</> {message}: <info>Removing...</info>".format( message=self.get_operation_message(operation), ) self._write(operation, message) return self._remove(operation) def _install(self, operation): package = operation.package if package.source_type == "directory": return self._install_directory(operation) if package.source_type == "git": return self._install_git(operation) if package.source_type == "file": archive = self._prepare_file(operation) elif package.source_type == "url": archive = self._download_link(operation, Link(package.source_url)) else: archive = self._download(operation) operation_message = self.get_operation_message(operation) message = " <fg=blue;options=bold>•</> {message}: <info>Installing...</info>".format( message=operation_message, ) self._write(operation, message) args = ["install", "--no-deps", str(archive)] if operation.job_type == "update": args.insert(2, "-U") return self.run_pip(*args) def _update(self, operation): return self._install(operation) def _remove(self, operation): package = operation.package # If we have a VCS package, remove its source directory if package.source_type == "git": src_dir = self._env.path / "src" / package.name if src_dir.exists(): safe_rmtree(str(src_dir)) try: return self.run_pip("uninstall", package.name, "-y") except CalledProcessError as e: if "not installed" in str(e): return 0 raise def _prepare_file(self, operation): package = operation.package message = " <fg=blue;options=bold>•</> {message}: <info>Preparing...</info>".format( message=self.get_operation_message(operation), ) self._write(operation, message) archive = Path(package.source_url) if not Path(package.source_url).is_absolute() and package.root_dir: archive = package.root_dir / archive archive = self._chef.prepare(archive) return archive def _install_directory(self, operation): from poetry.factory import Factory package = operation.package operation_message = self.get_operation_message(operation) message = " <fg=blue;options=bold>•</> {message}: <info>Building...</info>".format( message=operation_message, ) self._write(operation, message) if package.root_dir: req = os.path.join(str(package.root_dir), package.source_url) else: req = os.path.realpath(package.source_url) args = ["install", "--no-deps", "-U"] pyproject = PyProjectTOML(os.path.join(req, "pyproject.toml")) if pyproject.is_poetry_project(): # Even if there is a build system specified # some versions of pip (< 19.0.0) don't understand it # so we need to check the version of pip to know # if we can rely on the build system legacy_pip = self._env.pip_version < self._env.pip_version.__class__( 19, 0, 0) package_poetry = Factory().create_poetry( pyproject.file.path.parent) if package.develop and not package_poetry.package.build_script: from poetry.masonry.builders.editable import EditableBuilder # This is a Poetry package in editable mode # we can use the EditableBuilder without going through pip # to install it, unless it has a build script. builder = EditableBuilder(package_poetry, self._env, NullIO()) builder.build() return 0 elif legacy_pip or package_poetry.package.build_script: from poetry.core.masonry.builders.sdist import SdistBuilder # We need to rely on creating a temporary setup.py # file since the version of pip does not support # build-systems # We also need it for non-PEP-517 packages builder = SdistBuilder(package_poetry) with builder.setup_py(): if package.develop: args.append("-e") args.append(req) return self.run_pip(*args) if package.develop: args.append("-e") args.append(req) return self.run_pip(*args) def _install_git(self, operation): from poetry.core.vcs import Git package = operation.package operation_message = self.get_operation_message(operation) message = " <fg=blue;options=bold>•</> {message}: <info>Cloning...</info>".format( message=operation_message, ) self._write(operation, message) src_dir = self._env.path / "src" / package.name if src_dir.exists(): safe_rmtree(str(src_dir)) src_dir.parent.mkdir(exist_ok=True) git = Git() git.clone(package.source_url, src_dir) git.checkout(package.source_reference, src_dir) # Now we just need to install from the source directory package._source_url = str(src_dir) return self._install_directory(operation) def _download(self, operation): # type: (Operation) -> Path link = self._chooser.choose_for(operation.package) return self._download_link(operation, link) def _download_link(self, operation, link): package = operation.package archive = self._chef.get_cached_archive_for_link(link) if archive is link: # No cached distributions was found, so we download and prepare it try: archive = self._download_archive(operation, link) except BaseException: cache_directory = self._chef.get_cache_directory_for_link(link) cached_file = cache_directory.joinpath(link.filename) # We can't use unlink(missing_ok=True) because it's not available # in pathlib2 for Python 2.7 if cached_file.exists(): cached_file.unlink() raise # TODO: Check readability of the created archive if not link.is_wheel: archive = self._chef.prepare(archive) if package.files: archive_hash = "sha256:" + FileDependency(package.name, archive).hash() if archive_hash not in {f["hash"] for f in package.files}: raise RuntimeError( "Invalid hash for {} using archive {}".format( package, archive.name)) return archive def _download_archive(self, operation, link): # type: (Operation, Link) -> Path response = self._authenticator.request("get", link.url, stream=True, io=self._sections.get( id(operation), self._io)) wheel_size = response.headers.get("content-length") operation_message = self.get_operation_message(operation) message = " <fg=blue;options=bold>•</> {message}: <info>Downloading...</>".format( message=operation_message, ) progress = None if self.supports_fancy_output(): if wheel_size is None: self._write(operation, message) else: from clikit.ui.components.progress_bar import ProgressBar progress = ProgressBar(self._sections[id(operation)].output, max=int(wheel_size)) progress.set_format(message + " <b>%percent%%</b>") if progress: with self._lock: progress.start() done = 0 archive = self._chef.get_cache_directory_for_link(link) / link.filename archive.parent.mkdir(parents=True, exist_ok=True) with archive.open("wb") as f: for chunk in response.iter_content(chunk_size=4096): if not chunk: break done += len(chunk) if progress: with self._lock: progress.set_progress(done) f.write(chunk) if progress: with self._lock: progress.finish() return archive def _should_write_operation(self, operation): # type: (Operation) -> bool if not operation.skipped: return True return self._dry_run or self._verbose
def _parse_requirements( self, requirements): # type: (List[str]) -> List[Dict[str, str]] from poetry.puzzle.provider import Provider result = [] try: cwd = self.poetry.file.parent except RuntimeError: cwd = Path.cwd() for requirement in requirements: requirement = requirement.strip() extras = [] extras_m = re.search(r"\[([\w\d,-_]+)\]$", requirement) if extras_m: extras = [e.strip() for e in extras_m.group(1).split(",")] requirement, _ = requirement.split("[") url_parsed = urlparse.urlparse(requirement) if url_parsed.scheme and url_parsed.netloc: # Url if url_parsed.scheme in ["git+https", "git+ssh"]: from poetry.core.vcs.git import Git from poetry.core.vcs.git import ParsedUrl parsed = ParsedUrl.parse(requirement) url = Git.normalize_url(requirement) pair = OrderedDict([("name", parsed.name), ("git", url.url)]) if parsed.rev: pair["rev"] = url.revision if extras: pair["extras"] = extras package = Provider.get_package_from_vcs( "git", url.url, reference=pair.get("rev")) pair["name"] = package.name result.append(pair) continue elif url_parsed.scheme in ["http", "https"]: package = Provider.get_package_from_url(requirement) pair = OrderedDict([("name", package.name), ("url", package.source_url)]) if extras: pair["extras"] = extras result.append(pair) continue elif (os.path.sep in requirement or "/" in requirement) and cwd.joinpath(requirement).exists(): path = cwd.joinpath(requirement) if path.is_file(): package = Provider.get_package_from_file(path.resolve()) else: package = Provider.get_package_from_directory(path) result.append( OrderedDict([ ("name", package.name), ("path", path.relative_to(cwd).as_posix()), ] + ([("extras", extras)] if extras else []))) continue pair = re.sub("^([^@=: ]+)(?:@|==|(?<![<>~!])=|:| )(.*)$", "\\1 \\2", requirement) pair = pair.strip() require = OrderedDict() if " " in pair: name, version = pair.split(" ", 2) extras_m = re.search(r"\[([\w\d,-_]+)\]$", name) if extras_m: extras = [e.strip() for e in extras_m.group(1).split(",")] name, _ = name.split("[") require["name"] = name if version != "latest": require["version"] = version else: m = re.match(r"^([^><=!: ]+)((?:>=|<=|>|<|!=|~=|~|\^).*)$", requirement.strip()) if m: name, constraint = m.group(1), m.group(2) extras_m = re.search(r"\[([\w\d,-_]+)\]$", name) if extras_m: extras = [ e.strip() for e in extras_m.group(1).split(",") ] name, _ = name.split("[") require["name"] = name require["version"] = constraint else: extras_m = re.search(r"\[([\w\d,-_]+)\]$", pair) if extras_m: extras = [ e.strip() for e in extras_m.group(1).split(",") ] pair, _ = pair.split("[") require["name"] = pair if extras: require["extras"] = extras result.append(require) return result