def _list_config(self, project: Project, options: argparse.Namespace) -> None:
stream.echo(
"Home configuration ({}):".format(project.global_config._config_file)
)
with stream.indent(" "):
for key in sorted(project.global_config):
stream.echo(
stream.yellow(
"# " + project.global_config._config_map[key].description
),
verbosity=stream.DETAIL,
)
stream.echo(f"{stream.cyan(key)} = {project.global_config[key]}")
stream.echo()
stream.echo(
"Project configuration ({}):".format(project.project_config._config_file)
)
with stream.indent(" "):
for key in sorted(project.project_config):
stream.echo(
stream.yellow(
"# " + project.project_config._config_map[key].description
),
verbosity=stream.DETAIL,
)
stream.echo(f"{stream.cyan(key)} = {project.project_config[key]}")
def handle(self, project: Project, options: argparse.Namespace) -> None:
package = options.package
req = parse_requirement(package)
repository = project.get_repository()
# reverse the result so that latest is at first.
matches = repository.find_candidates(
req, project.environment.python_requires, True)
latest = next(iter(matches), None)
if not latest:
stream.echo(
stream.yellow(f"No match found for the package {package!r}"),
err=True)
return
latest_stable = next(filter(filter_stable, matches), None)
installed = project.environment.get_working_set().get(package)
metadata = latest.get_metadata()
if metadata._legacy:
result = ProjectInfo(dict(metadata._legacy.items()), True)
else:
result = ProjectInfo(dict(metadata._data), False)
if latest_stable:
result.latest_stable_version = str(latest_stable.version)
if installed:
result.installed_version = str(installed.version)
stream.display_columns(list(result.generate_rows()))
def ask_for_import(project: Project) -> None:
"""Show possible importable files and ask user to decide"""
importable_files = list(find_importable_files(project))
if not importable_files:
return
stream.echo(
stream.cyan("Found following files from other formats that you may import:")
)
for i, (key, filepath) in enumerate(importable_files):
stream.echo(f"{i}. {stream.green(filepath.as_posix())} ({key})")
stream.echo(
"{}. {}".format(
len(importable_files),
stream.yellow("don't do anything, I will import later."),
)
)
choice = click.prompt(
"Please select:",
type=click.Choice([str(i) for i in range(len(importable_files) + 1)]),
show_default=False,
)
if int(choice) == len(importable_files):
return
key, filepath = importable_files[int(choice)]
do_import(project, filepath, key)
def format_package(
graph: DirectedGraph,
package: Package,
required: str = "",
prefix: str = "",
visited=None,
) -> str:
"""Format one package.
:param graph: the dependency graph
:param package: the package instance
:param required: the version required by its parent
:param prefix: prefix text for children
:param visited: the visited package collection
"""
if visited is None:
visited = set()
result = []
version = (
stream.red("[ not installed ]")
if not package.version
else stream.red(package.version)
if required
and required != "Any"
and not SpecifierSet(required).contains(package.version)
else stream.yellow(package.version)
)
if package.name in visited:
version = stream.red("[circular]")
required = f"[ required: {required} ]" if required else ""
result.append(f"{stream.green(package.name, bold=True)} {version} {required}\n")
if package.name in visited:
return "".join(result)
visited.add(package.name)
try:
*children, last = sorted(graph.iter_children(package), key=lambda p: p.name)
except ValueError: # No children nodes
pass
else:
for child in children:
required = str(package.requirements[child.name].specifier or "Any")
result.append(
prefix
+ NON_LAST_CHILD
+ format_package(
graph, child, required, prefix + NON_LAST_PREFIX, visited.copy()
)
)
required = str(package.requirements[last.name].specifier or "Any")
result.append(
prefix
+ LAST_CHILD
+ format_package(
graph, last, required, prefix + LAST_PREFIX, visited.copy()
)
)
return "".join(result)
def format_help(self):
formatter = self._get_formatter()
formatter.io = stream
if getattr(self, "is_root", False):
banner = (
cfonts.render(
"PDM",
font="slick",
gradient=["bright_red", "bright_green"],
space=False,
)
+ "\n"
)
formatter._add_item(lambda x: x, [banner])
self._positionals.title = "Commands"
self._optionals.title = "Options"
# description
formatter.add_text(self.description)
# usage
formatter.add_usage(
self.usage,
self._actions,
self._mutually_exclusive_groups,
prefix=stream.yellow("Usage", bold=True) + ": ",
)
# positionals, optionals and user-defined groups
for action_group in self._action_groups:
formatter.start_section(
stream.yellow(action_group.title, bold=True)
if action_group.title
else None
)
formatter.add_text(action_group.description)
formatter.add_arguments(action_group._group_actions)
formatter.end_section()
# epilog
formatter.add_text(self.epilog)
# determine help from format above
return formatter.format_help()
def migrate_pyproject(project: Project):
"""Migrate the legacy pyproject format to PEP 621"""
if (not project.pyproject_file.exists()
or not FORMATS["legacy"].check_fingerprint(project,
project.pyproject_file)
or "project" in project.pyproject):
return
stream.echo(
stream.yellow(
"Legacy [tool.pdm] metadata detected, migrating to PEP 621..."),
err=True,
)
do_import(project, project.pyproject_file, "legacy")
stream.echo(
stream.green("pyproject.toml") + stream.yellow(
" has been migrated to PEP 621 successfully. "
"Now you can safely delete the legacy metadata under [tool.pdm] table."
),
err=True,
)
def __delitem__(self, key) -> None:
self._data.pop(key, None)
try:
del self._file_data[key]
except KeyError:
pass
else:
env_var = self._config_map[key].env_var
if env_var is not None and env_var in os.environ:
stream.echo(
stream.yellow(
"WARNING: the config is shadowed by env var '{}', "
"set value won't take effect.".format(env_var)))
self._save_config()
def __setitem__(self, key: str, value: Any) -> None:
if key not in self._config_map:
raise NoConfigError(key)
if not self.is_global and self._config_map[key].global_only:
raise ValueError(
f"Config item '{key}' is not allowed to set in project config."
)
value = self._config_map[key].coerce(value)
env_var = self._config_map[key].env_var
if env_var is not None and env_var in os.environ:
stream.echo(
stream.yellow(
"WARNING: the config is shadowed by env var '{}', "
"the value set won't take effect.".format(env_var)))
self._data[key] = value
self._file_data[key] = value
self._save_config()
def main(self, args=None, prog_name=None, obj=None, **extra):
"""The main entry function"""
from pdm.models.pip_shims import global_tempdir_manager
self.init_parser()
self.load_plugins()
self.parser.set_defaults(global_project=None)
options = self.parser.parse_args(args or None)
stream.set_verbosity(options.verbose)
if obj is not None:
options.project = obj
if options.global_project:
options.project = options.global_project
if options.pep582:
print_pep582_command(options.pep582)
sys.exit(0)
if not getattr(options, "project", None):
options.project = self.project_class()
# Add reverse reference for core object
options.project.core = self
migrate_pyproject(options.project)
try:
f = options.handler
except AttributeError:
self.parser.print_help()
sys.exit(1)
else:
try:
with global_tempdir_manager():
f(options.project, options)
except Exception:
etype, err, traceback = sys.exc_info()
if stream.verbosity > stream.NORMAL:
raise err.with_traceback(traceback)
stream.echo(f"{stream.red('[' + etype.__name__ + ']')}: {err}",
err=True)
stream.echo(
stream.yellow("Add '-v' to see the detailed traceback"))
sys.exit(1)
def print_results(hits, working_set, terminal_width=None):
if not hits:
return
name_column_width = (
max(
[
len(hit["name"]) + len(highest_version(hit.get("versions", ["-"])))
for hit in hits
]
)
+ 4
)
for hit in hits:
name = hit["name"]
summary = hit["summary"] or ""
latest = highest_version(hit.get("versions", ["-"]))
if terminal_width is not None:
target_width = terminal_width - name_column_width - 5
if target_width > 10:
# wrap and indent summary to fit terminal
summary = textwrap.wrap(summary, target_width)
summary = ("\n" + " " * (name_column_width + 2)).join(summary)
current_width = len(name) + len(latest) + 4
spaces = " " * (name_column_width - current_width)
line = "{name} ({latest}){spaces} - {summary}".format(
name=stream.green(name, bold=True),
latest=stream.yellow(latest),
spaces=spaces,
summary=summary,
)
try:
stream.echo(line)
if safe_name(name).lower() in working_set:
dist = working_set[safe_name(name).lower()]
if dist.version == latest:
stream.echo(" INSTALLED: %s (latest)" % dist.version)
else:
stream.echo(" INSTALLED: %s" % dist.version)
stream.echo(" LATEST: %s" % latest)
except UnicodeEncodeError:
pass
def search(self, query: str) -> SearchResult:
pypi_simple = self.sources[0]["url"].rstrip("/")
results = []
if pypi_simple.endswith("/simple"):
search_url = pypi_simple[:-6] + "search"
else:
search_url = pypi_simple + "/search"
with self.environment.get_finder() as finder:
session = finder.session
resp = session.get(search_url, params={"q": query})
if resp.status_code == 404:
stream.echo(
stream.yellow(
f"{pypi_simple!r} doesn't support '/search' endpoint, fallback "
f"to {self.DEFAULT_INDEX_URL!r} now.\n"
"This may take longer depending on your network condition."
),
err=True,
)
resp = session.get(
f"{self.DEFAULT_INDEX_URL}/search", params={"q": query}
)
resp.raise_for_status()
content = parse(resp.content, namespaceHTMLElements=False)
for result in content.findall(".//*[@class='package-snippet']"):
name = result.find("h3/*[@class='package-snippet__name']").text
version = result.find("h3/*[@class='package-snippet__version']").text
if not name or not version:
continue
description = result.find("p[@class='package-snippet__description']").text
if not description:
description = ""
result = Package(name, version, description)
results.append(result)
return results
def format_package(
graph: DirectedGraph,
package: Package,
required: str = "",
prefix: str = "",
visited=None,
) -> str:
"""Format one package.
:param graph: the dependency graph
:param package: the package instance
:param required: the version required by its parent
:param prefix: prefix text for children
:param visited: the visited package collection
"""
if visited is None:
visited = set()
result = []
version = (stream.red("[ not installed ]")
if not package.version else stream.red(package.version)
if required and required not in ("Any", "This project")
and not SpecifierSet(required).contains(package.version) else
stream.yellow(package.version))
if package.name in visited:
version = stream.red("[circular]")
required = f"[ required: {required} ]" if required else "[ Not required ]"
result.append(
f"{stream.green(package.name, bold=True)} {version} {required}\n")
if package.name in visited:
return "".join(result)
visited.add(package.name)
children = sorted(graph.iter_children(package), key=lambda p: p.name)
for i, child in enumerate(children):
is_last = i == len(children) - 1
head = LAST_CHILD if is_last else NON_LAST_CHILD
cur_prefix = LAST_PREFIX if is_last else NON_LAST_PREFIX
required = str(package.requirements[child.name].specifier or "Any")
result.append(prefix + head +
format_package(graph, child, required, prefix +
cur_prefix, visited.copy()))
return "".join(result)
def __setitem__(self, key: str, value: Any) -> None:
if key not in self._config_map:
raise NoConfigError(key)
if not self.is_global and self._config_map[key].global_only:
raise ValueError(
f"Config item '{key}' is not allowed to set in project config."
)
if isinstance(value, str):
if value.lower() == "false":
value = False
elif value.lower() == "true":
value = True
env_var = self._config_map[key].env_var
if env_var is not None and env_var in os.environ:
stream.echo(
stream.yellow(
"WARNING: the config is shadowed by env var '{}', "
"set value won't take effect.".format(env_var)))
self._data[key] = value
self._file_data[key] = value
self._save_config()
def format_reverse_package(
graph: DirectedGraph,
package: Package,
child: Optional[Package] = None,
requires: str = "",
prefix: str = "",
visited=None,
):
"""Format one package for output reverse dependency graph."""
if visited is None:
visited = set()
result = []
version = (stream.red("[ not installed ]")
if not package.version else stream.yellow(package.version))
if package.name in visited:
version = stream.red("[circular]")
requires = (f"[ requires: {stream.red(requires)} ]"
if requires not in ("Any", "") and child and child.version
and not SpecifierSet(requires).contains(child.version) else
"" if not requires else f"[ requires: {requires} ]")
result.append(
f"{stream.green(package.name, bold=True)} {version} {requires}\n")
if package.name in visited:
return "".join(result)
visited.add(package.name)
parents = sorted(filter(None, graph.iter_parents(package)),
key=lambda p: p.name)
for i, parent in enumerate(parents):
is_last = i == len(parents) - 1
head = LAST_CHILD if is_last else NON_LAST_CHILD
cur_prefix = LAST_PREFIX if is_last else NON_LAST_PREFIX
requires = str(parent.requirements[package.name].specifier or "Any")
result.append(
prefix + head +
format_reverse_package(graph, parent, package, requires, prefix +
cur_prefix, visited.copy()))
return "".join(result)
def main(self, args=None, prog_name=None, obj=None, **extra):
"""The main entry function"""
from pdm.models.pip_shims import global_tempdir_manager
self.init_parser()
self.load_plugins()
options = self.parser.parse_args(args or None)
stream.set_verbosity(options.verbose)
if options.ignore_python:
os.environ["PDM_IGNORE_SAVED_PYTHON"] = "1"
if options.pep582:
print_pep582_command(options.pep582)
sys.exit(0)
self.ensure_project(options, obj)
try:
f = options.handler
except AttributeError:
self.parser.print_help()
sys.exit(1)
else:
try:
with global_tempdir_manager():
f(options.project, options)
except Exception:
etype, err, traceback = sys.exc_info()
if stream.verbosity > stream.NORMAL:
raise err.with_traceback(traceback)
stream.echo(
f"{stream.red('[' + etype.__name__ + ']')}: {err}", err=True
)
stream.echo(stream.yellow("Add '-v' to see the detailed traceback"))
sys.exit(1)
def synchronize(self, clean: bool = True, dry_run: bool = False) -> None:
"""Synchronize the working set with pinned candidates.
:param clean: Whether to remove unneeded packages, defaults to True.
:param dry_run: If set to True, only prints actions without actually do them.
"""
to_add, to_update, to_remove = self.compare_with_working_set()
if not clean:
to_remove = []
if not any([to_add, to_update, to_remove]):
stream.echo(
stream.yellow(
"All packages are synced to date, nothing to do."))
if not dry_run:
with stream.logging("install"):
self.update_project_egg_info()
return
to_do = {"remove": to_remove, "update": to_update, "add": to_add}
self._show_headline(to_do)
if dry_run:
self._show_summary(to_do)
return
handlers = {
"add": self.install_candidate,
"update": self.update_candidate,
"remove": self.remove_distribution,
}
sequential_jobs = []
parallel_jobs = []
# Self package will be installed after all other dependencies are installed.
install_self = None
for kind in to_do:
for key in to_do[kind]:
if (key == self.environment.project.meta.name and
self.environment.project.meta.project_name.lower()):
install_self = (kind, key)
elif key in self.SEQUENTIAL_PACKAGES:
sequential_jobs.append((kind, key))
elif key in self.candidates and self.candidates[
key].req.editable:
# Editable packages are installed sequentially.
sequential_jobs.append((kind, key))
else:
parallel_jobs.append((kind, key))
errors: List[str] = []
failed_jobs: List[Tuple[str, str]] = []
def update_progress(future, kind, key):
if future.exception():
failed_jobs.append((kind, key))
error = future.exception()
errors.extend([f"{kind} {stream.green(key)} failed:\n"] +
traceback.format_exception(
type(error), error, error.__traceback__))
with stream.logging("install"), self.environment.activate():
with stream.indent(" "):
for job in sequential_jobs:
kind, key = job
handlers[kind](key)
for i in range(self.retry_times + 1):
with self.create_executor() as executor:
for job in parallel_jobs:
kind, key = job
future = executor.submit(handlers[kind], key)
future.add_done_callback(
functools.partial(update_progress,
kind=kind,
key=key))
if not failed_jobs or i == self.retry_times:
break
parallel_jobs, failed_jobs = failed_jobs, []
errors.clear()
stream.echo("Retry failed jobs")
if errors:
stream.echo(stream.red("\nERRORS:"))
stream.echo("".join(errors), err=True)
raise InstallationError(
"Some package operations are not complete yet")
if install_self:
stream.echo("Installing the project as an editable package...")
with stream.indent(" "):
handlers[install_self[0]](install_self[1])
else:
self.update_project_egg_info()
stream.echo(f"\n{CELE} All complete!")
def synchronize(self, clean: bool = True, dry_run: bool = False) -> None:
"""Synchronize the working set with pinned candidates.
:param clean: Whether to remove unneeded packages, defaults to True.
:param dry_run: If set to True, only prints actions without actually do them.
"""
to_add, to_update, to_remove = self.compare_with_working_set()
if not clean:
to_remove = []
lists_to_check = [to_add, to_update, to_remove]
if not any(lists_to_check):
if not dry_run:
self.environment.write_site_py()
stream.echo("All packages are synced to date, nothing to do.")
return
if dry_run:
result = dict(
add=[self.candidates[key] for key in to_add],
update=[(self.working_set[key], self.candidates[key])
for key in to_update],
remove=[self.working_set[key] for key in to_remove],
)
self.summarize(result, dry_run)
return
handlers = {
"add": self.install_candidate,
"update": self.update_candidate,
"remove": self.remove_distribution,
}
result = defaultdict(list)
failed = defaultdict(list)
to_do = {"add": to_add, "update": to_update, "remove": to_remove}
# Keep track of exceptions
errors = []
def update_progress(future, section, key, bar):
if future.exception():
failed[section].append(key)
errors.append(future.exception())
else:
result[section].append(future.result())
bar.update(1)
with stream.logging("install"):
with self.progressbar("Synchronizing:",
sum(len(lst)
for lst in to_do.values())) as (bar,
pool):
# First update packages, then remove and add
for section in sorted(to_do, reverse=True):
# setup toolkits are installed sequentially before other packages.
for key in sorted(
to_do[section],
key=lambda x: x not in self.SEQUENTIAL_PACKAGES):
future = pool.submit(handlers[section], key)
future.add_done_callback(
functools.partial(update_progress,
section=section,
key=key,
bar=bar))
if key in self.SEQUENTIAL_PACKAGES:
future.result()
# Retry for failed items
for i in range(self.RETRY_TIMES):
if not any(failed.values()):
break
stream.echo(
stream.yellow(
"\nSome packages failed to install, retrying..."))
to_do = failed
failed = defaultdict(list)
errors.clear()
with self.progressbar(
f"Retrying ({i + 1}/{self.RETRY_TIMES}):",
sum(len(lst) for lst in to_do.values()),
) as (bar, pool):
for section in sorted(to_do, reverse=True):
for key in sorted(
to_do[section],
key=lambda x: x not in self.
SEQUENTIAL_PACKAGES,
):
future = pool.submit(handlers[section], key)
future.add_done_callback(
functools.partial(update_progress,
section=section,
key=key,
bar=bar))
if key in self.SEQUENTIAL_PACKAGES:
future.result()
# End installation
self.summarize(result)
self.environment.write_site_py()
if not any(failed.values()):
return
stream.echo("\n")
error_msg = []
if failed["add"] + failed["update"]:
error_msg.append(
"Installation failed: "
f"{', '.join(failed['add'] + failed['update'])}")
if failed["remove"]:
error_msg.append(
f"Removal failed: {', '.join(failed['remove'])}")
for error in errors:
stream.echo(
"".join(
traceback.format_exception(type(error), error,
error.__traceback__)),
verbosity=stream.DEBUG,
)
raise InstallationError("\n" + "\n".join(error_msg))
def format_dist(dist: Distribution) -> str:
formatter = "{version}{path}"
path = ""
if is_dist_editable(dist):
path = f" (-e {dist.location})"
return formatter.format(version=stream.yellow(dist.version), path=path)
