def do_build(
project: Project,
sdist: bool = True,
wheel: bool = True,
dest: str = "dist",
clean: bool = True,
):
"""Build artifacts for distribution."""
if project.is_global:
raise ProjectError("Not allowed to build based on the global project.")
check_project_file(project)
if not wheel and not sdist:
stream.echo("All artifacts are disabled, nothing to do.", err=True)
return
if not os.path.isabs(dest):
dest = project.root.joinpath(dest).as_posix()
if clean:
shutil.rmtree(dest, ignore_errors=True)
with stream.logging("build"), EnvBuilder(project.root,
project.environment) as builder:
if sdist:
stream.echo("Building sdist...")
loc = builder.build_sdist(dest)
stream.echo(f"Built sdist at {loc}")
if wheel:
stream.echo("Building wheel...")
loc = builder.build_wheel(dest)
stream.echo(f"Built wheel at {loc}")
def do_lock(
project: Project,
strategy: str = "all",
tracked_names: Optional[Iterable[str]] = None,
requirements: Optional[List[Requirement]] = None,
) -> Dict[str, Candidate]:
"""Performs the locking process and update lockfile.
:param project: the project instance
:param strategy: update stratege: reuse/eager/all
:param tracked_names: required when using eager strategy
:param requirements: An optional dictionary of requirements, read from pyproject
if not given.
"""
check_project_file(project)
# TODO: multiple dependency definitions for the same package.
provider = project.get_provider(strategy, tracked_names)
if not requirements:
requirements = [
r for deps in project.all_dependencies.values()
for r in deps.values()
]
with stream.open_spinner(title="Resolving dependencies",
spinner="dots") as spin, stream.logging("lock"):
reporter = project.get_reporter(requirements, tracked_names, spin)
resolver = project.core.resolver_class(provider, reporter)
mapping, dependencies, summaries = resolve(
resolver, requirements, project.environment.python_requires)
data = format_lockfile(mapping, dependencies, summaries)
spin.succeed("Resolution success")
project.write_lockfile(data)
return mapping
def resolve_requirements(
repository,
lines,
requires_python="",
allow_prereleases=None,
strategy="all",
preferred_pins=None,
tracked_names=None,
):
requirements = []
for line in lines:
if line.startswith("-e "):
requirements.append(parse_requirement(line[3:], True))
else:
requirements.append(parse_requirement(line))
requires_python = PySpecSet(requires_python)
if not preferred_pins:
provider = BaseProvider(repository, requires_python, allow_prereleases)
else:
provider_class = (
ReusePinProvider if strategy == "reuse" else EagerUpdateProvider
)
provider = provider_class(
preferred_pins,
tracked_names or (),
repository,
requires_python,
allow_prereleases,
)
with stream.open_spinner("Resolving dependencies") as spin, stream.logging("lock"):
reporter = SpinnerReporter(spin, requirements)
resolver = Resolver(provider, reporter)
mapping, *_ = resolve(resolver, requirements, requires_python)
return mapping
def do_lock(
project: Project,
strategy: str = "all",
tracked_names: Optional[Iterable[str]] = None,
requirements: Optional[List[Requirement]] = None,
) -> Dict[str, Candidate]:
"""Performs the locking process and update lockfile.
:param project: the project instance
:param strategy: update stratege: reuse/eager/all
:param tracked_names: required when using eager strategy
:param requirements: An optional dictionary of requirements, read from pyproject
if not given.
"""
check_project_file(project)
# TODO: multiple dependency definitions for the same package.
provider = project.get_provider(strategy, tracked_names)
if not requirements:
requirements = [
r for deps in project.all_dependencies.values()
for r in deps.values()
]
resolve_max_rounds = int(project.config["strategy.resolve_max_rounds"])
with stream.logging("lock"):
# The context managers are nested to ensure the spinner is stopped before
# any message is thrown to the output.
with stream.open_spinner(title="Resolving dependencies",
spinner="dots") as spin:
reporter = project.get_reporter(requirements, tracked_names, spin)
resolver = project.core.resolver_class(provider, reporter)
try:
mapping, dependencies, summaries = resolve(
resolver,
requirements,
project.environment.python_requires,
resolve_max_rounds,
)
except ResolutionTooDeep:
spin.fail(f"{LOCK} Lock failed")
stream.echo(
"The dependency resolution exceeds the maximum loop depth of "
f"{resolve_max_rounds}, there may be some circular dependencies "
"in your project. Try to solve them or increase the "
f"{stream.green('`strategy.resolve_max_rounds`')} config.",
err=True,
)
raise
except ResolutionImpossible as err:
spin.fail(f"{LOCK} Lock failed")
stream.echo(format_resolution_impossible(err), err=True)
raise
else:
data = format_lockfile(mapping, dependencies, summaries)
spin.succeed(f"{LOCK} Lock successful")
project.write_lockfile(data)
return mapping
def convert(project, filename):
ireqs, finder = parse_requirement_file(str(filename))
with stream.logging("build"):
reqs = [ireq_as_line(ireq, project.environment) for ireq in ireqs]
data = {"dependencies": make_array(reqs, True)}
settings = {}
if finder.index_urls:
sources = [convert_url_to_source(finder.index_urls[0], "pypi")]
sources.extend(convert_url_to_source(url) for url in finder.index_urls[1:])
settings["source"] = sources
return data, settings
def convert(project, filename, options):
ireqs, finder = parse_requirement_file(str(filename))
with stream.logging("build"):
reqs = [ireq_as_line(ireq, project.environment) for ireq in ireqs]
deps = make_array(reqs, True)
data = {"dependencies": [], "dev-dependencies": []}
if options.dev and options.section:
raise PdmUsageError(
"Can't specify --dev and --section at the same time")
elif options.dev:
data["dev-dependencies"] = deps
elif options.section:
data["optional-dependencies"] = {options.section: deps}
else:
data["dependencies"] = deps
settings = {}
if finder.index_urls:
sources = [convert_url_to_source(finder.index_urls[0], "pypi")]
sources.extend(
convert_url_to_source(url) for url in finder.index_urls[1:])
settings["source"] = sources
return data, settings
def synchronize(self, clean: bool = True, dry_run: bool = False) -> None:
"""Synchronize the working set with pinned candidates.
:param clean: Whether to remove unneeded packages, defaults to True.
:param dry_run: If set to True, only prints actions without actually do them.
"""
to_add, to_update, to_remove = self.compare_with_working_set()
if not clean:
to_remove = []
if not any([to_add, to_update, to_remove]):
stream.echo(
stream.yellow(
"All packages are synced to date, nothing to do."))
if not dry_run:
with stream.logging("install"):
self.update_project_egg_info()
return
to_do = {"remove": to_remove, "update": to_update, "add": to_add}
self._show_headline(to_do)
if dry_run:
self._show_summary(to_do)
return
handlers = {
"add": self.install_candidate,
"update": self.update_candidate,
"remove": self.remove_distribution,
}
sequential_jobs = []
parallel_jobs = []
# Self package will be installed after all other dependencies are installed.
install_self = None
for kind in to_do:
for key in to_do[kind]:
if (key == self.environment.project.meta.name and
self.environment.project.meta.project_name.lower()):
install_self = (kind, key)
elif key in self.SEQUENTIAL_PACKAGES:
sequential_jobs.append((kind, key))
elif key in self.candidates and self.candidates[
key].req.editable:
# Editable packages are installed sequentially.
sequential_jobs.append((kind, key))
else:
parallel_jobs.append((kind, key))
errors: List[str] = []
failed_jobs: List[Tuple[str, str]] = []
def update_progress(future, kind, key):
if future.exception():
failed_jobs.append((kind, key))
error = future.exception()
errors.extend([f"{kind} {stream.green(key)} failed:\n"] +
traceback.format_exception(
type(error), error, error.__traceback__))
with stream.logging("install"), self.environment.activate():
with stream.indent(" "):
for job in sequential_jobs:
kind, key = job
handlers[kind](key)
for i in range(self.retry_times + 1):
with self.create_executor() as executor:
for job in parallel_jobs:
kind, key = job
future = executor.submit(handlers[kind], key)
future.add_done_callback(
functools.partial(update_progress,
kind=kind,
key=key))
if not failed_jobs or i == self.retry_times:
break
parallel_jobs, failed_jobs = failed_jobs, []
errors.clear()
stream.echo("Retry failed jobs")
if errors:
stream.echo(stream.red("\nERRORS:"))
stream.echo("".join(errors), err=True)
raise InstallationError(
"Some package operations are not complete yet")
if install_self:
stream.echo("Installing the project as an editable package...")
with stream.indent(" "):
handlers[install_self[0]](install_self[1])
else:
self.update_project_egg_info()
stream.echo(f"\n{CELE} All complete!")
def synchronize(self, clean: bool = True, dry_run: bool = False) -> None:
"""Synchronize the working set with pinned candidates.
:param clean: Whether to remove unneeded packages, defaults to True.
:param dry_run: If set to True, only prints actions without actually do them.
"""
to_add, to_update, to_remove = self.compare_with_working_set()
if not clean:
to_remove = []
lists_to_check = [to_add, to_update, to_remove]
if not any(lists_to_check):
if not dry_run:
self.environment.write_site_py()
stream.echo("All packages are synced to date, nothing to do.")
return
if dry_run:
result = dict(
add=[self.candidates[key] for key in to_add],
update=[(self.working_set[key], self.candidates[key])
for key in to_update],
remove=[self.working_set[key] for key in to_remove],
)
self.summarize(result, dry_run)
return
handlers = {
"add": self.install_candidate,
"update": self.update_candidate,
"remove": self.remove_distribution,
}
result = defaultdict(list)
failed = defaultdict(list)
to_do = {"add": to_add, "update": to_update, "remove": to_remove}
# Keep track of exceptions
errors = []
def update_progress(future, section, key, bar):
if future.exception():
failed[section].append(key)
errors.append(future.exception())
else:
result[section].append(future.result())
bar.update(1)
with stream.logging("install"):
with self.progressbar("Synchronizing:",
sum(len(lst)
for lst in to_do.values())) as (bar,
pool):
# First update packages, then remove and add
for section in sorted(to_do, reverse=True):
# setup toolkits are installed sequentially before other packages.
for key in sorted(
to_do[section],
key=lambda x: x not in self.SEQUENTIAL_PACKAGES):
future = pool.submit(handlers[section], key)
future.add_done_callback(
functools.partial(update_progress,
section=section,
key=key,
bar=bar))
if key in self.SEQUENTIAL_PACKAGES:
future.result()
# Retry for failed items
for i in range(self.RETRY_TIMES):
if not any(failed.values()):
break
stream.echo(
stream.yellow(
"\nSome packages failed to install, retrying..."))
to_do = failed
failed = defaultdict(list)
errors.clear()
with self.progressbar(
f"Retrying ({i + 1}/{self.RETRY_TIMES}):",
sum(len(lst) for lst in to_do.values()),
) as (bar, pool):
for section in sorted(to_do, reverse=True):
for key in sorted(
to_do[section],
key=lambda x: x not in self.
SEQUENTIAL_PACKAGES,
):
future = pool.submit(handlers[section], key)
future.add_done_callback(
functools.partial(update_progress,
section=section,
key=key,
bar=bar))
if key in self.SEQUENTIAL_PACKAGES:
future.result()
# End installation
self.summarize(result)
self.environment.write_site_py()
if not any(failed.values()):
return
stream.echo("\n")
error_msg = []
if failed["add"] + failed["update"]:
error_msg.append(
"Installation failed: "
f"{', '.join(failed['add'] + failed['update'])}")
if failed["remove"]:
error_msg.append(
f"Removal failed: {', '.join(failed['remove'])}")
for error in errors:
stream.echo(
"".join(
traceback.format_exception(type(error), error,
error.__traceback__)),
verbosity=stream.DEBUG,
)
raise InstallationError("\n" + "\n".join(error_msg))