def markers(**config_from_cli: Any) -> NoReturn:
"""Show all registered markers."""
config_from_cli["command"] = "markers"
try:
# Duplication of the same mechanism in :func:`pytask.main.main`.
pm = get_plugin_manager()
from _pytask import cli
pm.register(cli)
pm.hook.pytask_add_hooks(pm=pm)
config = pm.hook.pytask_configure(pm=pm,
config_from_cli=config_from_cli)
session = Session.from_config(config)
except (ConfigurationError, Exception):
console.print_exception()
session = Session({}, None)
session.exit_code = ExitCode.CONFIGURATION_FAILED
else:
table = Table("Marker", "Description", leading=1)
for name, description in config["markers"].items():
table.add_row(f"pytask.mark.{name}", description)
console.print(table)
sys.exit(session.exit_code)
def do_continue(self, arg): # type: ignore
ret = super().do_continue(arg)
if cls._recursive_debug == 0:
assert cls._config is not None
console.print()
capman = self._pytask_capman
capturing = PytaskPDB._is_capturing(capman)
if capturing:
console.rule(
"PDB continue (IO-capturing resumed)",
characters=">",
style=None,
)
assert capman is not None
capman.resume()
else:
console.rule("PDB continue",
characters=">",
style=None)
if not self._pytask_live_manager.is_started:
self._pytask_live_manager.resume()
assert cls._pluginmanager is not None
self._continued = True
return ret
def _print_errored_task_report(session: Session,
report: ExecutionReport) -> None:
"""Print the traceback and the exception of an errored report."""
task_name = format_task_id(
task=report.task,
editor_url_scheme=session.config["editor_url_scheme"],
short_name=True,
)
text = Text.assemble("Task ", task_name, " failed", style="failed")
console.rule(text, style=report.outcome.style)
console.print()
if report.exc_info and isinstance(report.exc_info[1], Exit):
console.print(format_exception_without_traceback(report.exc_info))
else:
console.print(
render_exc_info(*report.exc_info, session.config["show_locals"]))
console.print()
show_capture = session.config["show_capture"]
for when, key, content in report.sections:
if key in ("stdout", "stderr") and show_capture in (
ShowCapture[key.upper()],
ShowCapture.ALL,
):
console.rule(f"Captured {key} during {when}", style=None)
console.print(content)
def pytask_execute_task_log_end(session: Session,
report: ExecutionReport) -> None:
"""Log task outcome."""
url_style = create_url_style_for_task(report.task.function,
session.config["editor_url_scheme"])
console.print(
report.outcome.symbol,
style=unify_styles(report.outcome.style, url_style),
end="",
)
def collect(**config_from_cli: Any | None) -> NoReturn:
"""Collect tasks and report information about them."""
config_from_cli["command"] = "collect"
try:
# Duplication of the same mechanism in :func:`pytask.main.main`.
pm = get_plugin_manager()
from _pytask import cli
pm.register(cli)
pm.hook.pytask_add_hooks(pm=pm)
config = pm.hook.pytask_configure(pm=pm,
config_from_cli=config_from_cli)
session = Session.from_config(config)
except (ConfigurationError, Exception):
session = Session({}, None)
session.exit_code = ExitCode.CONFIGURATION_FAILED
console.print_exception()
else:
try:
session.hook.pytask_log_session_header(session=session)
session.hook.pytask_collect(session=session)
session.hook.pytask_resolve_dependencies(session=session)
tasks = _select_tasks_by_expressions_and_marker(session)
common_ancestor = _find_common_ancestor_of_all_nodes(
tasks, session.config["paths"], session.config["nodes"])
dictionary = _organize_tasks(tasks)
if dictionary:
_print_collected_tasks(
dictionary,
session.config["nodes"],
session.config["editor_url_scheme"],
common_ancestor,
)
console.print()
console.rule(style="neutral")
except CollectionError:
session.exit_code = ExitCode.COLLECTION_FAILED
except ResolvingDependenciesError:
session.exit_code = ExitCode.RESOLVING_DEPENDENCIES_FAILED
except Exception:
session.exit_code = ExitCode.FAILED
console.print_exception()
console.rule(style="failed")
sys.exit(session.exit_code)
def pytask_execute_build(self) -> Generator[None, None, None]:
"""Wrap the execution with the live manager and yield a complete table at the
end."""
self.live_manager.start()
yield
self.live_manager.stop(transient=True)
table = self._generate_table(reduce_table=False,
sort_table=self.sort_final_table,
add_caption=False)
if table is not None:
console.print(table)
def profile(**config_from_cli: Any) -> NoReturn:
"""Show information about tasks like runtime and memory consumption of products."""
config_from_cli["command"] = "profile"
try:
# Duplication of the same mechanism in :func:`pytask.main.main`.
pm = get_plugin_manager()
from _pytask import cli
pm.register(cli)
pm.hook.pytask_add_hooks(pm=pm)
config = pm.hook.pytask_configure(pm=pm, config_from_cli=config_from_cli)
session = Session.from_config(config)
except (ConfigurationError, Exception): # pragma: no cover
session = Session({}, None)
session.exit_code = ExitCode.CONFIGURATION_FAILED
exc_info: tuple[
type[BaseException], BaseException, TracebackType | None
] = sys.exc_info()
console.print(render_exc_info(*exc_info, show_locals=config["show_locals"]))
else:
try:
session.hook.pytask_log_session_header(session=session)
session.hook.pytask_collect(session=session)
session.hook.pytask_resolve_dependencies(session=session)
profile: dict[str, dict[str, Any]] = {
task.name: {} for task in session.tasks
}
session.hook.pytask_profile_add_info_on_task(
session=session, tasks=session.tasks, profile=profile
)
profile = _process_profile(profile)
_print_profile_table(profile, session.tasks, session.config)
session.hook.pytask_profile_export_profile(session=session, profile=profile)
console.rule(style="neutral")
except CollectionError: # pragma: no cover
session.exit_code = ExitCode.COLLECTION_FAILED
except Exception: # pragma: no cover
session.exit_code = ExitCode.FAILED
console.print_exception()
console.rule(style="failed")
sys.exit(session.exit_code)
def pytask_log_session_footer(session: Session) -> None:
"""Log warnings at the end of a session."""
if session.warnings:
grouped_warnings = defaultdict(list)
for warning in session.warnings:
location = (warning.id_ if warning.id_ is not None else
"{}:{}".format(*warning.fs_location))
grouped_warnings[warning.message].append(location)
sorted_gw = {k: sorted(v) for k, v in grouped_warnings.items()}
reduced_gw = _reduce_grouped_warnings(sorted_gw)
renderable = MyRenderable(reduced_gw)
panel = Panel(renderable, title="Warnings", style="warning")
console.print(panel)
def dag(**config_from_cli: Any) -> NoReturn:
"""Create a visualization of the project's directed acyclic graph."""
try:
pm = get_plugin_manager()
from _pytask import cli
pm.register(cli)
pm.hook.pytask_add_hooks(pm=pm)
config = pm.hook.pytask_configure(pm=pm, config_from_cli=config_from_cli)
session = Session.from_config(config)
except (ConfigurationError, Exception):
console.print_exception()
session = Session({}, None)
session.exit_code = ExitCode.CONFIGURATION_FAILED
else:
try:
session.hook.pytask_log_session_header(session=session)
import_optional_dependency("pydot")
check_for_optional_program(
session.config["layout"],
extra="The layout program is part of the graphviz package which you "
"can install with conda.",
)
session.hook.pytask_collect(session=session)
session.hook.pytask_resolve_dependencies(session=session)
dag = _refine_dag(session)
_write_graph(dag, session.config["output_path"], session.config["layout"])
except CollectionError:
session.exit_code = ExitCode.COLLECTION_FAILED
except ResolvingDependenciesError:
session.exit_code = ExitCode.RESOLVING_DEPENDENCIES_FAILED
except Exception:
session.exit_code = ExitCode.FAILED
exc_info = remove_internal_traceback_frames_from_exc_info(sys.exc_info())
console.print()
console.print(Traceback.from_exception(*exc_info))
console.rule(style="failed")
sys.exit(session.exit_code)
def _print_profile_table(
profile: dict[str, dict[str, Any]], tasks: list[Task], config: dict[str, Any]
) -> None:
"""Print the profile table."""
name_to_task = {task.name: task for task in tasks}
info_names = _get_info_names(profile)
console.print()
if profile:
table = Table("Task")
for name in info_names:
table.add_column(name, justify="right")
for task_name, info in profile.items():
task_id = format_task_id(
task=name_to_task[task_name],
editor_url_scheme=config["editor_url_scheme"],
short_name=True,
)
infos = [str(i) for i in info.values()]
table.add_row(task_id, *infos)
console.print(table)
else:
console.print("No information is stored on the collected tasks.")
def wrapper(*args: Any, **kwargs: Any) -> None:
capman = session.config["pm"].get_plugin("capturemanager")
live_manager = session.config["pm"].get_plugin("live_manager")
# Order is important! Pausing the live object before the capturemanager would
# flush the table to stdout and it will be visible in the captured output.
capman.suspend(in_=True)
out, err = capman.read()
live_manager.stop()
if out or err:
console.print()
if out:
console.rule("Captured stdout", style=None)
console.print(out)
if err:
console.rule("Captured stderr", style=None)
console.print(err)
_pdb.runcall(task_function, *args, **kwargs)
live_manager.resume()
capman.resume()
def wrapper(*args: Any, **kwargs: Any) -> None:
capman = session.config["pm"].get_plugin("capturemanager")
live_manager = session.config["pm"].get_plugin("live_manager")
try:
task_function(*args, **kwargs)
except Exception:
# Order is important! Pausing the live object before the capturemanager
# would flush the table to stdout and it will be visible in the captured
# output.
capman.suspend(in_=True)
out, err = capman.read()
live_manager.pause()
if out or err:
console.print()
if out:
console.rule("Captured stdout", style=None)
console.print(out)
if err:
console.rule("Captured stderr", style=None)
console.print(err)
exc_info = remove_internal_traceback_frames_from_exc_info(
sys.exc_info())
console.print()
console.rule("Traceback", characters=">", style=None)
console.print(
render_exc_info(*exc_info, session.config["show_locals"]))
post_mortem(exc_info[2])
live_manager.resume()
capman.resume()
raise
def pytask_log_session_header(session: Session) -> None:
"""Log the header of a pytask session."""
console.rule("Start pytask session", style=None)
console.print(
f"Platform: {sys.platform} -- Python {platform.python_version()}, "
f"pytask {_pytask.__version__}, pluggy {pluggy.__version__}")
console.print(f"Root: {session.config['root']}")
if session.config["config"] is not None:
console.print(f"Configuration: {session.config['config']}")
plugin_info = session.config["pm"].list_plugin_distinfo()
if plugin_info:
formatted_plugins_w_versions = ", ".join(
_format_plugin_names_and_versions(plugin_info))
console.print(f"Plugins: {formatted_plugins_w_versions}")
def pytask_resolve_dependencies_log(
session: Session, report: ResolvingDependenciesReport) -> None:
"""Log errors which happened while resolving dependencies."""
console.print()
console.rule(
Text("Failures during resolving dependencies", style="failed"),
style="failed",
)
console.print()
console.print(
render_exc_info(*report.exc_info, session.config["show_locals"]))
console.print()
console.rule(style="failed")
def _init_pdb(cls, method: str, *args: Any,
**kwargs: Any) -> pdb.Pdb: # noqa: U100
"""Initialize PDB debugging, dropping any IO capturing."""
if cls._pluginmanager is None:
capman = None
live_manager = None
else:
capman = cls._pluginmanager.get_plugin("capturemanager")
live_manager = cls._pluginmanager.get_plugin("live_manager")
if capman:
capman.suspend(in_=True)
if live_manager:
live_manager.pause()
if cls._config:
console.print()
if cls._recursive_debug == 0:
# Handle header similar to pdb.set_trace in py37+.
header = kwargs.pop("header", None)
if header is not None:
console.rule(header, characters=">", style=None)
else:
capturing = cls._is_capturing(capman)
if capturing:
console.rule(
f"PDB {method} (IO-capturing turned off)",
characters=">",
style=None,
)
else:
console.rule(f"PDB {method}",
characters=">",
style=None)
_pdb = cls._import_pdb_cls(capman, live_manager)(**kwargs)
return _pdb
def pytask_execute_log_end(session: Session,
reports: list[ExecutionReport]) -> bool:
"""Log information on the execution."""
session.execution_end = time.time()
counts = count_outcomes(reports, TaskOutcome)
if session.config["show_traceback"]:
console.print()
if counts[TaskOutcome.FAIL]:
console.rule(
Text("Failures", style=TaskOutcome.FAIL.style),
style=TaskOutcome.FAIL.style,
)
console.print()
for report in reports:
if report.outcome in (TaskOutcome.FAIL,
TaskOutcome.SKIP_PREVIOUS_FAILED):
_print_errored_task_report(session, report)
console.rule(style="dim")
panel = create_summary_panel(counts, TaskOutcome, "Collected tasks")
console.print(panel)
session.hook.pytask_log_session_footer(
session=session,
duration=session.execution_end - session.execution_start,
outcome=TaskOutcome.FAIL
if counts[TaskOutcome.FAIL] else TaskOutcome.SUCCESS,
)
if counts[TaskOutcome.FAIL]:
raise ExecutionError
return True
def pytask_log_session_header(session):
"""Add a note for how many workers are spawned."""
n_workers = session.config["n_workers"]
if n_workers > 1:
console.print(f"Started {n_workers} workers.")
def clean(**config_from_cli: Any) -> NoReturn:
"""Clean the provided paths by removing files unknown to pytask."""
config_from_cli["command"] = "clean"
try:
# Duplication of the same mechanism in :func:`pytask.main.main`.
pm = get_plugin_manager()
from _pytask import cli
pm.register(cli)
pm.hook.pytask_add_hooks(pm=pm)
config = pm.hook.pytask_configure(pm=pm,
config_from_cli=config_from_cli)
session = Session.from_config(config)
except Exception:
session = Session({}, None)
session.exit_code = ExitCode.CONFIGURATION_FAILED
exc_info: tuple[type[BaseException], BaseException,
TracebackType | None] = sys.exc_info()
console.print(render_exc_info(*exc_info))
else:
try:
session.hook.pytask_log_session_header(session=session)
session.hook.pytask_collect(session=session)
known_paths = _collect_all_paths_known_to_pytask(session)
exclude = session.config["exclude"]
include_directories = session.config["directories"]
unknown_paths = _find_all_unknown_paths(session, known_paths,
exclude,
include_directories)
common_ancestor = find_common_ancestor(*unknown_paths,
*session.config["paths"])
if unknown_paths:
targets = "Files"
if session.config["directories"]:
targets += " and directories"
console.print(f"\n{targets} which can be removed:\n")
for path in unknown_paths:
short_path = relative_to(path, common_ancestor)
if session.config["mode"] == "dry-run":
console.print(f"Would remove {short_path}")
else:
should_be_deleted = session.config[
"mode"] == "force" or click.confirm(
f"Would you like to remove {short_path}?")
if should_be_deleted:
if not session.config["quiet"]:
console.print(f"Remove {short_path}")
if path.is_dir():
shutil.rmtree(path)
else:
path.unlink()
else:
console.print()
console.print(
"There are no files and directories which can be deleted.")
console.print()
console.rule(style=None)
except CollectionError:
session.exit_code = ExitCode.COLLECTION_FAILED
console.rule(style="failed")
except Exception:
exc_info = sys.exc_info()
console.print(
render_exc_info(*exc_info, show_locals=config["show_locals"]))
console.rule(style="failed")
session.exit_code = ExitCode.FAILED
sys.exit(session.exit_code)
def _print_collected_tasks(
dictionary: dict[Path, list[Task]],
show_nodes: bool,
editor_url_scheme: str,
common_ancestor: Path,
) -> None:
"""Print the information on collected tasks.
Parameters
----------
dictionary : Dict[Path, List["Task"]]
A dictionary with path on the first level, tasks on the second, dependencies and
products on the third.
show_nodes : bool
Indicator for whether dependencies and products should be displayed.
editor_url_scheme : str
The scheme to create an url.
common_ancestor : Path
The path common to all tasks and nodes.
"""
# Have a new line between the number of collected tasks and this info.
console.print()
tree = Tree("Collected tasks:", highlight=True)
for module, tasks in dictionary.items():
reduced_module = relative_to(module, common_ancestor)
url_style = create_url_style_for_path(module, editor_url_scheme)
module_branch = tree.add(
Text.assemble(PYTHON_ICON, "<Module ",
Text(str(reduced_module), style=url_style), ">"))
for task in tasks:
reduced_task_name = format_task_id(
task,
editor_url_scheme=editor_url_scheme,
relative_to=common_ancestor)
task_branch = module_branch.add(
Text.assemble(TASK_ICON, "<Function ", reduced_task_name,
">"), )
if show_nodes:
for node in sorted(tree_just_flatten(task.depends_on),
key=lambda x: x.path):
reduced_node_name = relative_to(node.path, common_ancestor)
url_style = create_url_style_for_path(
node.path, editor_url_scheme)
task_branch.add(
Text.assemble(
FILE_ICON,
"<Dependency ",
Text(str(reduced_node_name), style=url_style),
">",
))
for node in sorted(tree_just_flatten(task.produces),
key=lambda x: x.path):
reduced_node_name = relative_to(node.path, common_ancestor)
url_style = create_url_style_for_path(
node.path, editor_url_scheme)
task_branch.add(
Text.assemble(
FILE_ICON,
"<Product ",
Text(str(reduced_node_name), style=url_style),
">",
))
console.print(tree)
def pytask_configure(pm: pluggy.PluginManager,
config_from_cli: dict[str, Any]) -> dict[str, Any]:
"""Configure pytask."""
config = {"pm": pm}
# Either the path to the configuration is passed via the CLI or it needs to be
# detected from the paths passed to pytask.
if config_from_cli.get("config"):
config["config"] = Path(config_from_cli["config"])
config["root"] = config["config"].parent
else:
paths = (parse_paths(config_from_cli.get("paths"))
if config_from_cli.get("paths") is not None else [Path.cwd()])
config["root"], config["config"] = _find_project_root_and_config(paths)
if config["config"] is None:
config_from_file = {}
else:
read_config = get_config_reader(config["config"])
config_from_file = read_config(config["config"])
if read_config.__name__ == "_read_ini_config":
toml_string = "# Content of pyproject.toml\n\n" + tomli_w.dumps(
{"tool": {
"pytask": {
"ini_options": config_from_file
}
}})
console.print(
Text(
_DEPRECATION_MESSAGE.format(
config["config"].with_name("pyproject.toml")),
style="warning",
))
console.print(Syntax(toml_string, "toml"))
# If paths are set in the configuration, process them.
if config_from_file.get("paths"):
paths_from_file = to_list(
parse_value_or_multiline_option(config_from_file.get("paths")))
config_from_file["paths"] = [
config["config"].parent.joinpath(p).resolve()
for p in paths_from_file
]
config["paths"] = get_first_non_none_value(
config_from_cli,
config_from_file,
key="paths",
default=[Path.cwd()],
callback=parse_paths,
)
config["markers"] = {
"depends_on":
("Add dependencies to a task. See this tutorial for more information: "
"[link https://bit.ly/3JlxylS]https://bit.ly/3JlxylS[/]."),
"produces":
("Add products to a task. See this tutorial for more information: "
"[link https://bit.ly/3JlxylS]https://bit.ly/3JlxylS[/]."),
"try_first":
"Try to execute a task a early as possible.",
"try_last":
"Try to execute a task a late as possible.",
}
pm.hook.pytask_parse_config(
config=config,
config_from_cli=config_from_cli,
config_from_file=config_from_file,
)
pm.hook.pytask_post_parse(config=config)
return config
def pytask_execute_log_start(session: Session) -> None:
"""Start logging."""
session.execution_start = time.time()
# New line to separate note on collected items from task statuses.
console.print()
def pytask_collect_log(
session: Session, reports: list[CollectionReport], tasks: list[Task]
) -> None:
"""Log collection."""
session.collection_end = time.time()
console.print(f"Collected {len(tasks)} task{'' if len(tasks) == 1 else 's'}.")
failed_reports = [r for r in reports if r.outcome == CollectionOutcome.FAIL]
if failed_reports:
counts = count_outcomes(reports, CollectionOutcome)
console.print()
console.rule(
Text("Failures during collection", style=CollectionOutcome.FAIL.style),
style=CollectionOutcome.FAIL.style,
)
for report in failed_reports:
if report.node is None:
header = "Error"
else:
if isinstance(report.node, Task):
short_name = format_task_id(
report.node, editor_url_scheme="no_link", short_name=True
)
else:
short_name = reduce_node_name(report.node, session.config["paths"])
header = f"Could not collect {short_name}"
console.rule(
Text(header, style=CollectionOutcome.FAIL.style),
style=CollectionOutcome.FAIL.style,
)
console.print()
console.print(
render_exc_info(*report.exc_info, session.config["show_locals"])
)
console.print()
panel = create_summary_panel(
counts, CollectionOutcome, "Collected errors and tasks"
)
console.print(panel)
session.hook.pytask_log_session_footer(
session=session,
duration=session.collection_end - session.collection_start,
outcome=CollectionOutcome.FAIL
if counts[CollectionOutcome.FAIL]
else CollectionOutcome.SUCCESS,
)
raise CollectionError
def main(config_from_cli: dict[str, Any]) -> Session:
"""Run pytask.
This is the main command to run pytask which usually receives kwargs from the
command line interface. It can also be used to run pytask interactively. Pass
configuration in a dictionary.
Parameters
----------
config_from_cli : dict[str, Any]
A dictionary with options passed to pytask. In general, this dictionary holds
the information passed via the command line interface.
Returns
-------
session : _pytask.session.Session
The session captures all the information of the current run.
"""
try:
pm = get_plugin_manager()
from _pytask import cli
pm.register(cli)
pm.hook.pytask_add_hooks(pm=pm)
config = pm.hook.pytask_configure(pm=pm,
config_from_cli=config_from_cli)
session = Session.from_config(config)
except (ConfigurationError, Exception):
exc_info = sys.exc_info()
exc_info = remove_internal_traceback_frames_from_exc_info(exc_info)
traceback = Traceback.from_exception(*exc_info)
console.print(traceback)
session = Session({}, None)
session.exit_code = ExitCode.CONFIGURATION_FAILED
else:
try:
session.hook.pytask_log_session_header(session=session)
session.hook.pytask_collect(session=session)
session.hook.pytask_resolve_dependencies(session=session)
session.hook.pytask_execute(session=session)
except CollectionError:
session.exit_code = ExitCode.COLLECTION_FAILED
except ResolvingDependenciesError:
session.exit_code = ExitCode.RESOLVING_DEPENDENCIES_FAILED
except ExecutionError:
session.exit_code = ExitCode.FAILED
except Exception:
exc_info = sys.exc_info()
exc_info = remove_internal_traceback_frames_from_exc_info(exc_info)
traceback = Traceback.from_exception(*exc_info)
console.print(traceback)
session.exit_code = ExitCode.FAILED
session.hook.pytask_unconfigure(session=session)
return session