def markers(**config_from_cli: Any) -> NoReturn: """Show all registered markers.""" config_from_cli["command"] = "markers" try: # Duplication of the same mechanism in :func:`pytask.main.main`. pm = get_plugin_manager() from _pytask import cli pm.register(cli) pm.hook.pytask_add_hooks(pm=pm) config = pm.hook.pytask_configure(pm=pm, config_from_cli=config_from_cli) session = Session.from_config(config) except (ConfigurationError, Exception): console.print_exception() session = Session({}, None) session.exit_code = ExitCode.CONFIGURATION_FAILED else: table = Table("Marker", "Description", leading=1) for name, description in config["markers"].items(): table.add_row(f"pytask.mark.{name}", description) console.print(table) sys.exit(session.exit_code)
def do_continue(self, arg): # type: ignore ret = super().do_continue(arg) if cls._recursive_debug == 0: assert cls._config is not None console.print() capman = self._pytask_capman capturing = PytaskPDB._is_capturing(capman) if capturing: console.rule( "PDB continue (IO-capturing resumed)", characters=">", style=None, ) assert capman is not None capman.resume() else: console.rule("PDB continue", characters=">", style=None) if not self._pytask_live_manager.is_started: self._pytask_live_manager.resume() assert cls._pluginmanager is not None self._continued = True return ret
def _print_errored_task_report(session: Session, report: ExecutionReport) -> None: """Print the traceback and the exception of an errored report.""" task_name = format_task_id( task=report.task, editor_url_scheme=session.config["editor_url_scheme"], short_name=True, ) text = Text.assemble("Task ", task_name, " failed", style="failed") console.rule(text, style=report.outcome.style) console.print() if report.exc_info and isinstance(report.exc_info[1], Exit): console.print(format_exception_without_traceback(report.exc_info)) else: console.print( render_exc_info(*report.exc_info, session.config["show_locals"])) console.print() show_capture = session.config["show_capture"] for when, key, content in report.sections: if key in ("stdout", "stderr") and show_capture in ( ShowCapture[key.upper()], ShowCapture.ALL, ): console.rule(f"Captured {key} during {when}", style=None) console.print(content)
def pytask_execute_task_log_end(session: Session, report: ExecutionReport) -> None: """Log task outcome.""" url_style = create_url_style_for_task(report.task.function, session.config["editor_url_scheme"]) console.print( report.outcome.symbol, style=unify_styles(report.outcome.style, url_style), end="", )
def collect(**config_from_cli: Any | None) -> NoReturn: """Collect tasks and report information about them.""" config_from_cli["command"] = "collect" try: # Duplication of the same mechanism in :func:`pytask.main.main`. pm = get_plugin_manager() from _pytask import cli pm.register(cli) pm.hook.pytask_add_hooks(pm=pm) config = pm.hook.pytask_configure(pm=pm, config_from_cli=config_from_cli) session = Session.from_config(config) except (ConfigurationError, Exception): session = Session({}, None) session.exit_code = ExitCode.CONFIGURATION_FAILED console.print_exception() else: try: session.hook.pytask_log_session_header(session=session) session.hook.pytask_collect(session=session) session.hook.pytask_resolve_dependencies(session=session) tasks = _select_tasks_by_expressions_and_marker(session) common_ancestor = _find_common_ancestor_of_all_nodes( tasks, session.config["paths"], session.config["nodes"]) dictionary = _organize_tasks(tasks) if dictionary: _print_collected_tasks( dictionary, session.config["nodes"], session.config["editor_url_scheme"], common_ancestor, ) console.print() console.rule(style="neutral") except CollectionError: session.exit_code = ExitCode.COLLECTION_FAILED except ResolvingDependenciesError: session.exit_code = ExitCode.RESOLVING_DEPENDENCIES_FAILED except Exception: session.exit_code = ExitCode.FAILED console.print_exception() console.rule(style="failed") sys.exit(session.exit_code)
def pytask_execute_build(self) -> Generator[None, None, None]: """Wrap the execution with the live manager and yield a complete table at the end.""" self.live_manager.start() yield self.live_manager.stop(transient=True) table = self._generate_table(reduce_table=False, sort_table=self.sort_final_table, add_caption=False) if table is not None: console.print(table)
def profile(**config_from_cli: Any) -> NoReturn: """Show information about tasks like runtime and memory consumption of products.""" config_from_cli["command"] = "profile" try: # Duplication of the same mechanism in :func:`pytask.main.main`. pm = get_plugin_manager() from _pytask import cli pm.register(cli) pm.hook.pytask_add_hooks(pm=pm) config = pm.hook.pytask_configure(pm=pm, config_from_cli=config_from_cli) session = Session.from_config(config) except (ConfigurationError, Exception): # pragma: no cover session = Session({}, None) session.exit_code = ExitCode.CONFIGURATION_FAILED exc_info: tuple[ type[BaseException], BaseException, TracebackType | None ] = sys.exc_info() console.print(render_exc_info(*exc_info, show_locals=config["show_locals"])) else: try: session.hook.pytask_log_session_header(session=session) session.hook.pytask_collect(session=session) session.hook.pytask_resolve_dependencies(session=session) profile: dict[str, dict[str, Any]] = { task.name: {} for task in session.tasks } session.hook.pytask_profile_add_info_on_task( session=session, tasks=session.tasks, profile=profile ) profile = _process_profile(profile) _print_profile_table(profile, session.tasks, session.config) session.hook.pytask_profile_export_profile(session=session, profile=profile) console.rule(style="neutral") except CollectionError: # pragma: no cover session.exit_code = ExitCode.COLLECTION_FAILED except Exception: # pragma: no cover session.exit_code = ExitCode.FAILED console.print_exception() console.rule(style="failed") sys.exit(session.exit_code)
def pytask_log_session_footer(session: Session) -> None: """Log warnings at the end of a session.""" if session.warnings: grouped_warnings = defaultdict(list) for warning in session.warnings: location = (warning.id_ if warning.id_ is not None else "{}:{}".format(*warning.fs_location)) grouped_warnings[warning.message].append(location) sorted_gw = {k: sorted(v) for k, v in grouped_warnings.items()} reduced_gw = _reduce_grouped_warnings(sorted_gw) renderable = MyRenderable(reduced_gw) panel = Panel(renderable, title="Warnings", style="warning") console.print(panel)
def dag(**config_from_cli: Any) -> NoReturn: """Create a visualization of the project's directed acyclic graph.""" try: pm = get_plugin_manager() from _pytask import cli pm.register(cli) pm.hook.pytask_add_hooks(pm=pm) config = pm.hook.pytask_configure(pm=pm, config_from_cli=config_from_cli) session = Session.from_config(config) except (ConfigurationError, Exception): console.print_exception() session = Session({}, None) session.exit_code = ExitCode.CONFIGURATION_FAILED else: try: session.hook.pytask_log_session_header(session=session) import_optional_dependency("pydot") check_for_optional_program( session.config["layout"], extra="The layout program is part of the graphviz package which you " "can install with conda.", ) session.hook.pytask_collect(session=session) session.hook.pytask_resolve_dependencies(session=session) dag = _refine_dag(session) _write_graph(dag, session.config["output_path"], session.config["layout"]) except CollectionError: session.exit_code = ExitCode.COLLECTION_FAILED except ResolvingDependenciesError: session.exit_code = ExitCode.RESOLVING_DEPENDENCIES_FAILED except Exception: session.exit_code = ExitCode.FAILED exc_info = remove_internal_traceback_frames_from_exc_info(sys.exc_info()) console.print() console.print(Traceback.from_exception(*exc_info)) console.rule(style="failed") sys.exit(session.exit_code)
def _print_profile_table( profile: dict[str, dict[str, Any]], tasks: list[Task], config: dict[str, Any] ) -> None: """Print the profile table.""" name_to_task = {task.name: task for task in tasks} info_names = _get_info_names(profile) console.print() if profile: table = Table("Task") for name in info_names: table.add_column(name, justify="right") for task_name, info in profile.items(): task_id = format_task_id( task=name_to_task[task_name], editor_url_scheme=config["editor_url_scheme"], short_name=True, ) infos = [str(i) for i in info.values()] table.add_row(task_id, *infos) console.print(table) else: console.print("No information is stored on the collected tasks.")
def wrapper(*args: Any, **kwargs: Any) -> None: capman = session.config["pm"].get_plugin("capturemanager") live_manager = session.config["pm"].get_plugin("live_manager") # Order is important! Pausing the live object before the capturemanager would # flush the table to stdout and it will be visible in the captured output. capman.suspend(in_=True) out, err = capman.read() live_manager.stop() if out or err: console.print() if out: console.rule("Captured stdout", style=None) console.print(out) if err: console.rule("Captured stderr", style=None) console.print(err) _pdb.runcall(task_function, *args, **kwargs) live_manager.resume() capman.resume()
def wrapper(*args: Any, **kwargs: Any) -> None: capman = session.config["pm"].get_plugin("capturemanager") live_manager = session.config["pm"].get_plugin("live_manager") try: task_function(*args, **kwargs) except Exception: # Order is important! Pausing the live object before the capturemanager # would flush the table to stdout and it will be visible in the captured # output. capman.suspend(in_=True) out, err = capman.read() live_manager.pause() if out or err: console.print() if out: console.rule("Captured stdout", style=None) console.print(out) if err: console.rule("Captured stderr", style=None) console.print(err) exc_info = remove_internal_traceback_frames_from_exc_info( sys.exc_info()) console.print() console.rule("Traceback", characters=">", style=None) console.print( render_exc_info(*exc_info, session.config["show_locals"])) post_mortem(exc_info[2]) live_manager.resume() capman.resume() raise
def pytask_log_session_header(session: Session) -> None: """Log the header of a pytask session.""" console.rule("Start pytask session", style=None) console.print( f"Platform: {sys.platform} -- Python {platform.python_version()}, " f"pytask {_pytask.__version__}, pluggy {pluggy.__version__}") console.print(f"Root: {session.config['root']}") if session.config["config"] is not None: console.print(f"Configuration: {session.config['config']}") plugin_info = session.config["pm"].list_plugin_distinfo() if plugin_info: formatted_plugins_w_versions = ", ".join( _format_plugin_names_and_versions(plugin_info)) console.print(f"Plugins: {formatted_plugins_w_versions}")
def pytask_resolve_dependencies_log( session: Session, report: ResolvingDependenciesReport) -> None: """Log errors which happened while resolving dependencies.""" console.print() console.rule( Text("Failures during resolving dependencies", style="failed"), style="failed", ) console.print() console.print( render_exc_info(*report.exc_info, session.config["show_locals"])) console.print() console.rule(style="failed")
def _init_pdb(cls, method: str, *args: Any, **kwargs: Any) -> pdb.Pdb: # noqa: U100 """Initialize PDB debugging, dropping any IO capturing.""" if cls._pluginmanager is None: capman = None live_manager = None else: capman = cls._pluginmanager.get_plugin("capturemanager") live_manager = cls._pluginmanager.get_plugin("live_manager") if capman: capman.suspend(in_=True) if live_manager: live_manager.pause() if cls._config: console.print() if cls._recursive_debug == 0: # Handle header similar to pdb.set_trace in py37+. header = kwargs.pop("header", None) if header is not None: console.rule(header, characters=">", style=None) else: capturing = cls._is_capturing(capman) if capturing: console.rule( f"PDB {method} (IO-capturing turned off)", characters=">", style=None, ) else: console.rule(f"PDB {method}", characters=">", style=None) _pdb = cls._import_pdb_cls(capman, live_manager)(**kwargs) return _pdb
def pytask_execute_log_end(session: Session, reports: list[ExecutionReport]) -> bool: """Log information on the execution.""" session.execution_end = time.time() counts = count_outcomes(reports, TaskOutcome) if session.config["show_traceback"]: console.print() if counts[TaskOutcome.FAIL]: console.rule( Text("Failures", style=TaskOutcome.FAIL.style), style=TaskOutcome.FAIL.style, ) console.print() for report in reports: if report.outcome in (TaskOutcome.FAIL, TaskOutcome.SKIP_PREVIOUS_FAILED): _print_errored_task_report(session, report) console.rule(style="dim") panel = create_summary_panel(counts, TaskOutcome, "Collected tasks") console.print(panel) session.hook.pytask_log_session_footer( session=session, duration=session.execution_end - session.execution_start, outcome=TaskOutcome.FAIL if counts[TaskOutcome.FAIL] else TaskOutcome.SUCCESS, ) if counts[TaskOutcome.FAIL]: raise ExecutionError return True
def pytask_log_session_header(session): """Add a note for how many workers are spawned.""" n_workers = session.config["n_workers"] if n_workers > 1: console.print(f"Started {n_workers} workers.")
def clean(**config_from_cli: Any) -> NoReturn: """Clean the provided paths by removing files unknown to pytask.""" config_from_cli["command"] = "clean" try: # Duplication of the same mechanism in :func:`pytask.main.main`. pm = get_plugin_manager() from _pytask import cli pm.register(cli) pm.hook.pytask_add_hooks(pm=pm) config = pm.hook.pytask_configure(pm=pm, config_from_cli=config_from_cli) session = Session.from_config(config) except Exception: session = Session({}, None) session.exit_code = ExitCode.CONFIGURATION_FAILED exc_info: tuple[type[BaseException], BaseException, TracebackType | None] = sys.exc_info() console.print(render_exc_info(*exc_info)) else: try: session.hook.pytask_log_session_header(session=session) session.hook.pytask_collect(session=session) known_paths = _collect_all_paths_known_to_pytask(session) exclude = session.config["exclude"] include_directories = session.config["directories"] unknown_paths = _find_all_unknown_paths(session, known_paths, exclude, include_directories) common_ancestor = find_common_ancestor(*unknown_paths, *session.config["paths"]) if unknown_paths: targets = "Files" if session.config["directories"]: targets += " and directories" console.print(f"\n{targets} which can be removed:\n") for path in unknown_paths: short_path = relative_to(path, common_ancestor) if session.config["mode"] == "dry-run": console.print(f"Would remove {short_path}") else: should_be_deleted = session.config[ "mode"] == "force" or click.confirm( f"Would you like to remove {short_path}?") if should_be_deleted: if not session.config["quiet"]: console.print(f"Remove {short_path}") if path.is_dir(): shutil.rmtree(path) else: path.unlink() else: console.print() console.print( "There are no files and directories which can be deleted.") console.print() console.rule(style=None) except CollectionError: session.exit_code = ExitCode.COLLECTION_FAILED console.rule(style="failed") except Exception: exc_info = sys.exc_info() console.print( render_exc_info(*exc_info, show_locals=config["show_locals"])) console.rule(style="failed") session.exit_code = ExitCode.FAILED sys.exit(session.exit_code)
def _print_collected_tasks( dictionary: dict[Path, list[Task]], show_nodes: bool, editor_url_scheme: str, common_ancestor: Path, ) -> None: """Print the information on collected tasks. Parameters ---------- dictionary : Dict[Path, List["Task"]] A dictionary with path on the first level, tasks on the second, dependencies and products on the third. show_nodes : bool Indicator for whether dependencies and products should be displayed. editor_url_scheme : str The scheme to create an url. common_ancestor : Path The path common to all tasks and nodes. """ # Have a new line between the number of collected tasks and this info. console.print() tree = Tree("Collected tasks:", highlight=True) for module, tasks in dictionary.items(): reduced_module = relative_to(module, common_ancestor) url_style = create_url_style_for_path(module, editor_url_scheme) module_branch = tree.add( Text.assemble(PYTHON_ICON, "<Module ", Text(str(reduced_module), style=url_style), ">")) for task in tasks: reduced_task_name = format_task_id( task, editor_url_scheme=editor_url_scheme, relative_to=common_ancestor) task_branch = module_branch.add( Text.assemble(TASK_ICON, "<Function ", reduced_task_name, ">"), ) if show_nodes: for node in sorted(tree_just_flatten(task.depends_on), key=lambda x: x.path): reduced_node_name = relative_to(node.path, common_ancestor) url_style = create_url_style_for_path( node.path, editor_url_scheme) task_branch.add( Text.assemble( FILE_ICON, "<Dependency ", Text(str(reduced_node_name), style=url_style), ">", )) for node in sorted(tree_just_flatten(task.produces), key=lambda x: x.path): reduced_node_name = relative_to(node.path, common_ancestor) url_style = create_url_style_for_path( node.path, editor_url_scheme) task_branch.add( Text.assemble( FILE_ICON, "<Product ", Text(str(reduced_node_name), style=url_style), ">", )) console.print(tree)
def pytask_configure(pm: pluggy.PluginManager, config_from_cli: dict[str, Any]) -> dict[str, Any]: """Configure pytask.""" config = {"pm": pm} # Either the path to the configuration is passed via the CLI or it needs to be # detected from the paths passed to pytask. if config_from_cli.get("config"): config["config"] = Path(config_from_cli["config"]) config["root"] = config["config"].parent else: paths = (parse_paths(config_from_cli.get("paths")) if config_from_cli.get("paths") is not None else [Path.cwd()]) config["root"], config["config"] = _find_project_root_and_config(paths) if config["config"] is None: config_from_file = {} else: read_config = get_config_reader(config["config"]) config_from_file = read_config(config["config"]) if read_config.__name__ == "_read_ini_config": toml_string = "# Content of pyproject.toml\n\n" + tomli_w.dumps( {"tool": { "pytask": { "ini_options": config_from_file } }}) console.print( Text( _DEPRECATION_MESSAGE.format( config["config"].with_name("pyproject.toml")), style="warning", )) console.print(Syntax(toml_string, "toml")) # If paths are set in the configuration, process them. if config_from_file.get("paths"): paths_from_file = to_list( parse_value_or_multiline_option(config_from_file.get("paths"))) config_from_file["paths"] = [ config["config"].parent.joinpath(p).resolve() for p in paths_from_file ] config["paths"] = get_first_non_none_value( config_from_cli, config_from_file, key="paths", default=[Path.cwd()], callback=parse_paths, ) config["markers"] = { "depends_on": ("Add dependencies to a task. See this tutorial for more information: " "[link https://bit.ly/3JlxylS]https://bit.ly/3JlxylS[/]."), "produces": ("Add products to a task. See this tutorial for more information: " "[link https://bit.ly/3JlxylS]https://bit.ly/3JlxylS[/]."), "try_first": "Try to execute a task a early as possible.", "try_last": "Try to execute a task a late as possible.", } pm.hook.pytask_parse_config( config=config, config_from_cli=config_from_cli, config_from_file=config_from_file, ) pm.hook.pytask_post_parse(config=config) return config
def pytask_execute_log_start(session: Session) -> None: """Start logging.""" session.execution_start = time.time() # New line to separate note on collected items from task statuses. console.print()
def pytask_collect_log( session: Session, reports: list[CollectionReport], tasks: list[Task] ) -> None: """Log collection.""" session.collection_end = time.time() console.print(f"Collected {len(tasks)} task{'' if len(tasks) == 1 else 's'}.") failed_reports = [r for r in reports if r.outcome == CollectionOutcome.FAIL] if failed_reports: counts = count_outcomes(reports, CollectionOutcome) console.print() console.rule( Text("Failures during collection", style=CollectionOutcome.FAIL.style), style=CollectionOutcome.FAIL.style, ) for report in failed_reports: if report.node is None: header = "Error" else: if isinstance(report.node, Task): short_name = format_task_id( report.node, editor_url_scheme="no_link", short_name=True ) else: short_name = reduce_node_name(report.node, session.config["paths"]) header = f"Could not collect {short_name}" console.rule( Text(header, style=CollectionOutcome.FAIL.style), style=CollectionOutcome.FAIL.style, ) console.print() console.print( render_exc_info(*report.exc_info, session.config["show_locals"]) ) console.print() panel = create_summary_panel( counts, CollectionOutcome, "Collected errors and tasks" ) console.print(panel) session.hook.pytask_log_session_footer( session=session, duration=session.collection_end - session.collection_start, outcome=CollectionOutcome.FAIL if counts[CollectionOutcome.FAIL] else CollectionOutcome.SUCCESS, ) raise CollectionError
def main(config_from_cli: dict[str, Any]) -> Session: """Run pytask. This is the main command to run pytask which usually receives kwargs from the command line interface. It can also be used to run pytask interactively. Pass configuration in a dictionary. Parameters ---------- config_from_cli : dict[str, Any] A dictionary with options passed to pytask. In general, this dictionary holds the information passed via the command line interface. Returns ------- session : _pytask.session.Session The session captures all the information of the current run. """ try: pm = get_plugin_manager() from _pytask import cli pm.register(cli) pm.hook.pytask_add_hooks(pm=pm) config = pm.hook.pytask_configure(pm=pm, config_from_cli=config_from_cli) session = Session.from_config(config) except (ConfigurationError, Exception): exc_info = sys.exc_info() exc_info = remove_internal_traceback_frames_from_exc_info(exc_info) traceback = Traceback.from_exception(*exc_info) console.print(traceback) session = Session({}, None) session.exit_code = ExitCode.CONFIGURATION_FAILED else: try: session.hook.pytask_log_session_header(session=session) session.hook.pytask_collect(session=session) session.hook.pytask_resolve_dependencies(session=session) session.hook.pytask_execute(session=session) except CollectionError: session.exit_code = ExitCode.COLLECTION_FAILED except ResolvingDependenciesError: session.exit_code = ExitCode.RESOLVING_DEPENDENCIES_FAILED except ExecutionError: session.exit_code = ExitCode.FAILED except Exception: exc_info = sys.exc_info() exc_info = remove_internal_traceback_frames_from_exc_info(exc_info) traceback = Traceback.from_exception(*exc_info) console.print(traceback) session.exit_code = ExitCode.FAILED session.hook.pytask_unconfigure(session=session) return session