def pytask_execute_task_setup(task: Task) -> None:
"""Take a short-cut for skipped tasks during setup with an exception."""
is_unchanged = has_mark(task, "skip_unchanged")
if is_unchanged:
raise SkippedUnchanged
ancestor_failed_marks = get_marks(task, "skip_ancestor_failed")
if ancestor_failed_marks:
message = "\n".join(
skip_ancestor_failed(*mark.args, **mark.kwargs)
for mark in ancestor_failed_marks)
raise SkippedAncestorFailed(message)
is_skipped = has_mark(task, "skip")
if is_skipped:
raise Skipped
skipif_marks = get_marks(task, "skipif")
if skipif_marks:
marker_args = [
skipif(*mark.args, **mark.kwargs) for mark in skipif_marks
]
message = "\n".join(arg[1] for arg in marker_args if arg[0])
should_skip = any(arg[0] for arg in marker_args)
if should_skip:
raise Skipped(message)
def pytask_collect_task(
session: Session, path: Path, name: str, obj: Any
) -> Task | None:
"""Collect a task which is a function.
There is some discussion on how to detect functions in this `thread
<https://stackoverflow.com/q/624926/7523785>`_. :class:`types.FunctionType` does not
detect built-ins which is not possible anyway.
"""
if (name.startswith("task_") or has_mark(obj, "task")) and callable(obj):
dependencies = parse_nodes(session, path, name, obj, depends_on)
products = parse_nodes(session, path, name, obj, produces)
markers = obj.pytask_meta.markers if hasattr(obj, "pytask_meta") else []
kwargs = obj.pytask_meta.kwargs if hasattr(obj, "pytask_meta") else {}
# Get the underlying function to avoid having different states of the function,
# e.g. due to pytask_meta, in different layers of the wrapping.
unwrapped = inspect.unwrap(obj)
return Task(
base_name=name,
path=path,
function=unwrapped,
depends_on=dependencies,
produces=products,
markers=markers,
kwargs=kwargs,
)
else:
return None
def _extract_priorities_from_tasks(tasks: list[Task]) -> dict[str, int]:
"""Extract priorities from tasks.
Priorities are set via the ``pytask.mark.try_first`` and ``pytask.mark.try_last``
markers. We recode these markers to numeric values to sort all available by
priorities. ``try_first`` is assigned the highest value such that it has the
rightmost position in the list. Then, we can simply call :meth:`list.pop` on the
list which is far more efficient than ``list.pop(0)``.
"""
priorities = {
task.name: {
"try_first": has_mark(task, "try_first"),
"try_last": has_mark(task, "try_last"),
}
for task in tasks
}
tasks_w_mixed_priorities = [
name for name, p in priorities.items()
if p["try_first"] and p["try_last"]
]
if tasks_w_mixed_priorities:
name_to_task = {task.name: task for task in tasks}
reduced_names = [
name_to_task[name].short_name for name in tasks_w_mixed_priorities
]
text = format_strings_as_flat_tree(reduced_names,
"Tasks with mixed priorities",
TASK_ICON)
raise ValueError(
"'try_first' and 'try_last' cannot be applied on the same task. See the "
f"following tasks for errors:\n\n{text}")
# Recode to numeric values for sorting.
numeric_mapping = {(True, False): 1, (False, False): 0, (False, True): -1}
numeric_priorities = {
name: numeric_mapping[(p["try_first"], p["try_last"])]
for name, p in priorities.items()
}
return numeric_priorities
def _check_if_task_is_skipped(task_name: str, dag: nx.DiGraph) -> bool:
task = dag.nodes[task_name]["task"]
is_skipped = has_mark(task, "skip")
if is_skipped:
return True
skip_if_markers = get_marks(task, "skipif")
is_any_true = any(
_skipif(*marker.args, **marker.kwargs)[0]
for marker in skip_if_markers)
return is_any_true
def pytask_collect_file(
session: Session, path: Path, reports: list[CollectionReport]
) -> list[CollectionReport] | None:
"""Collect a file."""
if any(path.match(pattern) for pattern in session.config["task_files"]):
spec = importlib_util.spec_from_file_location(path.stem, str(path))
if spec is None:
raise ImportError(f"Can't find module {path.stem!r} at location {path}.")
mod = importlib_util.module_from_spec(spec)
spec.loader.exec_module(mod)
collected_reports = []
for name, obj in inspect.getmembers(mod):
# Ensures that tasks with this decorator are only collected once.
if has_mark(obj, "task"):
continue
if has_mark(obj, "parametrize"):
names_and_objects = session.hook.pytask_parametrize_task(
session=session, name=name, obj=obj
)
else:
names_and_objects = [(name, obj)]
for name_, obj_ in names_and_objects:
report = session.hook.pytask_collect_task_protocol(
session=session, reports=reports, path=path, name=name_, obj=obj_
)
if report is not None:
collected_reports.append(report)
return collected_reports
else:
return None
def pytask_execute_task_setup(session: Session, task: Task) -> None:
"""Exit persisting tasks early.
The decorator needs to be set and all nodes need to exist.
"""
if has_mark(task, "persist"):
try:
for name in node_and_neighbors(session.dag, task.name):
node = (session.dag.nodes[name].get("task")
or session.dag.nodes[name]["node"])
node.state()
except NodeNotFoundError:
all_nodes_exist = False
else:
all_nodes_exist = True
if all_nodes_exist:
raise Persisted
def pytask_collect_file(
session: Session, path: Path,
reports: list[CollectionReport]) -> list[CollectionReport] | None:
"""Collect a file."""
if (any(path.match(pattern) for pattern in session.config["task_files"])
and COLLECTED_TASKS[path]):
# Remove tasks from the global to avoid re-collection if programmatic interface
# is used.
tasks = COLLECTED_TASKS.pop(path)
name_to_function = parse_collected_tasks_with_task_marker(tasks)
collected_reports = []
for name, function in name_to_function.items():
session.hook.pytask_parametrize_kwarg_to_marker(
obj=function,
kwargs=function.pytask_meta.
kwargs, # type: ignore[attr-defined]
)
if has_mark(function, "parametrize"):
names_and_objects = session.hook.pytask_parametrize_task(
session=session, name=name, obj=function)
else:
names_and_objects = [(name, function)]
for name_, obj_ in names_and_objects:
report = session.hook.pytask_collect_task_protocol(
session=session,
reports=reports,
path=path,
name=name_,
obj=obj_)
if report is not None:
collected_reports.append(report)
return collected_reports
else:
return None