def test_count_outcomes_tasks(outcome_in_report):
reports = [ExecutionReport(None, outcome_in_report, None, None)]
counts = count_outcomes(reports, TaskOutcome)
for outcome, count in counts.items():
if outcome == outcome_in_report:
assert count == 1
else:
assert count == 0
def test_live_execution_sequentially(capsys, tmp_path):
path = tmp_path.joinpath("task_module.py")
task = Task(base_name="task_example", path=path, function=lambda x: x)
task.short_name = "task_module.py::task_example"
live_manager = LiveManager()
live = LiveExecution(
live_manager=live_manager,
n_entries_in_table=20,
verbose=1,
editor_url_scheme="no_link",
)
live_manager.start()
live.update_running_tasks(task)
live_manager.pause()
# Test pause removes the table.
captured = capsys.readouterr()
assert "Task" not in captured.out
assert "Outcome" not in captured.out
assert "task_module.py::task_example" not in captured.out
assert "running" not in captured.out
assert "Completed: 0/x" not in captured.out
live_manager.resume()
live_manager.start()
live_manager.stop()
# Test table with running task.
captured = capsys.readouterr()
assert "Task" in captured.out
assert "Outcome" in captured.out
assert "task_module.py::task_example" in captured.out
assert "running" in captured.out
assert "Completed: 0/x" in captured.out
live_manager.start()
report = ExecutionReport(task=task,
outcome=TaskOutcome.SUCCESS,
exc_info=None)
live_manager.resume()
live.update_reports(report)
live_manager.stop()
# Test final table with reported outcome.
captured = capsys.readouterr()
assert "Task" in captured.out
assert "Outcome" in captured.out
assert "task_module.py::task_example" in captured.out
assert "running" not in captured.out
assert TaskOutcome.SUCCESS.symbol in captured.out
assert "Completed: 1/x" in captured.out
def test_live_execution_displays_subset_of_table(capsys, tmp_path,
n_entries_in_table):
path = tmp_path.joinpath("task_module.py")
running_task = Task(base_name="task_running",
path=path,
function=lambda x: x)
running_task.short_name = "task_module.py::task_running"
live_manager = LiveManager()
live = LiveExecution(
live_manager=live_manager,
n_entries_in_table=n_entries_in_table,
verbose=1,
editor_url_scheme="no_link",
n_tasks=2,
)
live_manager.start()
live.update_running_tasks(running_task)
live_manager.stop(transient=False)
captured = capsys.readouterr()
assert "Task" in captured.out
assert "Outcome" in captured.out
assert "::task_running" in captured.out
assert " running " in captured.out
assert "Completed: 0/2" in captured.out
completed_task = Task(base_name="task_completed",
path=path,
function=lambda x: x)
completed_task.short_name = "task_module.py::task_completed"
live.update_running_tasks(completed_task)
report = ExecutionReport(task=completed_task,
outcome=TaskOutcome.SUCCESS,
exc_info=None)
live_manager.resume()
live.update_reports(report)
live_manager.stop()
# Test that report is or is not included.
captured = capsys.readouterr()
assert "Task" in captured.out
assert "Outcome" in captured.out
assert "::task_running" in captured.out
assert " running " in captured.out
assert "Completed: 1/2" in captured.out
if n_entries_in_table == 1:
assert "task_module.py::task_completed" not in captured.out
assert "│ ." not in captured.out
else:
assert "task_module.py::task_completed" in captured.out
assert "│ ." in captured.out
def pytask_execute_task_process_report(session: Session,
report: ExecutionReport) -> bool | None:
"""Set task status to success.
Do not return ``True`` so that states will be updated in database.
"""
if report.exc_info and isinstance(report.exc_info[1], Persisted):
report.outcome = TaskOutcome.PERSISTENCE
update_states_in_database(session.dag, report.task.name)
return True
return None
def pytask_execute_task_process_report(session: Session,
report: ExecutionReport) -> bool | None:
"""Process the execution reports for skipped tasks.
This functions allows to turn skipped tasks to successful tasks.
"""
task = report.task
if report.exc_info:
if isinstance(report.exc_info[1], SkippedUnchanged):
report.outcome = TaskOutcome.SKIP_UNCHANGED
elif isinstance(report.exc_info[1], Skipped):
report.outcome = TaskOutcome.SKIP
for descending_task_name in descending_tasks(
task.name, session.dag):
descending_task = session.dag.nodes[descending_task_name][
"task"]
descending_task.markers.append(
Mark(
"skip",
(),
{
"reason":
f"Previous task {task.name!r} was skipped."
},
))
elif isinstance(report.exc_info[1], SkippedAncestorFailed):
report.outcome = TaskOutcome.SKIP_PREVIOUS_FAILED
report.exc_info = remove_traceback_from_exc_info(report.exc_info)
if report.exc_info and isinstance(
report.exc_info[1],
(Skipped, SkippedUnchanged, SkippedAncestorFailed)):
return True
else:
return None
def pytask_execute_task_protocol(session: Session,
task: Task) -> ExecutionReport:
"""Follow the protocol to execute each task."""
session.hook.pytask_execute_task_log_start(session=session, task=task)
try:
session.hook.pytask_execute_task_setup(session=session, task=task)
session.hook.pytask_execute_task(session=session, task=task)
session.hook.pytask_execute_task_teardown(session=session, task=task)
except KeyboardInterrupt:
short_exc_info = remove_traceback_from_exc_info(sys.exc_info())
report = ExecutionReport.from_task_and_exception(task, short_exc_info)
session.should_stop = True
except Exception:
report = ExecutionReport.from_task_and_exception(task, sys.exc_info())
else:
report = ExecutionReport.from_task(task)
session.hook.pytask_execute_task_process_report(session=session,
report=report)
session.hook.pytask_execute_task_log_end(session=session,
task=task,
report=report)
return report
def test_live_execution_displays_skips_and_persists(capsys, tmp_path, verbose,
outcome):
path = tmp_path.joinpath("task_module.py")
task = Task(base_name="task_example", path=path, function=lambda x: x)
task.short_name = "task_module.py::task_example"
live_manager = LiveManager()
live = LiveExecution(
live_manager=live_manager,
n_entries_in_table=20,
verbose=verbose,
editor_url_scheme="no_link",
)
live_manager.start()
live.update_running_tasks(task)
live_manager.pause()
report = ExecutionReport(task=task, outcome=outcome, exc_info=None)
live_manager.resume()
live.update_reports(report)
live_manager.stop()
# Test final table with reported outcome.
captured = capsys.readouterr()
if verbose < 2 and outcome in (
TaskOutcome.SKIP,
TaskOutcome.SKIP_UNCHANGED,
TaskOutcome.SKIP_PREVIOUS_FAILED,
TaskOutcome.PERSISTENCE,
):
# An empty table is not shown.
assert "Task" not in captured.out
assert "Outcome" not in captured.out
assert "task_module.py::task_example" not in captured.out
assert f"│ {outcome.symbol}" not in captured.out
else:
assert "Task" in captured.out
assert "Outcome" in captured.out
assert "task_module.py::task_example" in captured.out
assert f"│ {outcome.symbol}" in captured.out
assert "running" not in captured.out
def pytask_execute_build(session):
"""Execute tasks with a parallel backend.
There are three phases while the scheduler has tasks which need to be executed.
1. Take all ready tasks, set up their execution and submit them.
2. For all tasks which are running, find those which have finished and turn them
into a report.
3. Process all reports and report the result on the command line.
"""
if session.config["n_workers"] > 1:
reports = session.execution_reports
running_tasks = {}
parallel_backend = PARALLEL_BACKENDS[
session.config["parallel_backend"]]
with parallel_backend(
max_workers=session.config["n_workers"]) as executor:
session.executor = executor
while session.scheduler.is_active():
try:
newly_collected_reports = []
n_new_tasks = session.config["n_workers"] - len(
running_tasks)
if n_new_tasks >= 1:
ready_tasks = list(
session.scheduler.get_ready(n_new_tasks))
else:
ready_tasks = []
for task_name in ready_tasks:
task = session.dag.nodes[task_name]["task"]
session.hook.pytask_execute_task_log_start(
session=session, task=task)
try:
session.hook.pytask_execute_task_setup(
session=session, task=task)
except Exception:
report = ExecutionReport.from_task_and_exception(
task, sys.exc_info())
newly_collected_reports.append(report)
session.scheduler.done(task_name)
else:
running_tasks[
task_name] = session.hook.pytask_execute_task(
session=session, task=task)
for task_name in list(running_tasks):
future = running_tasks[task_name]
if future.done() and (future.exception() is not None
or future.result() is not None):
task = session.dag.nodes[task_name]["task"]
if future.exception() is not None:
exception = future.exception()
exc_info = (
type(exception),
exception,
exception.__traceback__,
)
else:
exc_info = future.result()
newly_collected_reports.append(
ExecutionReport.from_task_and_exception(
task, exc_info))
running_tasks.pop(task_name)
session.scheduler.done(task_name)
elif future.done() and future.exception() is None:
task = session.dag.nodes[task_name]["task"]
try:
session.hook.pytask_execute_task_teardown(
session=session, task=task)
except Exception:
report = ExecutionReport.from_task_and_exception(
task, sys.exc_info())
else:
report = ExecutionReport.from_task(task)
running_tasks.pop(task_name)
newly_collected_reports.append(report)
session.scheduler.done(task_name)
else:
pass
for report in newly_collected_reports:
session.hook.pytask_execute_task_process_report(
session=session, report=report)
session.hook.pytask_execute_task_log_end(
session=session, task=task, report=report)
reports.append(report)
if session.should_stop:
break
else:
time.sleep(session.config["delay"])
except KeyboardInterrupt:
break
return True
def test_live_execution_skips_do_not_crowd_out_displayed_tasks(
capsys, tmp_path):
path = tmp_path.joinpath("task_module.py")
task = Task(base_name="task_example", path=path, function=lambda x: x)
task.short_name = "task_module.py::task_example"
live_manager = LiveManager()
live = LiveExecution(
live_manager=live_manager,
n_entries_in_table=20,
verbose=1,
editor_url_scheme="no_link",
)
live_manager.start()
live.update_running_tasks(task)
live_manager.stop()
# Test table with running task.
captured = capsys.readouterr()
assert "Task" in captured.out
assert "Outcome" in captured.out
assert "task_module.py::task_example" in captured.out
assert "running" in captured.out
# Add one displayed reports and many more not displayed reports to crowd out the
# valid one.
successful_task = Task(base_name="task_success",
path=path,
function=lambda x: x)
successful_task.short_name = "task_module.py::task_success"
tasks = []
for i in range(25):
skipped_task = Task(base_name=f"task_skip_{i}",
path=path,
function=lambda x: x)
skipped_task.short_name = f"task_module.py::task_skip_{i}"
tasks.append(skipped_task)
live_manager.start()
live.update_running_tasks(successful_task)
for task in tasks:
live.update_running_tasks(task)
live_manager.stop()
captured = capsys.readouterr()
assert "running" in captured.out
assert "task_success" in captured.out
for i in range(25):
assert f"task_skip_{i}" in captured.out
live_manager.resume()
report = ExecutionReport(task=successful_task,
outcome=TaskOutcome.SUCCESS,
exc_info=None)
live.update_reports(report)
for task in tasks:
report = ExecutionReport(task=task,
outcome=TaskOutcome.SKIP,
exc_info=None)
live.update_reports(report)
live_manager.stop()
# Test final table with reported outcome.
captured = capsys.readouterr()
assert "Task" in captured.out
assert "Outcome" in captured.out
assert "task_module.py::task_example" in captured.out
assert "task_module.py::task_success" in captured.out
assert "running" in captured.out
assert TaskOutcome.SUCCESS.symbol in captured.out
assert "task_skip" not in captured.out