def test_resource_cleanup_trigger():
def generate(init, setup, task):
return {
Edge(Task(), Task(), key="mgr"): init,
Edge(Task(), Task(), key="resource"): setup,
Edge(Task(), Task()): task,
}
assert resource_cleanup_trigger(generate(Success(), Success(), Success()))
assert resource_cleanup_trigger(generate(Success(), Success(), Failed()))
assert resource_cleanup_trigger(generate(Success(), Success(), Skipped()))
# Not all finished
with pytest.raises(signals.TRIGGERFAIL):
resource_cleanup_trigger(generate(Success(), Success(), Pending()))
with pytest.raises(signals.SKIP, match="init failed"):
resource_cleanup_trigger(generate(Failed(), Success(), Success()))
with pytest.raises(signals.SKIP, match="init skipped"):
resource_cleanup_trigger(generate(Skipped(), Success(), Success()))
with pytest.raises(signals.SKIP, match="setup failed"):
resource_cleanup_trigger(generate(Success(), Failed(), Success()))
with pytest.raises(signals.SKIP, match="setup skipped"):
resource_cleanup_trigger(generate(Success(), Skipped(), Success()))
def check_task_ready_to_map(self, state: State,
upstream_states: Dict[Edge, State]) -> State:
"""
Checks if the parent task is ready to proceed with mapping.
Args:
- state (State): the current state of this task
- upstream_states (Dict[Edge, Union[State, List[State]]]): the upstream states
Raises:
- ENDRUN: either way, we dont continue past this point
"""
if state.is_mapped():
raise ENDRUN(state)
# we can't map if there are no success states with iterables upstream
if upstream_states and not any([
edge.mapped and state.is_successful()
for edge, state in upstream_states.items()
]):
new_state = Failed(
"No upstream states can be mapped over.") # type: State
raise ENDRUN(new_state)
elif not all([
hasattr(state.result, "__getitem__")
for edge, state in upstream_states.items() if
state.is_successful() and not state.is_mapped() and edge.mapped
]):
new_state = Failed(
"At least one upstream state has an unmappable result.")
raise ENDRUN(new_state)
else:
new_state = Mapped("Ready to proceed with mapping.")
raise ENDRUN(new_state)
def check_task_ready_to_map(self, state: State,
upstream_states: Dict[Edge, State]) -> State:
"""
Checks if the parent task is ready to proceed with mapping.
Args:
- state (State): the current state of this task
- upstream_states (Dict[Edge, Union[State, List[State]]]): the upstream states
Raises:
- ENDRUN: either way, we dont continue past this point
"""
if state.is_mapped():
# this indicates we are executing a re-run of a mapped pipeline;
# in this case, we populate both `map_states` and `cached_inputs`
# to ensure the flow runner can properly regenerate the child tasks,
# regardless of whether we mapped over an exchanged piece of data
# or a non-data-exchanging upstream dependency
if len(state.map_states
) == 0 and state.n_map_states > 0: # type: ignore
state.map_states = [None] * state.n_map_states # type: ignore
state.cached_inputs = {
edge.key: state._result # type: ignore
for edge, state in upstream_states.items() if edge.key
}
raise ENDRUN(state)
# we can't map if there are no success states with iterables upstream
if upstream_states and not any([
edge.mapped and state.is_successful()
for edge, state in upstream_states.items()
]):
new_state = Failed(
"No upstream states can be mapped over.") # type: State
raise ENDRUN(new_state)
elif not all([
hasattr(state.result, "__getitem__")
for edge, state in upstream_states.items() if
state.is_successful() and not state.is_mapped() and edge.mapped
]):
new_state = Failed(
"At least one upstream state has an unmappable result.")
raise ENDRUN(new_state)
else:
# compute and set n_map_states
n_map_states = min(
[
len(s.result) for e, s in upstream_states.items()
if e.mapped and s.is_successful() and not s.is_mapped()
] + [
s.n_map_states # type: ignore
for e, s in upstream_states.items()
if e.mapped and s.is_mapped()
],
default=0,
)
new_state = Mapped("Ready to proceed with mapping.",
n_map_states=n_map_states)
raise ENDRUN(new_state)
def deploy_flows(self, flow_runs: list) -> None:
"""
Deploy flow runs on your local machine as Docker containers
Args:
- flow_runs (list): A list of GraphQLResult flow run objects
"""
for flow_run in flow_runs:
self.logger.info("Deploying flow run {}".format(
flow_run.id) # type: ignore
)
storage = StorageSchema().load(flow_run.flow.storage)
if not isinstance(StorageSchema().load(flow_run.flow.storage),
Docker):
msg = "Storage for flow run {} is not of type Docker.".format(
flow_run.id)
state_msg = "Agent {} failed to run flow: ".format(
self.name) + msg
self.client.set_flow_run_state(flow_run.id,
version=flow_run.version,
state=Failed(state_msg))
self.logger.error(msg)
continue
env_vars = self.populate_env_vars(flow_run=flow_run)
if not self.no_pull and storage.registry_url:
self.logger.info("Pulling image {}...".format(storage.name))
try:
pull_output = self.docker_client.pull(storage.name,
stream=True,
decode=True)
for line in pull_output:
self.logger.debug(line)
self.logger.info("Successfully pulled image {}...".format(
storage.name))
except docker.errors.APIError as exc:
msg = "Issue pulling image {}".format(storage.name)
state_msg = (
"Agent {} failed to pull image for flow: ".format(
self.name) + msg)
self.client.set_flow_run_state(flow_run.id,
version=flow_run.version,
state=Failed(msg))
self.logger.error(msg)
# Create a container
self.logger.debug("Creating Docker container {}".format(
storage.name))
container = self.docker_client.create_container(
storage.name,
command="prefect execute cloud-flow",
environment=env_vars)
# Start the container
self.logger.debug("Starting Docker container with ID {}".format(
container.get("Id")))
self.docker_client.start(container=container.get("Id"))
def inner(self: "Runner", state: State, *args: Any,
**kwargs: Any) -> State:
raise_end_run = False
raise_on_exception = prefect.context.get("raise_on_exception", False)
try:
new_state = method(self, state, *args, **kwargs)
except ENDRUN as exc:
raise_end_run = True
new_state = exc.state
# PrefectStateSignals are trapped and turned into States
except signals.PrefectStateSignal as exc:
self.logger.debug("{name} signal raised: {rep}".format(
name=type(exc).__name__, rep=repr(exc)))
if raise_on_exception:
raise exc
new_state = exc.state
except Exception as exc:
formatted = "Unexpected error: {}".format(repr(exc))
self.logger.exception(formatted)
if raise_on_exception:
raise exc
new_state = Failed(formatted, result=exc)
if new_state is not state:
new_state = self.handle_state_change(old_state=state,
new_state=new_state)
# if an ENDRUN was raised, reraise so it can be trapped
if raise_end_run:
raise ENDRUN(new_state)
return new_state
def load_results(
self, state: State,
upstream_states: Dict[Edge,
State]) -> Tuple[State, Dict[Edge, State]]:
"""
Given the task's current state and upstream states, populates all relevant result objects for this task run.
Args:
- state (State): the task's current state.
- upstream_states (Dict[Edge, State]): the upstream state_handlers
Returns:
- Tuple[State, dict]: a tuple of (state, upstream_states)
"""
upstream_results = {}
try:
for edge, upstream_state in upstream_states.items():
upstream_states[edge] = upstream_state.load_result(
edge.upstream_task.result or self.default_result)
if edge.key is not None:
upstream_results[edge.key] = (edge.upstream_task.result
or self.default_result)
state.load_cached_results(upstream_results)
return state, upstream_states
except Exception as exc:
new_state = Failed(
message=f"Failed to retrieve task results: {exc}", result=exc)
final_state = self.handle_state_change(old_state=state,
new_state=new_state)
raise ENDRUN(final_state)
class TestRunFlowStep:
def test_running_state_finishes(self):
flow = Flow(name="test", tasks=[Task()])
new_state = FlowRunner(flow=flow).get_flow_run_state(
state=Running(),
task_states={},
task_contexts={},
return_tasks=set(),
task_runner_state_handlers=[],
executor=LocalExecutor(),
)
assert new_state.is_successful()
@pytest.mark.parametrize(
"state", [Pending(), Retrying(), Finished(), Success(), Failed(), Skipped()]
)
def test_other_states_raise_endrun(self, state):
flow = Flow(name="test", tasks=[Task()])
with pytest.raises(ENDRUN):
FlowRunner(flow=flow).get_flow_run_state(
state=state,
task_states={},
task_contexts={},
return_tasks=set(),
task_runner_state_handlers=[],
executor=Executor(),
)
def test_determine_final_state_has_final_say(self):
class MyFlowRunner(FlowRunner):
def determine_final_state(self, *args, **kwargs):
return Failed("Very specific error message")
flow = Flow(name="test", tasks=[Task()])
new_state = MyFlowRunner(flow=flow).get_flow_run_state(
state=Running(),
task_states={},
task_contexts={},
return_tasks=set(),
task_runner_state_handlers=[],
executor=LocalExecutor(),
)
assert new_state.is_failed()
assert new_state.message == "Very specific error message"
def test_determine_final_state_preserves_running_states_when_tasks_still_running(
self,
):
task = Task()
flow = Flow(name="test", tasks=[task])
old_state = Running()
new_state = FlowRunner(flow=flow).get_flow_run_state(
state=old_state,
task_states={task: Retrying(start_time=pendulum.now("utc").add(days=1))},
task_contexts={},
return_tasks=set(),
task_runner_state_handlers=[],
executor=LocalExecutor(),
)
assert new_state is old_state
def test_simple_two_task_flow_with_final_task_set_to_fail(
monkeypatch, executor):
flow_run_id = str(uuid.uuid4())
task_run_id_1 = str(uuid.uuid4())
task_run_id_2 = str(uuid.uuid4())
with prefect.Flow(name="test") as flow:
t1 = prefect.Task()
t2 = prefect.Task()
t2.set_upstream(t1)
client = MockedCloudClient(
flow_runs=[FlowRun(id=flow_run_id)],
task_runs=[
TaskRun(id=task_run_id_1, task_id=t1.id, flow_run_id=flow_run_id),
TaskRun(id=task_run_id_2,
task_id=t2.id,
flow_run_id=flow_run_id,
state=Failed()),
],
monkeypatch=monkeypatch,
)
with prefect.context(flow_run_id=flow_run_id):
state = CloudFlowRunner(flow=flow).run(return_tasks=flow.tasks,
executor=executor)
assert state.is_failed()
assert client.flow_runs[flow_run_id].state.is_failed()
assert client.task_runs[task_run_id_1].state.is_successful()
assert client.task_runs[task_run_id_1].version == 2
assert client.task_runs[task_run_id_2].state.is_failed()
assert client.task_runs[task_run_id_2].version == 0
def test_jira_notifier_pulls_creds_from_secret(monkeypatch):
client = MagicMock()
jira = MagicMock(client=client)
monkeypatch.setattr("jira.JIRA", jira)
state = Failed(message="1", result=0)
with set_temporary_config({"cloud.use_local_secrets": True}):
with prefect.context(
secrets=dict(
JIRASECRETS={
"JIRAUSER": "******",
"JIRATOKEN": "",
"JIRASERVER": "https://foo/bar",
}
)
):
jira_notifier(
Task(),
"",
state,
options={"project": "TEST", "issuetype": {"name": "Task"}},
)
with pytest.raises(ValueError, match="JIRASECRETS"):
jira_notifier(
Task(),
"",
state,
options={"project": "TEST", "issuetype": {"name": "Task"}},
)
kwargs = jira.call_args[1]
assert kwargs == {
"basic_auth": ("Bob", ""),
"options": {"server": "https://foo/bar"},
}
def test_viz_reflects_mapping_if_flow_state_provided(self):
ipython = MagicMock(
get_ipython=lambda: MagicMock(config=dict(IPKernelApp=True)))
add = AddTask(name="a_nice_task")
list_task = Task(name="a_list_task")
map_state = Mapped(map_states=[Success(), Failed()])
with patch.dict("sys.modules", IPython=ipython):
with Flow(name="test") as f:
res = add.map(x=list_task, y=8)
graph = f.visualize(flow_state=Success(result={
res: map_state,
list_task: Success()
}))
# one colored node for each mapped result
assert 'label="a_nice_task <map>" color="#00800080"' in graph.source
assert 'label="a_nice_task <map>" color="#FF000080"' in graph.source
assert 'label=a_list_task color="#00800080"' in graph.source
assert 'label=8 color="#00000080"' in graph.source
# two edges for each input to add()
for var in ["x", "y"]:
for index in [0, 1]:
assert "{0} [label={1} style=dashed]".format(
index, var) in graph.source
def test_slack_notifier_returns_new_state_and_old_state_is_ignored(
monkeypatch):
ok = MagicMock(ok=True)
monkeypatch.setattr(prefect.utilities.notifications.requests, "post", ok)
new_state = Failed(message="1", result=0)
with set_temporary_config({"cloud.use_local_secrets": True}):
assert slack_notifier(Task(), "", new_state) is new_state
def test_viz_reflects_multiple_mapping_if_flow_state_provided(self):
ipython = MagicMock(
get_ipython=lambda: MagicMock(config=dict(IPKernelApp=True)))
add = AddTask(name="a_nice_task")
list_task = Task(name="a_list_task")
map_state1 = Mapped(map_states=[Success(), TriggerFailed()])
map_state2 = Mapped(map_states=[Success(), Failed()])
with patch.dict("sys.modules", IPython=ipython):
with Flow(name="test") as f:
first_res = add.map(x=list_task, y=8)
with pytest.warns(
UserWarning
): # making a copy of a task with dependencies
res = first_res.map(x=first_res, y=9)
graph = f.visualize(flow_state=Success(
result={
res: map_state1,
list_task: Success(),
first_res: map_state2,
}))
assert "{first} -> {second} [label=x style=dashed]".format(
first=str(id(first_res)) + "0", second=str(id(res)) + "0")
assert "{first} -> {second} [label=x style=dashed]".format(
first=str(id(first_res)) + "1", second=str(id(res)) + "1")
def test_slack_notifier_returns_new_state_and_old_state_is_ignored(monkeypatch):
ok = MagicMock(ok=True)
monkeypatch.setattr(requests, "post", ok)
new_state = Failed(message="1", result=0)
with set_temporary_config({"cloud.use_local_secrets": True}):
with prefect.context(secrets=dict(SLACK_WEBHOOK_URL="")):
assert slack_notifier(Task(), "", new_state) is new_state
class TestCheckScheduledStep:
@pytest.mark.parametrize(
"state", [Failed(), Pending(),
Running(), Success()])
def test_non_scheduled_states(self, state):
assert (FlowRunner(flow=Flow(
name="test")).check_flow_reached_start_time(state=state) is state)
def test_scheduled_states_without_start_time(self):
state = Scheduled(start_time=None)
assert (FlowRunner(flow=Flow(
name="test")).check_flow_reached_start_time(state=state) is state)
def test_scheduled_states_with_future_start_time(self):
state = Scheduled(start_time=pendulum.now("utc") +
datetime.timedelta(minutes=10))
with pytest.raises(ENDRUN) as exc:
FlowRunner(flow=Flow(name="test")).check_flow_reached_start_time(
state=state)
assert exc.value.state is state
def test_scheduled_states_with_past_start_time(self):
state = Scheduled(start_time=pendulum.now("utc") -
datetime.timedelta(minutes=1))
assert (FlowRunner(flow=Flow(
name="test")).check_flow_reached_start_time(state=state) is state)
async def reap_zombie_cancelling_flow_runs(
self, heartbeat_cutoff: datetime.datetime = None) -> int:
"""
Marks flow runs that are in a `Cancelling` state but fail to move to a
`Cancelled` state as `Failed`.
Returns:
- int: the number of flow runs that were handled
"""
zombies = 0
heartbeat_cutoff = heartbeat_cutoff or pendulum.now("utc").subtract(
minutes=10)
where_clause = await self.get_flow_runs_where_clause(
heartbeat_cutoff=heartbeat_cutoff)
flow_runs = await models.FlowRun.where(where_clause).get(
selection_set={"id", "tenant_id"},
limit=5000,
order_by={"updated": EnumValue("desc")},
)
if flow_runs:
self.logger.info(
f"Zombie killer found {len(flow_runs)} flow runs.")
# Set flow run states to failed
for fr in flow_runs:
try:
message = "No heartbeat detected from the flow run; marking the run as failed."
await prefect.api.states.set_flow_run_state(
flow_run_id=fr.id,
state=Failed(message=message),
)
# log the state change to the flow run
await prefect.api.logs.create_logs(
[
dict(
tenant_id=fr.tenant_id,
flow_run_id=fr.id,
name=f"{self.logger.name}.FlowRun",
message=message,
level="ERROR",
)
],
defer_db_write=False,
)
zombies += 1
except ValueError:
self.logger.error("Error updating flow run %s",
fr.id,
exc_info=True)
if zombies:
self.logger.info(f"Addressed {zombies} zombie flow runs.")
return zombies
def test_flow_run_respects_task_state_kwarg():
t, s = Task(), Task()
f = Flow(name="test", tasks=[t, s])
flow_state = f.run(task_states={t: Failed("unique.")})
assert flow_state.is_failed()
assert flow_state.result[t].is_failed()
assert flow_state.result[t].message == "unique."
assert flow_state.result[s].is_successful()
async def test_version_locking_disabled_if_version_locking_flag_not_set(
self, flow_id, flow_group_id, flow_run_id):
await models.FlowGroup.where(id=flow_group_id
).update(set={"settings": {}})
# pass weird version numbers to confirm version locking is disabled
await api.states.set_flow_run_state(flow_run_id=flow_run_id,
state=Failed(),
version=1000)
def test_slack_notifier_uses_proxies(monkeypatch):
post = MagicMock(ok=True)
monkeypatch.setattr(requests, "post", post)
state = Failed(message="1", result=0)
with set_temporary_config({"cloud.use_local_secrets": True}):
with prefect.context(secrets=dict(SLACK_WEBHOOK_URL="")):
slack_notifier(Task(),
"",
state,
proxies={"http": "some.proxy.I.P"})
assert post.call_args[1]["proxies"] == {"http": "some.proxy.I.P"}
async def test_set_task_run_state_with_version_succeeds_if_version_matches(
self, task_run_id):
result = await api.states.set_task_run_state(task_run_id=task_run_id,
state=Failed(),
version=0)
query = await models.TaskRun.where(id=task_run_id
).first({"version", "state"})
assert query.version == 1
assert query.state == "Failed"
async def test_set_flow_run_state_with_version_succeeds_if_version_matches(
self, flow_run_id):
result = await api.states.set_flow_run_state(flow_run_id=flow_run_id,
state=Failed(),
version=1)
query = await models.FlowRun.where(id=flow_run_id
).first({"version", "state"})
assert query.version == 2
assert query.state == "Failed"
def imb_handler(obj, old_state, new_state):
# Hamdle an empty dataframe to return a fail message.
# The result of a succesfull
if isinstance(new_state, Success) and new_state.result.empty:
return_state = Failed(
message=f"No tsx imbalance data: No trading or Data was not published to {new_state.cached_inputs['url']}",
result=new_state.result
)
else:
return_state = new_state
return return_state
def test_fail_flow_run(cloud_mocks):
_fail_flow_run(flow_run_id="flow-run-id", message="fail message")
cloud_mocks.Client().set_flow_run_state.assert_called_once_with(
flow_run_id="flow-run-id", state=Failed("fail message"))
cloud_mocks.Client().write_run_logs.assert_called_once_with([
dict(
flow_run_id="flow-run-id",
name="prefect.backend.execution",
message="fail message",
level="ERROR",
)
])
class TestCheckFlowPendingOrRunning:
@pytest.mark.parametrize("state", [Pending(), Running(), Retrying(), Scheduled()])
def test_pending_or_running_are_ok(self, state):
flow = Flow(name="test", tasks=[Task()])
new_state = FlowRunner(flow=flow).check_flow_is_pending_or_running(state=state)
assert new_state is state
@pytest.mark.parametrize("state", [Finished(), Success(), Failed(), Skipped()])
def test_not_pending_or_running_raise_endrun(self, state):
flow = Flow(name="test", tasks=[Task()])
with pytest.raises(ENDRUN):
FlowRunner(flow=flow).check_flow_is_pending_or_running(state=state)
async def test_set_task_run_state(self, task_run_id):
result = await api.states.set_task_run_state(task_run_id=task_run_id,
state=Failed())
assert result.task_run_id == task_run_id
query = await models.TaskRun.where(id=task_run_id).first(
{"version", "state", "serialized_state"})
assert query.version == 2
assert query.state == "Failed"
assert query.serialized_state["type"] == "Failed"
def test_state_type_methods_with_failed_state(self):
state = Failed(message="")
assert not state.is_cached()
assert not state.is_pending()
assert not state.is_retrying()
assert not state.is_running()
assert state.is_finished()
assert not state.is_skipped()
assert not state.is_scheduled()
assert not state.is_successful()
assert state.is_failed()
assert not state.is_mapped()
assert not state.is_meta_state()
def determine_final_state(
self,
state: State,
key_states: Set[State],
return_states: Dict[Task, State],
terminal_states: Set[State],
) -> State:
"""
Implements the logic for determining the final state of the flow run.
Args:
- state (State): the current state of the Flow
- key_states (Set[State]): the states which will determine the success / failure of
the flow run
- return_states (Dict[Task, State]): states to return as results
- terminal_states (Set[State]): the states of the terminal tasks for this flow
Returns:
- State: the final state of the flow run
"""
# check that the flow is finished
if not all(s.is_finished() for s in terminal_states):
self.logger.info(
"Flow run RUNNING: terminal tasks are incomplete.")
state.result = return_states
# check if any key task failed
elif any(s.is_failed() for s in key_states):
self.logger.info("Flow run FAILED: some reference tasks failed.")
state = Failed(message="Some reference tasks failed.",
result=return_states)
# check if all reference tasks succeeded
elif all(s.is_successful() for s in key_states):
self.logger.info("Flow run SUCCESS: all reference tasks succeeded")
state = Success(message="All reference tasks succeeded.",
result=return_states)
# check for any unanticipated state that is finished but neither success nor failed
else:
self.logger.info("Flow run SUCCESS: no reference tasks failed")
state = Success(message="No reference tasks failed.",
result=return_states)
if self.flow.terminal_state_handler:
new_state = self.flow.terminal_state_handler(
self.flow, state, key_states)
if new_state is not None:
return new_state
return state
def call_runner_target_handlers(self, old_state: State,
new_state: State) -> State:
"""
A special state handler that the TaskRunner uses to call its task's state handlers.
This method is called as part of the base Runner's `handle_state_change()` method.
Args:
- old_state (State): the old (previous) state
- new_state (State): the new (current) state
Returns:
- State: the new state
"""
raise_on_exception = prefect.context.get("raise_on_exception", False)
try:
new_state = super().call_runner_target_handlers(
old_state=old_state, new_state=new_state)
except Exception as exc:
msg = "Exception raised while calling state handlers: {}".format(
repr(exc))
self.logger.exception(msg)
if raise_on_exception:
raise exc
new_state = Failed(msg, result=exc)
task_run_id = prefect.context.get("task_run_id")
version = prefect.context.get("task_run_version")
try:
cloud_state = new_state
state = self.client.set_task_run_state(
task_run_id=task_run_id,
version=version,
state=cloud_state,
cache_for=self.task.cache_for,
)
except Exception as exc:
self.logger.exception(
"Failed to set task state with error: {}".format(repr(exc)))
raise ENDRUN(state=ClientFailed(state=new_state))
if state.is_queued():
state.state = old_state # type: ignore
raise ENDRUN(state=state)
if version is not None:
prefect.context.update(task_run_version=version +
1) # type: ignore
return new_state
def mark_failed(self, flow_run: GraphQLResult, exc: Exception) -> None:
"""
Mark a flow run as `Failed`
Args:
- flow_run (GraphQLResult): A GraphQLResult flow run object
- exc (Exception): An exception that was raised to use as the `Failed`
message
"""
self.client.set_flow_run_state(
flow_run_id=flow_run.id,
version=flow_run.version,
state=Failed(message=str(exc)),
)
self.logger.error("Error while deploying flow: {}".format(repr(exc)))
def test_mark_flow_as_failed(monkeypatch, cloud_api):
agent = Agent()
agent.client = MagicMock()
agent._mark_flow_as_failed(
flow_run=GraphQLResult({
"id": "id",
"serialized_state": Scheduled().serialize(),
"version": 1,
"task_runs": [],
}),
message="foo",
)
agent.client.set_flow_run_state.assert_called_with(
flow_run_id="id", version=1, state=Failed(message="foo"))
def test_does_not_write_checkpoint_file_to_disk_on_failure(self, tmp_path):
result_handler = PandasResultHandler(tmp_path / "dummy.csv",
"csv",
write_kwargs={"index": False})
task = Task(name="Task", result_handler=result_handler)
result = pd.DataFrame({"one": [1, 2, 3], "two": [4, 5, 6]})
task_runner = DSTaskRunner(task)
task_runner.upstream_states = {}
old_state = Running()
new_state = Failed(result=result)
dsh.checkpoint_handler(task_runner, old_state, new_state)
with pytest.raises(IOError):
pd.read_csv(tmp_path / "dummy.csv")
