def check_task_is_cached(self, state: State,
inputs: Dict[str, Result]) -> State:
"""
Checks if task is cached and whether the cache is still valid.
Args:
- state (State): the current state of this task
- inputs (Dict[str, Result]): a dictionary of inputs whose keys correspond
to the task's `run()` arguments.
Returns:
- State: the state of the task after running the check
Raises:
- ENDRUN: if the task is not ready to run
"""
if state.is_cached():
assert isinstance(state, Cached) # mypy assert
if self.task.cache_validator(state, inputs,
prefect.context.get("parameters")):
state._result = state._result.to_result()
return state
else:
self.logger.warning(
"Task '{name}': can't use cache because it "
"is now invalid".format(name=prefect.context.get(
"task_full_name", self.task.name)))
return Pending("Cache was invalid; ready to run.")
return state
def prepare_state_for_cloud(state: State) -> State:
"""
Prepares a Prefect State for being sent to Cloud; this ensures that any data attributes
are properly handled prior to being shipped off to a database.
Args:
- state (State): the Prefect State to prepare
Returns:
- State: a sanitized copy of the original state
"""
if state.is_cached():
state._result.store_safe_value()
if (isinstance(state, Failed) and state.cached_inputs is not None
and all(r.result_handler is not None
for k, r in state.cached_inputs.items())): # type: ignore
for res in state.cached_inputs.values():
res.store_safe_value()
elif (hasattr(state, "cached_inputs")
and state.cached_inputs is not None # type: ignore
and not state.is_failed()):
for res in state.cached_inputs.values(): # type: ignore
res.store_safe_value()
return state
def check_task_is_ready(self, state: State) -> State:
"""
Checks to make sure the task is ready to run (Pending or Mapped).
Args:
- state (State): the current state of this task
Returns:
- State: the state of the task after running the check
Raises:
- ENDRUN: if the task is not ready to run
"""
# the task is ready
if state.is_pending():
return state
# the task is mapped, in which case we still proceed so that the children tasks
# are generated (note that if the children tasks)
elif state.is_mapped():
self.logger.debug(
"Task '{name}': task is mapped, but run will proceed so children are generated.".format(
name=prefect.context.get("task_full_name", self.task.name)
)
)
return state
# this task is already running
elif state.is_running():
self.logger.debug(
"Task '{name}': task is already running.".format(
name=prefect.context.get("task_full_name", self.task.name)
)
)
raise ENDRUN(state)
elif state.is_cached():
return state
# this task is already finished
elif state.is_finished():
self.logger.debug(
"Task '{name}': task is already finished.".format(
name=prefect.context.get("task_full_name", self.task.name)
)
)
raise ENDRUN(state)
# this task is not pending
else:
self.logger.debug(
"Task '{name}' is not ready to run or state was unrecognized ({state}).".format(
name=prefect.context.get("task_full_name", self.task.name),
state=state,
)
)
raise ENDRUN(state)
def check_task_is_cached(self, state: State, inputs: Dict[str, Result]) -> State:
"""
Checks if task is cached and whether the cache is still valid.
Args:
- state (State): the current state of this task
- inputs (Dict[str, Result]): a dictionary of inputs whose keys correspond
to the task's `run()` arguments.
Returns:
- State: the state of the task after running the check
Raises:
- ENDRUN: if the task is not ready to run
"""
if state.is_cached():
assert isinstance(state, Cached) # mypy assert
sanitized_inputs = {key: res.value for key, res in inputs.items()}
if self.task.cache_validator(
state, sanitized_inputs, prefect.context.get("parameters")
):
state._result = state._result.to_result(self.task.result_handler)
return state
else:
state = Pending("Cache was invalid; ready to run.")
if self.task.cache_for is not None:
candidate_states = prefect.context.caches.get(
self.task.cache_key or self.task.name, []
)
sanitized_inputs = {key: res.value for key, res in inputs.items()}
for candidate in candidate_states:
if self.task.cache_validator(
candidate, sanitized_inputs, prefect.context.get("parameters")
):
candidate._result = candidate._result.to_result(
self.task.result_handler
)
return candidate
if self.task.cache_for is not None:
self.logger.warning(
"Task '{name}': can't use cache because it "
"is now invalid".format(
name=prefect.context.get("task_full_name", self.task.name)
)
)
return state or Pending("Cache was invalid; ready to run.")
def prepare_state_for_cloud(state: State) -> State:
"""
Prepares a Prefect State for being sent to Cloud; this ensures that any data attributes
are properly handled prior to being shipped off to a database.
Args:
- state (State): the Prefect State to prepare
Returns:
- State: a sanitized copy of the original state
"""
if state.is_cached():
state._result.store_safe_value()
if state.cached_inputs:
for res in state.cached_inputs.values():
if res.result_handler:
res.store_safe_value()
return state
def check_task_is_cached(self, state: State,
inputs: Dict[str, Result]) -> State:
"""
Checks if task is cached in the DB and whether any of the caches are still valid.
Args:
- state (State): the current state of this task
- inputs (Dict[str, Result]): a dictionary of inputs whose keys correspond
to the task's `run()` arguments.
Returns:
- State: the state of the task after running the check
Raises:
- ENDRUN: if the task is not ready to run
"""
if state.is_cached() is True:
assert isinstance(state, Cached) # mypy assert
sanitized_inputs = {key: res.value for key, res in inputs.items()}
if self.task.cache_validator(state, sanitized_inputs,
prefect.context.get("parameters")):
state = state.load_result(self.result)
return state
if self.task.cache_for is not None:
oldest_valid_cache = datetime.datetime.utcnow(
) - self.task.cache_for
cached_states = self.client.get_latest_cached_states(
task_id=prefect.context.get("task_id", ""),
cache_key=self.task.cache_key,
created_after=oldest_valid_cache,
)
if cached_states:
self.logger.debug(
"Task '{name}': {num} candidate cached states were found".
format(
name=prefect.context.get("task_full_name",
self.task.name),
num=len(cached_states),
))
for candidate_state in cached_states:
assert isinstance(candidate_state, Cached) # mypy assert
candidate_state.load_cached_results(inputs)
sanitized_inputs = {
key: res.value
for key, res in inputs.items()
}
if self.task.cache_validator(
candidate_state,
sanitized_inputs,
prefect.context.get("parameters"),
):
return candidate_state.load_result(self.result)
self.logger.debug(
"Task '{name}': can't use cache because no candidate Cached states "
"were valid".format(name=prefect.context.get(
"task_full_name", self.task.name)))
else:
self.logger.debug(
"Task '{name}': can't use cache because no Cached states were found"
.format(name=prefect.context.get("task_full_name",
self.task.name)))
return state
