def should_succeed(_):
# This is to have at least one other step that retried to properly test
# the step key filter on `get_run_step_stats`
if _count["total"] < 2:
_count["total"] += 1
raise RetryRequested(max_retries=3)
yield Output("yay")
def solid_execution_error_boundary(error_cls, msg_fn, step_context, **kwargs):
"""
A specialization of user_code_error_boundary for the steps involved in executing a solid.
This variant supports the control flow exceptions RetryRequested and Failure as well
as respecting the RetryPolicy if present.
"""
from dagster.core.execution.context.system import StepExecutionContext
check.callable_param(msg_fn, "msg_fn")
check.subclass_param(error_cls, "error_cls", DagsterUserCodeExecutionError)
check.inst_param(step_context, "step_context", StepExecutionContext)
with raise_execution_interrupts():
try:
yield
except (RetryRequested, Failure) as cf:
# A control flow exception has occurred and should be propagated
raise cf
except DagsterError as de:
# The system has thrown an error that is part of the user-framework contract
raise de
except Exception as e: # pylint: disable=W0703
# An exception has been thrown by user code and computation should cease
# with the error reported further up the stack
policy = step_context.solid_retry_policy
if policy:
# could check exc against a whitelist of exceptions
raise RetryRequested(
max_retries=policy.max_retries,
# could support an enum of "delay curves" which use delay and
# step_context.previous_attempt_count to calculate wait time
seconds_to_wait=policy.delay,
) from e
raise error_cls(
msg_fn(),
user_exception=e,
original_exc_info=sys.exc_info(),
**kwargs,
) from e
def should_retry(_):
raise RetryRequested(max_retries=3)
def should_succeed_after_retries(_):
if _count["total"] < 2:
_count["total"] += 1
raise RetryRequested(max_retries=3)
return "foo"
def should_retry(context, _input):
raise RetryRequested(max_retries=3)
def should_retry(_, _resource_config):
raise RetryRequested(max_retries=3)
def should_succeed(_, _resource_config):
if _count["total"] < 2:
_count["total"] += 1
raise RetryRequested(max_retries=3)
return "foo"
def solid_execution_error_boundary(error_cls, msg_fn, step_context, **kwargs):
"""
A specialization of user_code_error_boundary for the steps involved in executing a solid.
This variant supports the control flow exceptions RetryRequested and Failure as well
as respecting the RetryPolicy if present.
"""
from dagster.core.execution.context.system import StepExecutionContext
check.callable_param(msg_fn, "msg_fn")
check.class_param(error_cls,
"error_cls",
superclass=DagsterUserCodeExecutionError)
check.inst_param(step_context, "step_context", StepExecutionContext)
with raise_execution_interrupts():
step_context.log.begin_python_log_capture()
retry_policy = step_context.solid_retry_policy
try:
yield
except DagsterError as de:
# The system has thrown an error that is part of the user-framework contract
raise de
except Exception as e: # pylint: disable=W0703
# An exception has been thrown by user code and computation should cease
# with the error reported further up the stack
# Directly thrown RetryRequested escalate before evaluating the retry policy.
if isinstance(e, RetryRequested):
raise e
if retry_policy:
raise RetryRequested(
max_retries=retry_policy.max_retries,
seconds_to_wait=retry_policy.calculate_delay(
step_context.previous_attempt_count + 1),
) from e
# Failure exceptions get re-throw without wrapping
if isinstance(e, Failure):
raise e
# Otherwise wrap the user exception with context
raise error_cls(
msg_fn(),
user_exception=e,
original_exc_info=sys.exc_info(),
**kwargs,
) from e
except (DagsterExecutionInterruptedError, KeyboardInterrupt) as ie:
# respect retry policy when interrupts occur
if retry_policy:
raise RetryRequested(
max_retries=retry_policy.max_retries,
seconds_to_wait=retry_policy.calculate_delay(
step_context.previous_attempt_count + 1),
) from ie
else:
raise ie
finally:
step_context.log.end_python_log_capture()