def test_error_document():
ce = errors.ContainerError("code", "my message", errors.ContainerError.Kind.RECOVERABLE)
obj = errors.ErrorDocument(ce)
assert obj.error == ce
obj2 = errors.ErrorDocument.from_flyte_idl(obj.to_flyte_idl())
assert obj == obj2
assert obj2.error == ce
def test_container_error():
obj = errors.ContainerError("code", "my message", errors.ContainerError.Kind.RECOVERABLE)
assert obj.code == "code"
assert obj.message == "my message"
assert obj.kind == errors.ContainerError.Kind.RECOVERABLE
obj2 = errors.ContainerError.from_flyte_idl(obj.to_flyte_idl())
assert obj == obj2
assert obj2.code == "code"
assert obj2.message == "my message"
assert obj2.kind == errors.ContainerError.Kind.RECOVERABLE
def test_container_error():
obj = errors.ContainerError("code", "my message",
errors.ContainerError.Kind.RECOVERABLE,
execution.ExecutionError.ErrorKind.SYSTEM)
assert obj.code == "code"
assert obj.message == "my message"
assert obj.kind == errors.ContainerError.Kind.RECOVERABLE
assert obj.origin == 2
obj2 = errors.ContainerError.from_flyte_idl(obj.to_flyte_idl())
assert obj == obj2
assert obj2.code == "code"
assert obj2.message == "my message"
assert obj2.kind == errors.ContainerError.Kind.RECOVERABLE
assert obj2.origin == 2
def execute(self, inputs, context=None):
"""
Just execute the task and write the outputs to where they belong
:param flytekit.models.literals.LiteralMap inputs:
:param dict[Text, Text] context:
:rtype: dict[Text, flytekit.models.common.FlyteIdlEntity]
"""
with _common_utils.AutoDeletingTempDir("engine_dir") as temp_dir:
with _common_utils.AutoDeletingTempDir("task_dir") as task_dir:
with _data_proxy.LocalWorkingDirectoryContext(task_dir):
with _data_proxy.RemoteDataContext():
output_file_dict = dict()
# This sets the logging level for user code and is the only place an sdk setting gets
# used at runtime. Optionally, Propeller can set an internal config setting which
# takes precedence.
log_level = _internal_config.LOGGING_LEVEL.get(
) or _sdk_config.LOGGING_LEVEL.get()
_logging.getLogger().setLevel(log_level)
try:
output_file_dict = self.sdk_task.execute(
_common_engine.EngineContext(
execution_id=_identifier.
WorkflowExecutionIdentifier(
project=_internal_config.
EXECUTION_PROJECT.get(),
domain=_internal_config.
EXECUTION_DOMAIN.get(),
name=_internal_config.EXECUTION_NAME.
get()),
execution_date=_datetime.utcnow(),
stats=_get_stats(
# Stats metric path will be:
# registration_project.registration_domain.app.module.task_name.user_stats
# and it will be tagged with execution-level values for project/domain/wf/lp
"{}.{}.{}.user_stats".format(
_internal_config.TASK_PROJECT.get(
)
or _internal_config.PROJECT.get(),
_internal_config.TASK_DOMAIN.get()
or _internal_config.DOMAIN.get(),
_internal_config.TASK_NAME.get()
or _internal_config.NAME.get()),
tags={
'exec_project':
_internal_config.EXECUTION_PROJECT.
get(),
'exec_domain':
_internal_config.EXECUTION_DOMAIN.
get(),
'exec_workflow':
_internal_config.
EXECUTION_WORKFLOW.get(),
'exec_launchplan':
_internal_config.
EXECUTION_LAUNCHPLAN.get(),
'api_version':
_api_version
}),
logging=_logging,
tmp_dir=task_dir),
inputs)
except _exception_scopes.FlyteScopedException as e:
_logging.error(
"!!! Begin Error Captured by Flyte !!!")
output_file_dict[
_constants.
ERROR_FILE_NAME] = _error_models.ErrorDocument(
_error_models.ContainerError(
e.error_code, e.verbose_message,
e.kind))
_logging.error(e.verbose_message)
_logging.error(
"!!! End Error Captured by Flyte !!!")
except Exception:
_logging.error(
"!!! Begin Unknown System Error Captured by Flyte !!!"
)
exc_str = _traceback.format_exc()
output_file_dict[
_constants.
ERROR_FILE_NAME] = _error_models.ErrorDocument(
_error_models.ContainerError(
"SYSTEM:Unknown", exc_str,
_error_models.ContainerError.Kind.
RECOVERABLE))
_logging.error(exc_str)
_logging.error(
"!!! End Error Captured by Flyte !!!")
finally:
for k, v in _six.iteritems(output_file_dict):
_common_utils.write_proto_to_file(
v.to_flyte_idl(),
_os.path.join(temp_dir.name, k))
_data_proxy.Data.put_data(temp_dir.name,
context['output_prefix'],
is_multipart=True)
def _dispatch_execute(ctx: FlyteContext, task_def: PythonTask, inputs_path: str, output_prefix: str):
"""
Dispatches execute to PythonTask
Step1: Download inputs and load into a literal map
Step2: Invoke task - dispatch_execute
Step3:
a: [Optional] Record outputs to output_prefix
b: OR if IgnoreOutputs is raised, then ignore uploading outputs
c: OR if an unhandled exception is retrieved - record it as an errors.pb
"""
output_file_dict = {}
try:
# Step1
local_inputs_file = _os.path.join(ctx.execution_state.working_dir, "inputs.pb")
ctx.file_access.get_data(inputs_path, local_inputs_file)
input_proto = _utils.load_proto_from_file(_literals_pb2.LiteralMap, local_inputs_file)
idl_input_literals = _literal_models.LiteralMap.from_flyte_idl(input_proto)
# Step2
outputs = task_def.dispatch_execute(ctx, idl_input_literals)
# Step3a
if isinstance(outputs, VoidPromise):
_logging.getLogger().warning("Task produces no outputs")
output_file_dict = {_constants.OUTPUT_FILE_NAME: _literal_models.LiteralMap(literals={})}
elif isinstance(outputs, _literal_models.LiteralMap):
output_file_dict = {_constants.OUTPUT_FILE_NAME: outputs}
elif isinstance(outputs, _dynamic_job.DynamicJobSpec):
output_file_dict = {_constants.FUTURES_FILE_NAME: outputs}
else:
_logging.getLogger().error(f"SystemError: received unknown outputs from task {outputs}")
output_file_dict[_constants.ERROR_FILE_NAME] = _error_models.ErrorDocument(
_error_models.ContainerError(
"UNKNOWN_OUTPUT",
f"Type of output received not handled {type(outputs)} outputs: {outputs}",
_error_models.ContainerError.Kind.RECOVERABLE,
)
)
except _scoped_exceptions.FlyteScopedException as e:
_logging.error("!! Begin Error Captured by Flyte !!")
output_file_dict[_constants.ERROR_FILE_NAME] = _error_models.ErrorDocument(
_error_models.ContainerError(e.error_code, e.verbose_message, e.kind)
)
_logging.error(e.verbose_message)
_logging.error("!! End Error Captured by Flyte !!")
except Exception as e:
if isinstance(e, IgnoreOutputs):
# Step 3b
_logging.warning(f"IgnoreOutputs received! Outputs.pb will not be uploaded. reason {e}")
return
# Step 3c
_logging.error(f"Exception when executing task {task_def.name}, reason {str(e)}")
_logging.error("!! Begin Unknown System Error Captured by Flyte !!")
exc_str = _traceback.format_exc()
output_file_dict[_constants.ERROR_FILE_NAME] = _error_models.ErrorDocument(
_error_models.ContainerError("SYSTEM:Unknown", exc_str, _error_models.ContainerError.Kind.RECOVERABLE,)
)
_logging.error(exc_str)
_logging.error("!! End Error Captured by Flyte !!")
for k, v in output_file_dict.items():
_common_utils.write_proto_to_file(v.to_flyte_idl(), _os.path.join(ctx.execution_state.engine_dir, k))
ctx.file_access.upload_directory(ctx.execution_state.engine_dir, output_prefix)
_logging.info(f"Engine folder written successfully to the output prefix {output_prefix}")
def _dispatch_execute(
ctx: FlyteContext,
task_def: PythonTask,
inputs_path: str,
output_prefix: str,
):
"""
Dispatches execute to PythonTask
Step1: Download inputs and load into a literal map
Step2: Invoke task - dispatch_execute
Step3:
a: [Optional] Record outputs to output_prefix
b: OR if IgnoreOutputs is raised, then ignore uploading outputs
c: OR if an unhandled exception is retrieved - record it as an errors.pb
"""
output_file_dict = {}
logger.debug(f"Starting _dispatch_execute for {task_def.name}")
try:
# Step1
local_inputs_file = os.path.join(ctx.execution_state.working_dir, "inputs.pb")
ctx.file_access.get_data(inputs_path, local_inputs_file)
input_proto = utils.load_proto_from_file(_literals_pb2.LiteralMap, local_inputs_file)
idl_input_literals = _literal_models.LiteralMap.from_flyte_idl(input_proto)
# Step2
# Decorate the dispatch execute function before calling it, this wraps all exceptions into one
# of the FlyteScopedExceptions
outputs = _scoped_exceptions.system_entry_point(task_def.dispatch_execute)(ctx, idl_input_literals)
# Step3a
if isinstance(outputs, VoidPromise):
logger.warning("Task produces no outputs")
output_file_dict = {_constants.OUTPUT_FILE_NAME: _literal_models.LiteralMap(literals={})}
elif isinstance(outputs, _literal_models.LiteralMap):
output_file_dict = {_constants.OUTPUT_FILE_NAME: outputs}
elif isinstance(outputs, _dynamic_job.DynamicJobSpec):
output_file_dict = {_constants.FUTURES_FILE_NAME: outputs}
else:
logger.error(f"SystemError: received unknown outputs from task {outputs}")
output_file_dict[_constants.ERROR_FILE_NAME] = _error_models.ErrorDocument(
_error_models.ContainerError(
"UNKNOWN_OUTPUT",
f"Type of output received not handled {type(outputs)} outputs: {outputs}",
_error_models.ContainerError.Kind.RECOVERABLE,
_execution_models.ExecutionError.ErrorKind.SYSTEM,
)
)
# Handle user-scoped errors
except _scoped_exceptions.FlyteScopedUserException as e:
if isinstance(e.value, IgnoreOutputs):
logger.warning(f"User-scoped IgnoreOutputs received! Outputs.pb will not be uploaded. reason {e}!!")
return
output_file_dict[_constants.ERROR_FILE_NAME] = _error_models.ErrorDocument(
_error_models.ContainerError(
e.error_code, e.verbose_message, e.kind, _execution_models.ExecutionError.ErrorKind.USER
)
)
logger.error("!! Begin User Error Captured by Flyte !!")
logger.error(e.verbose_message)
logger.error("!! End Error Captured by Flyte !!")
# Handle system-scoped errors
except _scoped_exceptions.FlyteScopedSystemException as e:
if isinstance(e.value, IgnoreOutputs):
logger.warning(f"System-scoped IgnoreOutputs received! Outputs.pb will not be uploaded. reason {e}!!")
return
output_file_dict[_constants.ERROR_FILE_NAME] = _error_models.ErrorDocument(
_error_models.ContainerError(
e.error_code, e.verbose_message, e.kind, _execution_models.ExecutionError.ErrorKind.SYSTEM
)
)
logger.error("!! Begin System Error Captured by Flyte !!")
logger.error(e.verbose_message)
logger.error("!! End Error Captured by Flyte !!")
# Interpret all other exceptions (some of which may be caused by the code in the try block outside of
# dispatch_execute) as recoverable system exceptions.
except Exception as e:
# Step 3c
exc_str = _traceback.format_exc()
output_file_dict[_constants.ERROR_FILE_NAME] = _error_models.ErrorDocument(
_error_models.ContainerError(
"SYSTEM:Unknown",
exc_str,
_error_models.ContainerError.Kind.RECOVERABLE,
_execution_models.ExecutionError.ErrorKind.SYSTEM,
)
)
logger.error(f"Exception when executing task {task_def.name or task_def.id.name}, reason {str(e)}")
logger.error("!! Begin Unknown System Error Captured by Flyte !!")
logger.error(exc_str)
logger.error("!! End Error Captured by Flyte !!")
for k, v in output_file_dict.items():
utils.write_proto_to_file(v.to_flyte_idl(), os.path.join(ctx.execution_state.engine_dir, k))
ctx.file_access.put_data(ctx.execution_state.engine_dir, output_prefix, is_multipart=True)
logger.info(f"Engine folder written successfully to the output prefix {output_prefix}")
logger.debug("Finished _dispatch_execute")
