def test_error_document(): ce = errors.ContainerError("code", "my message", errors.ContainerError.Kind.RECOVERABLE) obj = errors.ErrorDocument(ce) assert obj.error == ce obj2 = errors.ErrorDocument.from_flyte_idl(obj.to_flyte_idl()) assert obj == obj2 assert obj2.error == ce
def test_container_error(): obj = errors.ContainerError("code", "my message", errors.ContainerError.Kind.RECOVERABLE) assert obj.code == "code" assert obj.message == "my message" assert obj.kind == errors.ContainerError.Kind.RECOVERABLE obj2 = errors.ContainerError.from_flyte_idl(obj.to_flyte_idl()) assert obj == obj2 assert obj2.code == "code" assert obj2.message == "my message" assert obj2.kind == errors.ContainerError.Kind.RECOVERABLE
def test_container_error(): obj = errors.ContainerError("code", "my message", errors.ContainerError.Kind.RECOVERABLE, execution.ExecutionError.ErrorKind.SYSTEM) assert obj.code == "code" assert obj.message == "my message" assert obj.kind == errors.ContainerError.Kind.RECOVERABLE assert obj.origin == 2 obj2 = errors.ContainerError.from_flyte_idl(obj.to_flyte_idl()) assert obj == obj2 assert obj2.code == "code" assert obj2.message == "my message" assert obj2.kind == errors.ContainerError.Kind.RECOVERABLE assert obj2.origin == 2
def execute(self, inputs, context=None): """ Just execute the task and write the outputs to where they belong :param flytekit.models.literals.LiteralMap inputs: :param dict[Text, Text] context: :rtype: dict[Text, flytekit.models.common.FlyteIdlEntity] """ with _common_utils.AutoDeletingTempDir("engine_dir") as temp_dir: with _common_utils.AutoDeletingTempDir("task_dir") as task_dir: with _data_proxy.LocalWorkingDirectoryContext(task_dir): with _data_proxy.RemoteDataContext(): output_file_dict = dict() # This sets the logging level for user code and is the only place an sdk setting gets # used at runtime. Optionally, Propeller can set an internal config setting which # takes precedence. log_level = _internal_config.LOGGING_LEVEL.get( ) or _sdk_config.LOGGING_LEVEL.get() _logging.getLogger().setLevel(log_level) try: output_file_dict = self.sdk_task.execute( _common_engine.EngineContext( execution_id=_identifier. WorkflowExecutionIdentifier( project=_internal_config. EXECUTION_PROJECT.get(), domain=_internal_config. EXECUTION_DOMAIN.get(), name=_internal_config.EXECUTION_NAME. get()), execution_date=_datetime.utcnow(), stats=_get_stats( # Stats metric path will be: # registration_project.registration_domain.app.module.task_name.user_stats # and it will be tagged with execution-level values for project/domain/wf/lp "{}.{}.{}.user_stats".format( _internal_config.TASK_PROJECT.get( ) or _internal_config.PROJECT.get(), _internal_config.TASK_DOMAIN.get() or _internal_config.DOMAIN.get(), _internal_config.TASK_NAME.get() or _internal_config.NAME.get()), tags={ 'exec_project': _internal_config.EXECUTION_PROJECT. get(), 'exec_domain': _internal_config.EXECUTION_DOMAIN. get(), 'exec_workflow': _internal_config. EXECUTION_WORKFLOW.get(), 'exec_launchplan': _internal_config. EXECUTION_LAUNCHPLAN.get(), 'api_version': _api_version }), logging=_logging, tmp_dir=task_dir), inputs) except _exception_scopes.FlyteScopedException as e: _logging.error( "!!! Begin Error Captured by Flyte !!!") output_file_dict[ _constants. ERROR_FILE_NAME] = _error_models.ErrorDocument( _error_models.ContainerError( e.error_code, e.verbose_message, e.kind)) _logging.error(e.verbose_message) _logging.error( "!!! End Error Captured by Flyte !!!") except Exception: _logging.error( "!!! Begin Unknown System Error Captured by Flyte !!!" ) exc_str = _traceback.format_exc() output_file_dict[ _constants. ERROR_FILE_NAME] = _error_models.ErrorDocument( _error_models.ContainerError( "SYSTEM:Unknown", exc_str, _error_models.ContainerError.Kind. RECOVERABLE)) _logging.error(exc_str) _logging.error( "!!! End Error Captured by Flyte !!!") finally: for k, v in _six.iteritems(output_file_dict): _common_utils.write_proto_to_file( v.to_flyte_idl(), _os.path.join(temp_dir.name, k)) _data_proxy.Data.put_data(temp_dir.name, context['output_prefix'], is_multipart=True)
def _dispatch_execute(ctx: FlyteContext, task_def: PythonTask, inputs_path: str, output_prefix: str): """ Dispatches execute to PythonTask Step1: Download inputs and load into a literal map Step2: Invoke task - dispatch_execute Step3: a: [Optional] Record outputs to output_prefix b: OR if IgnoreOutputs is raised, then ignore uploading outputs c: OR if an unhandled exception is retrieved - record it as an errors.pb """ output_file_dict = {} try: # Step1 local_inputs_file = _os.path.join(ctx.execution_state.working_dir, "inputs.pb") ctx.file_access.get_data(inputs_path, local_inputs_file) input_proto = _utils.load_proto_from_file(_literals_pb2.LiteralMap, local_inputs_file) idl_input_literals = _literal_models.LiteralMap.from_flyte_idl(input_proto) # Step2 outputs = task_def.dispatch_execute(ctx, idl_input_literals) # Step3a if isinstance(outputs, VoidPromise): _logging.getLogger().warning("Task produces no outputs") output_file_dict = {_constants.OUTPUT_FILE_NAME: _literal_models.LiteralMap(literals={})} elif isinstance(outputs, _literal_models.LiteralMap): output_file_dict = {_constants.OUTPUT_FILE_NAME: outputs} elif isinstance(outputs, _dynamic_job.DynamicJobSpec): output_file_dict = {_constants.FUTURES_FILE_NAME: outputs} else: _logging.getLogger().error(f"SystemError: received unknown outputs from task {outputs}") output_file_dict[_constants.ERROR_FILE_NAME] = _error_models.ErrorDocument( _error_models.ContainerError( "UNKNOWN_OUTPUT", f"Type of output received not handled {type(outputs)} outputs: {outputs}", _error_models.ContainerError.Kind.RECOVERABLE, ) ) except _scoped_exceptions.FlyteScopedException as e: _logging.error("!! Begin Error Captured by Flyte !!") output_file_dict[_constants.ERROR_FILE_NAME] = _error_models.ErrorDocument( _error_models.ContainerError(e.error_code, e.verbose_message, e.kind) ) _logging.error(e.verbose_message) _logging.error("!! End Error Captured by Flyte !!") except Exception as e: if isinstance(e, IgnoreOutputs): # Step 3b _logging.warning(f"IgnoreOutputs received! Outputs.pb will not be uploaded. reason {e}") return # Step 3c _logging.error(f"Exception when executing task {task_def.name}, reason {str(e)}") _logging.error("!! Begin Unknown System Error Captured by Flyte !!") exc_str = _traceback.format_exc() output_file_dict[_constants.ERROR_FILE_NAME] = _error_models.ErrorDocument( _error_models.ContainerError("SYSTEM:Unknown", exc_str, _error_models.ContainerError.Kind.RECOVERABLE,) ) _logging.error(exc_str) _logging.error("!! End Error Captured by Flyte !!") for k, v in output_file_dict.items(): _common_utils.write_proto_to_file(v.to_flyte_idl(), _os.path.join(ctx.execution_state.engine_dir, k)) ctx.file_access.upload_directory(ctx.execution_state.engine_dir, output_prefix) _logging.info(f"Engine folder written successfully to the output prefix {output_prefix}")
def _dispatch_execute( ctx: FlyteContext, task_def: PythonTask, inputs_path: str, output_prefix: str, ): """ Dispatches execute to PythonTask Step1: Download inputs and load into a literal map Step2: Invoke task - dispatch_execute Step3: a: [Optional] Record outputs to output_prefix b: OR if IgnoreOutputs is raised, then ignore uploading outputs c: OR if an unhandled exception is retrieved - record it as an errors.pb """ output_file_dict = {} logger.debug(f"Starting _dispatch_execute for {task_def.name}") try: # Step1 local_inputs_file = os.path.join(ctx.execution_state.working_dir, "inputs.pb") ctx.file_access.get_data(inputs_path, local_inputs_file) input_proto = utils.load_proto_from_file(_literals_pb2.LiteralMap, local_inputs_file) idl_input_literals = _literal_models.LiteralMap.from_flyte_idl(input_proto) # Step2 # Decorate the dispatch execute function before calling it, this wraps all exceptions into one # of the FlyteScopedExceptions outputs = _scoped_exceptions.system_entry_point(task_def.dispatch_execute)(ctx, idl_input_literals) # Step3a if isinstance(outputs, VoidPromise): logger.warning("Task produces no outputs") output_file_dict = {_constants.OUTPUT_FILE_NAME: _literal_models.LiteralMap(literals={})} elif isinstance(outputs, _literal_models.LiteralMap): output_file_dict = {_constants.OUTPUT_FILE_NAME: outputs} elif isinstance(outputs, _dynamic_job.DynamicJobSpec): output_file_dict = {_constants.FUTURES_FILE_NAME: outputs} else: logger.error(f"SystemError: received unknown outputs from task {outputs}") output_file_dict[_constants.ERROR_FILE_NAME] = _error_models.ErrorDocument( _error_models.ContainerError( "UNKNOWN_OUTPUT", f"Type of output received not handled {type(outputs)} outputs: {outputs}", _error_models.ContainerError.Kind.RECOVERABLE, _execution_models.ExecutionError.ErrorKind.SYSTEM, ) ) # Handle user-scoped errors except _scoped_exceptions.FlyteScopedUserException as e: if isinstance(e.value, IgnoreOutputs): logger.warning(f"User-scoped IgnoreOutputs received! Outputs.pb will not be uploaded. reason {e}!!") return output_file_dict[_constants.ERROR_FILE_NAME] = _error_models.ErrorDocument( _error_models.ContainerError( e.error_code, e.verbose_message, e.kind, _execution_models.ExecutionError.ErrorKind.USER ) ) logger.error("!! Begin User Error Captured by Flyte !!") logger.error(e.verbose_message) logger.error("!! End Error Captured by Flyte !!") # Handle system-scoped errors except _scoped_exceptions.FlyteScopedSystemException as e: if isinstance(e.value, IgnoreOutputs): logger.warning(f"System-scoped IgnoreOutputs received! Outputs.pb will not be uploaded. reason {e}!!") return output_file_dict[_constants.ERROR_FILE_NAME] = _error_models.ErrorDocument( _error_models.ContainerError( e.error_code, e.verbose_message, e.kind, _execution_models.ExecutionError.ErrorKind.SYSTEM ) ) logger.error("!! Begin System Error Captured by Flyte !!") logger.error(e.verbose_message) logger.error("!! End Error Captured by Flyte !!") # Interpret all other exceptions (some of which may be caused by the code in the try block outside of # dispatch_execute) as recoverable system exceptions. except Exception as e: # Step 3c exc_str = _traceback.format_exc() output_file_dict[_constants.ERROR_FILE_NAME] = _error_models.ErrorDocument( _error_models.ContainerError( "SYSTEM:Unknown", exc_str, _error_models.ContainerError.Kind.RECOVERABLE, _execution_models.ExecutionError.ErrorKind.SYSTEM, ) ) logger.error(f"Exception when executing task {task_def.name or task_def.id.name}, reason {str(e)}") logger.error("!! Begin Unknown System Error Captured by Flyte !!") logger.error(exc_str) logger.error("!! End Error Captured by Flyte !!") for k, v in output_file_dict.items(): utils.write_proto_to_file(v.to_flyte_idl(), os.path.join(ctx.execution_state.engine_dir, k)) ctx.file_access.put_data(ctx.execution_state.engine_dir, output_prefix, is_multipart=True) logger.info(f"Engine folder written successfully to the output prefix {output_prefix}") logger.debug("Finished _dispatch_execute")