def inputs(self) -> Dict[str, Any]: """ Returns the inputs to the execution in the standard python format as dictated by the type engine. """ if self._inputs is None: client = _flyte_engine.get_client() execution_data = client.get_execution_data(self.id) # Inputs are returned inline unless they are too big, in which case a url blob pointing to them is returned. input_map: _literal_models.LiteralMap = _literal_models.LiteralMap( {}) if bool(execution_data.full_inputs.literals): input_map = execution_data.full_inputs elif execution_data.inputs.bytes > 0: with _common_utils.AutoDeletingTempDir() as tmp_dir: tmp_name = _os.path.join(tmp_dir.name, "inputs.pb") _data_proxy.Data.get_data(execution_data.inputs.url, tmp_name) input_map = _literal_models.LiteralMap.from_flyte_idl( _common_utils.load_proto_from_file( _literals_pb2.Literalmap, tmp_name)) lp_id = self.spec.launch_plan workflow = _workflow.FlyteWorkflow.fetch(lp_id.project, lp_id.domain, lp_id.name, lp_id.version) self._inputs = TypeEngine.literal_map_to_kwargs( ctx=FlyteContextManager.current_context(), lm=input_map, python_types=TypeEngine.guess_python_types( workflow.interface.inputs), ) return self._inputs
def inputs(self) -> Dict[str, Any]: """ Returns the inputs of the task execution in the standard Python format that is produced by the type engine. """ from flytekit.control_plane.tasks.task import FlyteTask if self._inputs is None: client = _flyte_engine.get_client() execution_data = client.get_task_execution_data(self.id) # Inputs are returned inline unless they are too big, in which case a url blob pointing to them is returned. input_map = _literal_models.LiteralMap({}) if bool(execution_data.full_inputs.literals): input_map = execution_data.full_inputs elif execution_data.inputs.bytes > 0: with _common_utils.AutoDeletingTempDir() as tmp_dir: tmp_name = os.path.join(tmp_dir.name, "inputs.pb") _data_proxy.Data.get_data(execution_data.inputs.url, tmp_name) input_map = _literal_models.LiteralMap.from_flyte_idl( _common_utils.load_proto_from_file( _literals_pb2.LiteralMap, tmp_name)) task = FlyteTask.fetch(self.id.task_id.project, self.id.task_id.domain, self.id.task_id.name, self.id.task_id.version) self._inputs = TypeEngine.literal_map_to_kwargs( ctx=FlyteContextManager.current_context(), lm=input_map, python_types=TypeEngine.guess_python_types( task.interface.inputs), ) return self._inputs
def outputs(self): """ Returns the outputs of the task execution, if available, in the standard Python format that is produced by the type engine. If not available, perhaps due to execution being in progress or an error being produced, this will raise an exception. :rtype: dict[Text, T] """ if not self.is_complete: raise _user_exceptions.FlyteAssertion( "Please what until the task execution has completed before requesting the outputs." ) if self.error: raise _user_exceptions.FlyteAssertion("Outputs could not be found because the execution ended in failure.") if self._outputs is None: client = _flyte_engine.get_client() execution_data = client.get_task_execution_data(self.id) # Inputs are returned inline unless they are too big, in which case a url blob pointing to them is returned. if bool(execution_data.full_outputs.literals): output_map = execution_data.full_outputs elif execution_data.outputs.bytes > 0: with _common_utils.AutoDeletingTempDir() as t: tmp_name = _os.path.join(t.name, "outputs.pb") _data_proxy.Data.get_data(execution_data.outputs.url, tmp_name) output_map = _literal_models.LiteralMap.from_flyte_idl( _common_utils.load_proto_from_file(_literals_pb2.LiteralMap, tmp_name) ) else: output_map = _literal_models.LiteralMap({}) self._outputs = _type_helpers.unpack_literal_map_to_sdk_python_std(output_map) return self._outputs
def test_single_step_entrypoint_out_of_proc(): with _TemporaryConfiguration(os.path.join(os.path.dirname(__file__), 'fake.config'), internal_overrides={ 'project': 'test', 'domain': 'development' }): with _utils.AutoDeletingTempDir("in") as input_dir: literal_map = _type_helpers.pack_python_std_map_to_literal_map({'a': 9}, _type_map_from_variable_map( _task_defs.add_one.interface.inputs)) input_file = os.path.join(input_dir.name, "inputs.pb") _utils.write_proto_to_file(literal_map.to_flyte_idl(), input_file) with _utils.AutoDeletingTempDir("out") as output_dir: cmd = [] cmd.extend(["--task-module", _task_defs.add_one.task_module]) cmd.extend(["--task-name", _task_defs.add_one.task_function_name]) cmd.extend(["--inputs", input_file]) cmd.extend(["--output-prefix", output_dir.name]) result = CliRunner().invoke(execute_task_cmd, cmd) assert result.exit_code == 0 p = _utils.load_proto_from_file( _literals_pb2.LiteralMap, os.path.join(output_dir.name, _constants.OUTPUT_FILE_NAME) ) raw_map = _type_helpers.unpack_literal_map_to_sdk_python_std( _literal_models.LiteralMap.from_flyte_idl(p), _type_map_from_variable_map(_task_defs.add_one.interface.outputs) ) assert raw_map['b'] == 10 assert len(raw_map) == 1
def test_single_step_entrypoint_in_proc(): with _TemporaryConfiguration(os.path.join(os.path.dirname(__file__), 'fake.config'), internal_overrides={ 'project': 'test', 'domain': 'development' }): with _utils.AutoDeletingTempDir("in") as input_dir: literal_map = _type_helpers.pack_python_std_map_to_literal_map( {'a': 9}, _type_map_from_variable_map(_task_defs.add_one.interface.inputs)) input_file = os.path.join(input_dir.name, "inputs.pb") _utils.write_proto_to_file(literal_map.to_flyte_idl(), input_file) with _utils.AutoDeletingTempDir("out") as output_dir: _execute_task( _task_defs.add_one.task_module, _task_defs.add_one.task_function_name, input_file, output_dir.name, False ) p = _utils.load_proto_from_file( _literals_pb2.LiteralMap, os.path.join(output_dir.name, _constants.OUTPUT_FILE_NAME) ) raw_map = _type_helpers.unpack_literal_map_to_sdk_python_std( _literal_models.LiteralMap.from_flyte_idl(p), _type_map_from_variable_map(_task_defs.add_one.interface.outputs) ) assert raw_map['b'] == 10 assert len(raw_map) == 1
def inputs(self): """ Returns the inputs of the task execution in the standard Python format that is produced by the type engine. :rtype: dict[Text, T] """ if self._inputs is None: client = _flyte_engine.get_client() execution_data = client.get_task_execution_data(self.id) # Inputs are returned inline unless they are too big, in which case a url blob pointing to them is returned. if bool(execution_data.full_inputs.literals): input_map = execution_data.full_inputs elif execution_data.inputs.bytes > 0: with _common_utils.AutoDeletingTempDir() as t: tmp_name = _os.path.join(t.name, "inputs.pb") _data_proxy.Data.get_data(execution_data.inputs.url, tmp_name) input_map = _literal_models.LiteralMap.from_flyte_idl( _common_utils.load_proto_from_file(_literals_pb2.LiteralMap, tmp_name) ) else: input_map = _literal_models.LiteralMap({}) self._inputs = _type_helpers.unpack_literal_map_to_sdk_python_std(input_map) return self._inputs
def outputs(self) -> Dict[str, Any]: """ Returns the outputs to the execution in the standard python format as dictated by the type engine. :raises: ``FlyteAssertion`` error if execution is in progress or execution ended in error. """ if not self.is_complete: raise _user_exceptions.FlyteAssertion( "Please wait until the node execution has completed before requesting the outputs." ) if self.error: raise _user_exceptions.FlyteAssertion( "Outputs could not be found because the execution ended in failure." ) if self._outputs is None: client = _flyte_engine.get_client() execution_data = client.get_execution_data(self.id) # Outputs are returned inline unless they are too big, in which case a url blob pointing to them is returned. output_map: LiteralMap = _literal_models.LiteralMap({}) if bool(execution_data.full_outputs.literals): output_map = execution_data.full_outputs elif execution_data.outputs.bytes > 0: with _common_utils.AutoDeletingTempDir() as tmp_dir: tmp_name = _os.path.join(tmp_dir.name, "outputs.pb") _data_proxy.Data.get_data(execution_data.outputs.url, tmp_name) output_map = _literal_models.LiteralMap.from_flyte_idl( _common_utils.load_proto_from_file( _literals_pb2.LiteralMap, tmp_name)) # TODO: need to convert flyte literals to python types. For now just use literals # self._outputs = TypeEngine.literal_map_to_kwargs(ctx=FlyteContext.current_context(), lm=output_map) self._outputs = output_map return self._outputs
def execute_task(task_module, task_name, inputs, output_prefix, test): with _TemporaryConfiguration(_internal_config.CONFIGURATION_PATH.get()): with _utils.AutoDeletingTempDir('input_dir') as input_dir: # Load user code task_module = _importlib.import_module(task_module) task_def = getattr(task_module, task_name) if not test: local_inputs_file = input_dir.get_named_tempfile('inputs.pb') # Handle inputs/outputs for array job. if _os.environ.get('BATCH_JOB_ARRAY_INDEX_VAR_NAME'): job_index = _compute_array_job_index() # TODO: Perhaps remove. This is a workaround to an issue we perceived with limited entropy in # TODO: AWS batch array jobs. _flyte_random.seed_flyte_random("{} {} {}".format( _random.random(), _datetime.datetime.utcnow(), job_index)) # If an ArrayTask is discoverable, the original job index may be different than the one specified in # the environment variable. Look up the correct input/outputs in the index lookup mapping file. job_index = _map_job_index_to_child_index( input_dir, inputs, job_index) inputs = _os.path.join(inputs, str(job_index), 'inputs.pb') output_prefix = _os.path.join(output_prefix, str(job_index)) _data_proxy.Data.get_data(inputs, local_inputs_file) input_proto = _utils.load_proto_from_file( _literals_pb2.LiteralMap, local_inputs_file) _engine_loader.get_engine().get_task(task_def).execute( _literal_models.LiteralMap.from_flyte_idl(input_proto), context={'output_prefix': output_prefix})
def _dispatch_execute(ctx: FlyteContext, task_def: PythonTask, inputs_path: str, output_prefix: str): """ Dispatches execute to PythonTask Step1: Download inputs and load into a literal map Step2: Invoke task - dispatch_execute Step3: a: [Optional] Record outputs to output_prefix b: OR if IgnoreOutputs is raised, then ignore uploading outputs c: OR if an unhandled exception is retrieved - record it as an errors.pb """ try: # Step1 local_inputs_file = _os.path.join(ctx.execution_state.working_dir, "inputs.pb") ctx.file_access.get_data(inputs_path, local_inputs_file) input_proto = _utils.load_proto_from_file(_literals_pb2.LiteralMap, local_inputs_file) idl_input_literals = _literal_models.LiteralMap.from_flyte_idl( input_proto) # Step2 outputs = task_def.dispatch_execute(ctx, idl_input_literals) if isinstance(outputs, VoidPromise): _logging.getLogger().warning("Task produces no outputs") output_file_dict = { _constants.OUTPUT_FILE_NAME: _literal_models.LiteralMap(literals={}) } elif isinstance(outputs, _literal_models.LiteralMap): output_file_dict = {_constants.OUTPUT_FILE_NAME: outputs} elif isinstance(outputs, _dynamic_job.DynamicJobSpec): output_file_dict = {_constants.FUTURES_FILE_NAME: outputs} else: _logging.getLogger().error( f"SystemError: received unknown outputs from task {outputs}") # TODO This should probably cause an error file return for k, v in output_file_dict.items(): _common_utils.write_proto_to_file( v.to_flyte_idl(), _os.path.join(ctx.execution_state.engine_dir, k)) # Step3a ctx.file_access.upload_directory(ctx.execution_state.engine_dir, output_prefix) _logging.info( f"Outputs written successful the the output prefix {output_prefix}" ) except Exception as e: if isinstance(e, IgnoreOutputs): # Step 3b _logging.warning( f"IgnoreOutputs received! Outputs.pb will not be uploaded. reason {e}" ) return # Step 3c _logging.error( f"Exception when executing task {task_def.name}, reason {str(e)}") raise e
def inputs(self) -> Dict[str, Any]: """ Returns the inputs to the execution in the standard python format as dicatated by the type engine. """ if self._inputs is None: client = _flyte_engine.get_client() execution_data = client.get_node_execution_data(self.id) # Inputs are returned inline unless they are too big, in which case a url blob pointing to them is returned. input_map: _literal_models.LiteralMap = _literal_models.LiteralMap( {}) if bool(execution_data.full_inputs.literals): input_map = execution_data.full_inputs elif execution_data.inputs.bytes > 0: with _common_utils.AutoDeletingTempDir() as tmp_dir: tmp_name = _os.path.join(tmp_dir.name, "inputs.pb") _data_proxy.Data.get_data(execution_data.inputs.url, tmp_name) input_map = _literal_models.LiteralMap.from_flyte_idl( _common_utils.load_proto_from_file( _literals_pb2.LiteralMap, tmp_name)) # TODO: need to convert flyte literals to python types. For now just use literals # self._inputs = TypeEngine.literal_map_to_kwargs(ctx=FlyteContext.current_context(), lm=input_map) self._inputs = input_map return self._inputs
def test_task_system_failure(): with TemporaryConfiguration(os.path.join( os.path.dirname(os.path.realpath(__file__)), '../../../common/configs/local.config'), internal_overrides={ 'image': 'myflyteimage:{}'.format( os.environ.get('IMAGE_VERSION', 'sha')), 'project': 'myflyteproject', 'domain': 'development' }): m = MagicMock() m.execute = _raise_system_exception with utils.AutoDeletingTempDir("test") as tmp: engine.FlyteTask(m).execute(None, {'output_prefix': tmp.name}) doc = errors.ErrorDocument.from_flyte_idl( utils.load_proto_from_file( errors_pb2.ErrorDocument, os.path.join(tmp.name, constants.ERROR_FILE_NAME))) assert doc.error.code == "SYSTEM:Unknown" assert doc.error.kind == errors.ContainerError.Kind.RECOVERABLE assert "errorERRORerror" in doc.error.message
def test_arrayjob_entrypoint_in_proc(): with _TemporaryConfiguration(os.path.join(os.path.dirname(__file__), 'fake.config'), internal_overrides={ 'project': 'test', 'domain': 'development' }): with _utils.AutoDeletingTempDir("dir") as dir: literal_map = _type_helpers.pack_python_std_map_to_literal_map( {'a': 9}, _type_map_from_variable_map( _task_defs.add_one.interface.inputs)) input_dir = os.path.join(dir.name, "1") os.mkdir( input_dir) # auto cleanup will take this subdir into account input_file = os.path.join(input_dir, "inputs.pb") _utils.write_proto_to_file(literal_map.to_flyte_idl(), input_file) # construct indexlookup.pb which has array: [1] mapped_index = _literals.Literal( _literals.Scalar(primitive=_literals.Primitive(integer=1))) index_lookup_collection = _literals.LiteralCollection( [mapped_index]) index_lookup_file = os.path.join(dir.name, "indexlookup.pb") _utils.write_proto_to_file(index_lookup_collection.to_flyte_idl(), index_lookup_file) # fake arrayjob task by setting environment variables orig_env_index_var_name = os.environ.get( 'BATCH_JOB_ARRAY_INDEX_VAR_NAME') orig_env_array_index = os.environ.get('AWS_BATCH_JOB_ARRAY_INDEX') os.environ[ 'BATCH_JOB_ARRAY_INDEX_VAR_NAME'] = 'AWS_BATCH_JOB_ARRAY_INDEX' os.environ['AWS_BATCH_JOB_ARRAY_INDEX'] = '0' execute_task(_task_defs.add_one.task_module, _task_defs.add_one.task_function_name, dir.name, dir.name, False) raw_map = _type_helpers.unpack_literal_map_to_sdk_python_std( _literal_models.LiteralMap.from_flyte_idl( _utils.load_proto_from_file( _literals_pb2.LiteralMap, os.path.join(input_dir, _constants.OUTPUT_FILE_NAME))), _type_map_from_variable_map( _task_defs.add_one.interface.outputs)) assert raw_map['b'] == 10 assert len(raw_map) == 1 # reset the env vars if orig_env_index_var_name: os.environ[ 'BATCH_JOB_ARRAY_INDEX_VAR_NAME'] = orig_env_index_var_name if orig_env_array_index: os.environ['AWS_BATCH_JOB_ARRAY_INDEX'] = orig_env_array_index
def get_outputs(self): """ :rtype: flytekit.models.literals.LiteralMap """ with _common_utils.AutoDeletingTempDir() as t: tmp_name = _os.path.join(t.name, "outputs.pb") _data_proxy.Data.get_data( self.sdk_task_execution.closure.output_uri, tmp_name) return _literals.LiteralMap.from_flyte_idl( _common_utils.load_proto_from_file(_literals_pb2.LiteralMap, tmp_name))
def test_task_user_failure(): m = MagicMock() m.execute = _raise_user_exception with utils.AutoDeletingTempDir("test") as tmp: engine.FlyteTask(m).execute(None, {'output_prefix': tmp.name}) doc = errors.ErrorDocument.from_flyte_idl( utils.load_proto_from_file(errors_pb2.ErrorDocument, os.path.join(tmp.name, constants.ERROR_FILE_NAME)) ) assert doc.error.code == "USER:Unknown" assert doc.error.kind == errors.ContainerError.Kind.NON_RECOVERABLE assert "userUSERuser" in doc.error.message
def load_task(self, loader_args: List[str]) -> ExecutableTemplateShimTask: logger.info(f"Task template loader args: {loader_args}") ctx = FlyteContext.current_context() task_template_local_path = os.path.join( ctx.execution_state.working_dir, "task_template.pb") ctx.file_access.get_data(loader_args[0], task_template_local_path) task_template_proto = common_utils.load_proto_from_file( _tasks_pb2.TaskTemplate, task_template_local_path) task_template_model = _task_model.TaskTemplate.from_flyte_idl( task_template_proto) executor_class = load_object_from_module(loader_args[1]) return ExecutableTemplateShimTask(task_template_model, executor_class)
def get_outputs(self): """ :rtype: flytekit.models.literals.LiteralMap """ client = _FlyteClientManager(_platform_config.URL.get(), insecure=_platform_config.INSECURE.get()).client url_blob = client.get_task_execution_data(self.sdk_task_execution.id) if url_blob.outputs.bytes > 0: with _common_utils.AutoDeletingTempDir() as t: tmp_name = _os.path.join(t.name, "outputs.pb") _data_proxy.Data.get_data(url_blob.outputs.url, tmp_name) return _literals.LiteralMap.from_flyte_idl( _common_utils.load_proto_from_file(_literals_pb2.LiteralMap, tmp_name) ) return _literals.LiteralMap({})
def _map_job_index_to_child_index(local_input_dir, datadir, index): local_lookup_file = local_input_dir.get_named_tempfile('indexlookup.pb') idx_lookup_file = _os.path.join(datadir, 'indexlookup.pb') # if the indexlookup.pb does not exist, then just return the index if not _data_proxy.Data.data_exists(idx_lookup_file): return index _data_proxy.Data.get_data(idx_lookup_file, local_lookup_file) mapping_proto = _utils.load_proto_from_file(_literals_pb2.LiteralCollection, local_lookup_file) if len(mapping_proto.literals) < index: raise _system_exceptions.FlyteSystemAssertion( "dynamic task index lookup array size: {} is smaller than lookup index {}".format( len(mapping_proto.literals), index)) return mapping_proto.literals[index].scalar.primitive.integer
def outputs(self) -> Dict[str, Any]: """ Returns the outputs of the task execution, if available, in the standard Python format that is produced by the type engine. :raises: ``FlyteAssertion`` error if execution is in progress or execution ended in error. """ from flytekit.control_plane.tasks.task import FlyteTask if not self.is_complete: raise _user_exceptions.FlyteAssertion( "Please what until the task execution has completed before requesting the outputs." ) if self.error: raise _user_exceptions.FlyteAssertion( "Outputs could not be found because the execution ended in failure." ) if self._outputs is None: client = _flyte_engine.get_client() execution_data = client.get_task_execution_data(self.id) # Inputs are returned inline unless they are too big, in which case a url blob pointing to them is returned. output_map = _literal_models.LiteralMap({}) if bool(execution_data.full_outputs.literals): output_map = execution_data.full_outputs elif execution_data.outputs.bytes > 0: with _common_utils.AutoDeletingTempDir() as t: tmp_name = os.path.join(t.name, "outputs.pb") _data_proxy.Data.get_data(execution_data.outputs.url, tmp_name) output_map = _literal_models.LiteralMap.from_flyte_idl( _common_utils.load_proto_from_file( _literals_pb2.LiteralMap, tmp_name)) task = FlyteTask.fetch(self.id.task_id.project, self.id.task_id.domain, self.id.task_id.name, self.id.task_id.version) self._outputs = TypeEngine.literal_map_to_kwargs( ctx=FlyteContextManager.current_context(), lm=output_map, python_types=TypeEngine.guess_python_types( task.interface.outputs), ) return self._outputs
def get_outputs(self): """ :rtype: flytekit.models.literals.LiteralMap """ client = _FlyteClientManager( _platform_config.URL.get(), insecure=_platform_config.INSECURE.get()).client execution_data = client.get_task_execution_data( self.sdk_task_execution.id) # Inputs are returned inline unless they are too big, in which case a url blob pointing to them is returned. if bool(execution_data.full_outputs.literals): return execution_data.full_outputs if execution_data.outputs.bytes > 0: with _common_utils.AutoDeletingTempDir() as t: tmp_name = _os.path.join(t.name, "outputs.pb") _data_proxy.Data.get_data(execution_data.outputs.url, tmp_name) return _literals.LiteralMap.from_flyte_idl( _common_utils.load_proto_from_file( _literals_pb2.LiteralMap, tmp_name)) return _literals.LiteralMap({})
def _dispatch_execute(ctx: FlyteContext, task_def: PythonTask, inputs_path: str, output_prefix: str): """ Dispatches execute to PythonTask Step1: Download inputs and load into a literal map Step2: Invoke task - dispatch_execute Step3: a: [Optional] Record outputs to output_prefix b: OR if IgnoreOutputs is raised, then ignore uploading outputs c: OR if an unhandled exception is retrieved - record it as an errors.pb """ output_file_dict = {} try: # Step1 local_inputs_file = _os.path.join(ctx.execution_state.working_dir, "inputs.pb") ctx.file_access.get_data(inputs_path, local_inputs_file) input_proto = _utils.load_proto_from_file(_literals_pb2.LiteralMap, local_inputs_file) idl_input_literals = _literal_models.LiteralMap.from_flyte_idl(input_proto) # Step2 outputs = task_def.dispatch_execute(ctx, idl_input_literals) # Step3a if isinstance(outputs, VoidPromise): _logging.getLogger().warning("Task produces no outputs") output_file_dict = {_constants.OUTPUT_FILE_NAME: _literal_models.LiteralMap(literals={})} elif isinstance(outputs, _literal_models.LiteralMap): output_file_dict = {_constants.OUTPUT_FILE_NAME: outputs} elif isinstance(outputs, _dynamic_job.DynamicJobSpec): output_file_dict = {_constants.FUTURES_FILE_NAME: outputs} else: _logging.getLogger().error(f"SystemError: received unknown outputs from task {outputs}") output_file_dict[_constants.ERROR_FILE_NAME] = _error_models.ErrorDocument( _error_models.ContainerError( "UNKNOWN_OUTPUT", f"Type of output received not handled {type(outputs)} outputs: {outputs}", _error_models.ContainerError.Kind.RECOVERABLE, ) ) except _scoped_exceptions.FlyteScopedException as e: _logging.error("!! Begin Error Captured by Flyte !!") output_file_dict[_constants.ERROR_FILE_NAME] = _error_models.ErrorDocument( _error_models.ContainerError(e.error_code, e.verbose_message, e.kind) ) _logging.error(e.verbose_message) _logging.error("!! End Error Captured by Flyte !!") except Exception as e: if isinstance(e, IgnoreOutputs): # Step 3b _logging.warning(f"IgnoreOutputs received! Outputs.pb will not be uploaded. reason {e}") return # Step 3c _logging.error(f"Exception when executing task {task_def.name}, reason {str(e)}") _logging.error("!! Begin Unknown System Error Captured by Flyte !!") exc_str = _traceback.format_exc() output_file_dict[_constants.ERROR_FILE_NAME] = _error_models.ErrorDocument( _error_models.ContainerError("SYSTEM:Unknown", exc_str, _error_models.ContainerError.Kind.RECOVERABLE,) ) _logging.error(exc_str) _logging.error("!! End Error Captured by Flyte !!") for k, v in output_file_dict.items(): _common_utils.write_proto_to_file(v.to_flyte_idl(), _os.path.join(ctx.execution_state.engine_dir, k)) ctx.file_access.upload_directory(ctx.execution_state.engine_dir, output_prefix) _logging.info(f"Engine folder written successfully to the output prefix {output_prefix}")