def execute(self, inputs):
"""
:param flytekit.models.literals.LiteralMap inputs:
:rtype: dict[Text, flytekit.models.common.FlyteIdlEntity]
:returns: This function must return a dictionary mapping 'filenames' to Flyte Interface Entities. These
entities will be used by the engine to pass data from node to node, populate metadata, etc. etc.. Each
engine will have different behavior. For instance, the Flyte engine will upload the entities to a remote
working directory (with the names provided), which will in turn allow Flyte Propeller to push along the
workflow. Where as local engine will merely feed the outputs directly into the next node.
"""
inputs_dict = _type_helpers.unpack_literal_map_to_sdk_python_std(inputs, {
k: _type_helpers.get_sdk_type_from_literal_type(v.type) for k, v in _six.iteritems(self.interface.inputs)
})
outputs_dict = {
name: _task_output.OutputReference(_type_helpers.get_sdk_type_from_literal_type(variable.type))
for name, variable in _six.iteritems(self.interface.outputs)
}
inputs_dict.update(outputs_dict)
_exception_scopes.user_entry_point(self.task_function)(
**inputs_dict
)
return {
_constants.OUTPUT_FILE_NAME: _literal_models.LiteralMap(
literals={k: v.sdk_value for k, v in _six.iteritems(outputs_dict)}
)
}
def get_engine(engine_name=None):
"""
:param Text engine_name:
:rtype: flytekit.engines.common.BaseExecutionEngineFactory
"""
engine_name = engine_name or _sdk_config.EXECUTION_ENGINE.get()
# TODO: Allow users to plug-in their own engine code via a config
if engine_name not in _ENGINE_NAME_TO_MODULES_CACHE:
raise _user_exceptions.FlyteValueException(
engine_name,
"Could not load an engine with the identifier '{}'. Known engines are: {}".format(
engine_name, list(_ENGINE_NAME_TO_MODULES_CACHE.keys())
),
)
module_path, attr, engine_impl = _ENGINE_NAME_TO_MODULES_CACHE[engine_name]
if engine_impl is None:
module = _exception_scopes.user_entry_point(_importlib.import_module)(module_path)
if not hasattr(module, attr):
raise _user_exceptions.FlyteValueException(
module,
"Failed to load the engine because the attribute named '{}' could not be found"
"in the module '{}'.".format(attr, module_path),
)
engine_impl = getattr(module, attr)()
_ENGINE_NAME_TO_MODULES_CACHE[engine_name] = (module_path, attr, engine_impl)
return engine_impl
def _execute_user_code(self, context, inputs):
"""
:param flytekit.engines.common.tasks.sagemaker.distribution.DistributedTrainingEngineContext context:
:param dict[Text, T] inputs: This variable is a bit of a misnomer, since it's both inputs and outputs. The
dictionary passed here will be passed to the user-defined function, and will have values that are a
variety of types. The T's here are Python std values for inputs. If there isn't a native Python type for
something (like Schema or Blob), they are the Flyte classes. For outputs they are OutputReferences.
(Note that these are not the same OutputReferences as in BindingData's)
:rtype: Any: the returned object from user code.
:returns: This function must return a dictionary mapping 'filenames' to Flyte Interface Entities. These
entities will be used by the engine to pass data from node to node, populate metadata, etc. etc.. Each
engine will have different behavior. For instance, the Flyte engine will upload the entities to a remote
working directory (with the names provided), which will in turn allow Flyte Propeller to push along the
workflow. Where as local engine will merely feed the outputs directly into the next node.
"""
return _exception_scopes.user_entry_point(self.task_function)(
_sm_distribution.DistributedTrainingExecutionParam(
execution_date=context.execution_date,
# TODO: it might be better to consider passing the full struct
execution_id=_six.text_type(
WorkflowExecutionIdentifier.promote_from_model(
context.execution_id)),
stats=context.stats,
logging=context.logging,
tmp_dir=context.working_directory,
distributed_training_context=context.
distributed_training_context,
),
**inputs)
def _load_engines(cls):
config = _sdk_config.TYPE_ENGINES.get()
if cls._LOADED_ENGINES is None or config != cls._LAST_LOADED:
cls._LAST_LOADED = config
cls._LOADED_ENGINES = []
for fqdn in config:
split = fqdn.split(".")
module_path, attr = ".".join(split[:-1]), split[-1]
module = _exception_scopes.user_entry_point(
_importlib.import_module)(module_path)
if not hasattr(module, attr):
raise _user_exceptions.FlyteValueException(
module,
"Failed to load the type engine because the attribute named '{}' could not be found"
"in the module '{}'.".format(attr, module_path))
engine_impl = getattr(module, attr)()
cls._LOADED_ENGINES.append(engine_impl)
from flytekit.type_engines.default.flyte import FlyteDefaultTypeEngine as _DefaultEngine
cls._LOADED_ENGINES.append(_DefaultEngine())
