def register_and_launch(self, project, domain, name=None, version=None, inputs=None):
"""
:param Text project: The project in which to register and launch this task.
:param Text domain: The domain in which to register and launch this task.
:param Text name: The name to give this task.
:param Text version: The version in which to register this task
:param dict[Text, Any] inputs: A dictionary of Python standard inputs that will be type-checked, then compiled
to a LiteralMap.
:rtype: flytekit.common.workflow_execution.SdkWorkflowExecution
"""
self.validate()
version = self._produce_deterministic_version(version)
if name is None:
try:
self.auto_assign_name()
generated_name = self._platform_valid_name
except _system_exceptions.FlyteSystemException:
# If we're not able to assign a platform valid name, use the deterministically-produced version instead.
generated_name = version
name = name if name else generated_name
id_to_register = _identifier.Identifier(_identifier_model.ResourceType.TASK, project, domain, name, version)
old_id = self.id
try:
self._id = id_to_register
_engine_loader.get_engine().get_task(self).register(id_to_register)
except:
self._id = old_id
raise
return self.launch(project, domain, inputs=inputs)
def _sync_closure(self):
"""
Syncs the closure of the underlying execution artifact with the state observed by the platform.
:rtype: None
"""
if not self.is_complete:
_engine_loader.get_engine().get_workflow_execution(self).sync()
def execute_task(task_module, task_name, inputs, output_prefix, test):
with _TemporaryConfiguration(_internal_config.CONFIGURATION_PATH.get()):
with _utils.AutoDeletingTempDir('input_dir') as input_dir:
# Load user code
task_module = _importlib.import_module(task_module)
task_def = getattr(task_module, task_name)
if not test:
local_inputs_file = input_dir.get_named_tempfile('inputs.pb')
# Handle inputs/outputs for array job.
if _os.environ.get('BATCH_JOB_ARRAY_INDEX_VAR_NAME'):
job_index = _compute_array_job_index()
# TODO: Perhaps remove. This is a workaround to an issue we perceived with limited entropy in
# TODO: AWS batch array jobs.
_flyte_random.seed_flyte_random("{} {} {}".format(
_random.random(), _datetime.datetime.utcnow(),
job_index))
# If an ArrayTask is discoverable, the original job index may be different than the one specified in
# the environment variable. Look up the correct input/outputs in the index lookup mapping file.
job_index = _map_job_index_to_child_index(
input_dir, inputs, job_index)
inputs = _os.path.join(inputs, str(job_index), 'inputs.pb')
output_prefix = _os.path.join(output_prefix,
str(job_index))
_data_proxy.Data.get_data(inputs, local_inputs_file)
input_proto = _utils.load_proto_from_file(
_literals_pb2.LiteralMap, local_inputs_file)
_engine_loader.get_engine().get_task(task_def).execute(
_literal_models.LiteralMap.from_flyte_idl(input_proto),
context={'output_prefix': output_prefix})
def sync(self):
"""
Syncs the state of the underlying execution artifact with the state observed by the platform.
:rtype: None
"""
if not self.is_complete or self._node_executions is None:
_engine_loader.get_engine().get_workflow_execution(self).sync()
self._node_executions = self.get_node_executions()
def register(self, project, domain, name, version):
"""
:param Text project:
:param Text domain:
:param Text name:
:param Text version:
"""
self.validate()
id_to_register = _identifier.Identifier(
_identifier_model.ResourceType.LAUNCH_PLAN, project, domain, name,
version)
_engine_loader.get_engine().get_launch_plan(self).register(
id_to_register)
self._id = id_to_register
return _six.text_type(self.id)
def fetch(cls, project, domain, name, version=None):
"""
This function uses the engine loader to call create a hydrated task from Admin.
:param Text project:
:param Text domain:
:param Text name:
:param Text version:
:rtype: SdkWorkflow
"""
version = version or _internal_config.VERSION.get()
workflow_id = _identifier.Identifier(
_identifier_model.ResourceType.WORKFLOW, project, domain, name,
version)
admin_workflow = _engine_loader.get_engine().fetch_workflow(
workflow_id)
cwc = admin_workflow.closure.compiled_workflow
primary_template = cwc.primary.template
sub_workflow_map = {
sw.template.id: sw.template
for sw in cwc.sub_workflows
}
task_map = {t.template.id: t.template for t in cwc.tasks}
sdk_workflow = cls.promote_from_model(primary_template,
sub_workflow_map, task_map)
sdk_workflow._id = workflow_id
return sdk_workflow
def launch_with_literals(self, project, domain, literal_inputs, name=None, notification_overrides=None,
label_overrides=None, annotation_overrides=None):
"""
Launches a single task execution and returns the execution identifier.
:param Text project:
:param Text domain:
:param flytekit.models.literals.LiteralMap literal_inputs: Inputs to the execution.
:param Text name: [Optional] If specified, an execution will be created with this name. Note: the name must
be unique within the context of the project and domain.
:param list[flytekit.common.notifications.Notification] notification_overrides: [Optional] If specified, these
are the notifications that will be honored for this execution. An empty list signals to disable all
notifications.
:param flytekit.models.common.Labels label_overrides:
:param flytekit.models.common.Annotations annotation_overrides:
:rtype: flytekit.common.workflow_execution.SdkWorkflowExecution
"""
execution = _engine_loader.get_engine().get_task(self).launch(
project,
domain,
name=name,
inputs=literal_inputs,
notification_overrides=notification_overrides,
label_overrides=label_overrides,
annotation_overrides=annotation_overrides,
)
return _workflow_execution.SdkWorkflowExecution.promote_from_model(execution)
def _sync_closure(self):
"""
Syncs the closure of the underlying execution artifact with the state observed by the platform.
:rtype: None
"""
ne = _engine_loader.get_engine().get_node_execution(self)
ne.sync()
def register(self, project, domain, name, version):
"""
:param Text project: The project in which to register this task.
:param Text domain: The domain in which to register this task.
:param Text name: The name to give this task.
:param Text version: The version in which to register this task.
"""
self.validate()
id_to_register = _identifier.Identifier(_identifier_model.ResourceType.TASK, project, domain, name, version)
old_id = self.id
try:
self._id = id_to_register
_engine_loader.get_engine().get_task(self).register(id_to_register)
return _six.text_type(self.id)
except:
self._id = old_id
raise
def update(self, state):
"""
:param int state: Enum value from flytekit.models.launch_plan.LaunchPlanState
"""
if not self.id:
raise _user_exceptions.FlyteAssertion(
"Failed to update launch plan because the launch plan's ID is not set. Please call register to fetch "
"or register the identifier first")
return _engine_loader.get_engine().get_launch_plan(self).update(
self.id, state)
def inputs(self):
"""
Returns the inputs to the execution in the standard Python format as dictated by the type engine.
:rtype: dict[Text, T]
"""
if self._inputs is None:
self._inputs = _type_helpers.unpack_literal_map_to_sdk_python_std(
_engine_loader.get_engine().get_node_execution(
self).get_inputs())
return self._inputs
def register(self, project, domain, name, version):
"""
:param Text project:
:param Text domain:
:param Text name:
:param Text version:
"""
self.validate()
id_to_register = _identifier.Identifier(
_identifier_model.ResourceType.WORKFLOW, project, domain, name,
version)
old_id = self.id
try:
self._id = id_to_register
_engine_loader.get_engine().get_workflow(self).register(
id_to_register)
return _six.text_type(self.id)
except:
self._id = old_id
raise
def get_node_executions(self, filters=None):
"""
:param list[flytekit.models.filters.Filter] filters:
:rtype: dict[Text, flytekit.common.nodes.SdkNodeExecution]
"""
models = _engine_loader.get_engine().get_workflow_execution(
self).get_node_executions(filters=filters)
return {
k: _nodes.SdkNodeExecution.promote_from_model(v)
for k, v in _six.iteritems(models)
}
def fetch(cls, project, domain, name):
"""
:param Text project:
:param Text domain:
:param Text name:
:rtype: SdkWorkflowExecution
"""
return cls.promote_from_model(
_engine_loader.get_engine().fetch_workflow_execution(
_core_identifier.WorkflowExecutionIdentifier(project=project,
domain=domain,
name=name)))
def sync(self):
"""
Syncs the state of this object with that held by the platform.
:rtype: None
"""
if not self.is_complete or self.task_executions is not None:
ne = _engine_loader.get_engine().get_node_execution(self)
ne.sync()
self._task_executions = [
_task_executions.SdkTaskExecution.promote_from_model(te)
for te in ne.get_task_executions()
]
def unit_test(self, **input_map):
"""
:param dict[Text, T] input_map: Python Std input from users. We will cast these to the appropriate Flyte
literals.
:returns: Depends on the behavior of the specific task in the unit engine.
"""
return _engine_loader.get_engine('unit').get_task(self).execute(
_type_helpers.pack_python_std_map_to_literal_map(
input_map, {
k: _type_helpers.get_sdk_type_from_literal_type(v.type)
for k, v in _six.iteritems(self.interface.inputs)
}))
def local_execute(self, **input_map):
"""
:param dict[Text, T] input_map: Python Std input from users. We will cast these to the appropriate Flyte
literals.
:rtype: dict[Text, T]
:returns: The output produced by this task in Python standard format.
"""
return _engine_loader.get_engine('local').get_task(self).execute(
_type_helpers.pack_python_std_map_to_literal_map(
input_map, {
k: _type_helpers.get_sdk_type_from_literal_type(v.type)
for k, v in _six.iteritems(self.interface.inputs)
}))
def fetch(cls, project, domain, name, version):
"""
This function uses the engine loader to call create a hydrated task from Admin.
:param Text project:
:param Text domain:
:param Text name:
:param Text version:
:rtype: SdkTask
"""
task_id = _identifier.Identifier(_identifier_model.ResourceType.TASK, project, domain, name, version)
admin_task = _engine_loader.get_engine().fetch_task(task_id=task_id)
sdk_task = cls.promote_from_model(admin_task.closure.compiled_task.template)
sdk_task._id = task_id
return sdk_task
def fetch_latest(cls, project, domain, name):
"""
This function uses the engine loader to call create a latest hydrated task from Admin.
:param Text project:
:param Text domain:
:param Text name:
:rtype: SdkTask
"""
named_task = _common_model.NamedEntityIdentifier(project, domain, name)
admin_task = _engine_loader.get_engine().fetch_latest_task(named_task)
if not admin_task:
raise _user_exceptions.FlyteEntityNotExistException("Named task {} not found".format(named_task))
sdk_task = cls.promote_from_model(admin_task.closure.compiled_task.template)
sdk_task._id = admin_task.id
return sdk_task
def get_child_executions(self, filters=None):
"""
:param list[flytekit.models.filters.Filter] filters:
:rtype: dict[Text, flytekit.common.nodes.SdkNodeExecution]
"""
from flytekit.common import nodes as _nodes
if not self.is_parent:
raise _user_exceptions.FlyteAssertion(
"Only task executions marked with 'is_parent' have child executions."
)
models = _engine_loader.get_engine().get_task_execution(
self).get_child_executions(filters=filters)
return {
k: _nodes.SdkNodeExecution.promote_from_model(v)
for k, v in _six.iteritems(models)
}
def fetch(cls, project, domain, name, version=None):
"""
This function uses the engine loader to call create a hydrated task from Admin.
:param Text project:
:param Text domain:
:param Text name:
:param Text version:
:rtype: SdkLaunchPlan
"""
version = version or _internal_config.VERSION.get()
launch_plan_id = _identifier.Identifier(
_identifier_model.ResourceType.LAUNCH_PLAN, project, domain, name,
version)
lp = _engine_loader.get_engine().fetch_launch_plan(launch_plan_id)
sdk_lp = cls.promote_from_model(lp.spec)
sdk_lp._id = launch_plan_id
return sdk_lp
def outputs(self):
"""
Returns the outputs to the execution in the standard Python format as dictated by the type engine. If the
execution ended in error or the execution is in progress, an exception will be raised.
:rtype: dict[Text, T]
"""
if not self.is_complete:
raise _user_exceptions.FlyteAssertion(
"Please what until the node execution has completed before "
"requesting the outputs.")
if self.error:
raise _user_exceptions.FlyteAssertion(
"Outputs could not be found because the execution ended in failure."
)
if self._outputs is None:
self._outputs = _type_helpers.unpack_literal_map_to_sdk_python_std(
_engine_loader.get_engine().get_node_execution(
self).get_outputs())
return self._outputs
def outputs(self):
"""
Returns the outputs of the task execution, if available, in the standard Python format that is produced by
the type engine. If not available, perhaps due to execution being in progress or an error being produced,
this will raise an exception.
:rtype: dict[Text, T]
"""
if not self.is_complete:
raise _user_exceptions.FlyteAssertion(
"Please what until the task execution has completed before "
"requesting the outputs.")
if self.error:
raise _user_exceptions.FlyteAssertion(
"Outputs could not be found because the execution ended in failure."
)
if self._outputs is None:
self._outputs = _type_helpers.unpack_literal_map_to_sdk_python_std(
_engine_loader.get_engine().get_task_execution(
self).get_outputs())
return self._outputs
def execute_with_literals(self,
project,
domain,
literal_inputs,
name=None,
notification_overrides=None,
label_overrides=None,
annotation_overrides=None):
"""
Executes the launch plan and returns the execution identifier. This version of execution is meant for when
you already have a LiteralMap of inputs.
:param Text project:
:param Text domain:
:param flytekit.models.literals.LiteralMap literal_inputs: Inputs to the execution.
:param Text name: [Optional] If specified, an execution will be created with this name. Note: the name must
be unique within the context of the project and domain.
:param list[flytekit.common.notifications.Notification] notification_overrides: [Optional] If specified, these
are the notifications that will be honored for this execution. An empty list signals to disable all
notifications.
:param flytekit.models.common.Labels label_overrides:
:param flytekit.models.common.Annotations annotation_overrides:
:rtype: flytekit.common.workflow_execution.SdkWorkflowExecution
"""
# Kubernetes requires names starting with an alphabet for some resources.
name = name or "f" + _uuid.uuid4().hex[:19]
execution = _engine_loader.get_engine().get_launch_plan(self).execute(
project,
domain,
name,
literal_inputs,
notification_overrides=notification_overrides,
label_overrides=label_overrides,
annotation_overrides=annotation_overrides,
)
return _workflow_execution.SdkWorkflowExecution.promote_from_model(
execution)
def fetch(cls, project, domain, name, version=None):
"""
This function uses the engine loader to call create a hydrated task from Admin.
:param Text project:
:param Text domain:
:param Text name:
:param Text version: [Optional] If not set, the SDK will fetch the active launch plan for the given project,
domain, and name.
:rtype: SdkLaunchPlan
"""
from flytekit.common import workflow as _workflow
launch_plan_id = _identifier.Identifier(
_identifier_model.ResourceType.LAUNCH_PLAN, project, domain, name,
version)
lp = _engine_loader.get_engine().fetch_launch_plan(launch_plan_id)
sdk_lp = cls.promote_from_model(lp.spec)
sdk_lp._id = lp.id
# TODO: Add a test for this, and this function as a whole
wf_id = sdk_lp.workflow_id
lp_wf = _workflow.SdkWorkflow.fetch(wf_id.project, wf_id.domain,
wf_id.name, wf_id.version)
sdk_lp._interface = lp_wf.interface
return sdk_lp
def test_unit_load():
assert isinstance(loader.get_engine("unit"), _unit_engine.UnitTestEngineFactory)
def test_bad_load():
with pytest.raises(Exception):
loader.get_engine("badname")
def terminate(self, cause):
"""
:param Text cause:
"""
_engine_loader.get_engine().get_workflow_execution(self).terminate(
cause)
def sync(self):
_engine_loader.get_engine().get_task_execution(self).sync()
