def register_and_launch(self, project, domain, name=None, version=None, inputs=None): """ :param Text project: The project in which to register and launch this task. :param Text domain: The domain in which to register and launch this task. :param Text name: The name to give this task. :param Text version: The version in which to register this task :param dict[Text, Any] inputs: A dictionary of Python standard inputs that will be type-checked, then compiled to a LiteralMap. :rtype: flytekit.common.workflow_execution.SdkWorkflowExecution """ self.validate() version = self._produce_deterministic_version(version) if name is None: try: self.auto_assign_name() generated_name = self._platform_valid_name except _system_exceptions.FlyteSystemException: # If we're not able to assign a platform valid name, use the deterministically-produced version instead. generated_name = version name = name if name else generated_name id_to_register = _identifier.Identifier(_identifier_model.ResourceType.TASK, project, domain, name, version) old_id = self.id try: self._id = id_to_register _engine_loader.get_engine().get_task(self).register(id_to_register) except: self._id = old_id raise return self.launch(project, domain, inputs=inputs)
def _sync_closure(self): """ Syncs the closure of the underlying execution artifact with the state observed by the platform. :rtype: None """ if not self.is_complete: _engine_loader.get_engine().get_workflow_execution(self).sync()
def execute_task(task_module, task_name, inputs, output_prefix, test): with _TemporaryConfiguration(_internal_config.CONFIGURATION_PATH.get()): with _utils.AutoDeletingTempDir('input_dir') as input_dir: # Load user code task_module = _importlib.import_module(task_module) task_def = getattr(task_module, task_name) if not test: local_inputs_file = input_dir.get_named_tempfile('inputs.pb') # Handle inputs/outputs for array job. if _os.environ.get('BATCH_JOB_ARRAY_INDEX_VAR_NAME'): job_index = _compute_array_job_index() # TODO: Perhaps remove. This is a workaround to an issue we perceived with limited entropy in # TODO: AWS batch array jobs. _flyte_random.seed_flyte_random("{} {} {}".format( _random.random(), _datetime.datetime.utcnow(), job_index)) # If an ArrayTask is discoverable, the original job index may be different than the one specified in # the environment variable. Look up the correct input/outputs in the index lookup mapping file. job_index = _map_job_index_to_child_index( input_dir, inputs, job_index) inputs = _os.path.join(inputs, str(job_index), 'inputs.pb') output_prefix = _os.path.join(output_prefix, str(job_index)) _data_proxy.Data.get_data(inputs, local_inputs_file) input_proto = _utils.load_proto_from_file( _literals_pb2.LiteralMap, local_inputs_file) _engine_loader.get_engine().get_task(task_def).execute( _literal_models.LiteralMap.from_flyte_idl(input_proto), context={'output_prefix': output_prefix})
def sync(self): """ Syncs the state of the underlying execution artifact with the state observed by the platform. :rtype: None """ if not self.is_complete or self._node_executions is None: _engine_loader.get_engine().get_workflow_execution(self).sync() self._node_executions = self.get_node_executions()
def register(self, project, domain, name, version): """ :param Text project: :param Text domain: :param Text name: :param Text version: """ self.validate() id_to_register = _identifier.Identifier( _identifier_model.ResourceType.LAUNCH_PLAN, project, domain, name, version) _engine_loader.get_engine().get_launch_plan(self).register( id_to_register) self._id = id_to_register return _six.text_type(self.id)
def fetch(cls, project, domain, name, version=None): """ This function uses the engine loader to call create a hydrated task from Admin. :param Text project: :param Text domain: :param Text name: :param Text version: :rtype: SdkWorkflow """ version = version or _internal_config.VERSION.get() workflow_id = _identifier.Identifier( _identifier_model.ResourceType.WORKFLOW, project, domain, name, version) admin_workflow = _engine_loader.get_engine().fetch_workflow( workflow_id) cwc = admin_workflow.closure.compiled_workflow primary_template = cwc.primary.template sub_workflow_map = { sw.template.id: sw.template for sw in cwc.sub_workflows } task_map = {t.template.id: t.template for t in cwc.tasks} sdk_workflow = cls.promote_from_model(primary_template, sub_workflow_map, task_map) sdk_workflow._id = workflow_id return sdk_workflow
def launch_with_literals(self, project, domain, literal_inputs, name=None, notification_overrides=None, label_overrides=None, annotation_overrides=None): """ Launches a single task execution and returns the execution identifier. :param Text project: :param Text domain: :param flytekit.models.literals.LiteralMap literal_inputs: Inputs to the execution. :param Text name: [Optional] If specified, an execution will be created with this name. Note: the name must be unique within the context of the project and domain. :param list[flytekit.common.notifications.Notification] notification_overrides: [Optional] If specified, these are the notifications that will be honored for this execution. An empty list signals to disable all notifications. :param flytekit.models.common.Labels label_overrides: :param flytekit.models.common.Annotations annotation_overrides: :rtype: flytekit.common.workflow_execution.SdkWorkflowExecution """ execution = _engine_loader.get_engine().get_task(self).launch( project, domain, name=name, inputs=literal_inputs, notification_overrides=notification_overrides, label_overrides=label_overrides, annotation_overrides=annotation_overrides, ) return _workflow_execution.SdkWorkflowExecution.promote_from_model(execution)
def _sync_closure(self): """ Syncs the closure of the underlying execution artifact with the state observed by the platform. :rtype: None """ ne = _engine_loader.get_engine().get_node_execution(self) ne.sync()
def register(self, project, domain, name, version): """ :param Text project: The project in which to register this task. :param Text domain: The domain in which to register this task. :param Text name: The name to give this task. :param Text version: The version in which to register this task. """ self.validate() id_to_register = _identifier.Identifier(_identifier_model.ResourceType.TASK, project, domain, name, version) old_id = self.id try: self._id = id_to_register _engine_loader.get_engine().get_task(self).register(id_to_register) return _six.text_type(self.id) except: self._id = old_id raise
def update(self, state): """ :param int state: Enum value from flytekit.models.launch_plan.LaunchPlanState """ if not self.id: raise _user_exceptions.FlyteAssertion( "Failed to update launch plan because the launch plan's ID is not set. Please call register to fetch " "or register the identifier first") return _engine_loader.get_engine().get_launch_plan(self).update( self.id, state)
def inputs(self): """ Returns the inputs to the execution in the standard Python format as dictated by the type engine. :rtype: dict[Text, T] """ if self._inputs is None: self._inputs = _type_helpers.unpack_literal_map_to_sdk_python_std( _engine_loader.get_engine().get_node_execution( self).get_inputs()) return self._inputs
def register(self, project, domain, name, version): """ :param Text project: :param Text domain: :param Text name: :param Text version: """ self.validate() id_to_register = _identifier.Identifier( _identifier_model.ResourceType.WORKFLOW, project, domain, name, version) old_id = self.id try: self._id = id_to_register _engine_loader.get_engine().get_workflow(self).register( id_to_register) return _six.text_type(self.id) except: self._id = old_id raise
def get_node_executions(self, filters=None): """ :param list[flytekit.models.filters.Filter] filters: :rtype: dict[Text, flytekit.common.nodes.SdkNodeExecution] """ models = _engine_loader.get_engine().get_workflow_execution( self).get_node_executions(filters=filters) return { k: _nodes.SdkNodeExecution.promote_from_model(v) for k, v in _six.iteritems(models) }
def fetch(cls, project, domain, name): """ :param Text project: :param Text domain: :param Text name: :rtype: SdkWorkflowExecution """ return cls.promote_from_model( _engine_loader.get_engine().fetch_workflow_execution( _core_identifier.WorkflowExecutionIdentifier(project=project, domain=domain, name=name)))
def sync(self): """ Syncs the state of this object with that held by the platform. :rtype: None """ if not self.is_complete or self.task_executions is not None: ne = _engine_loader.get_engine().get_node_execution(self) ne.sync() self._task_executions = [ _task_executions.SdkTaskExecution.promote_from_model(te) for te in ne.get_task_executions() ]
def unit_test(self, **input_map): """ :param dict[Text, T] input_map: Python Std input from users. We will cast these to the appropriate Flyte literals. :returns: Depends on the behavior of the specific task in the unit engine. """ return _engine_loader.get_engine('unit').get_task(self).execute( _type_helpers.pack_python_std_map_to_literal_map( input_map, { k: _type_helpers.get_sdk_type_from_literal_type(v.type) for k, v in _six.iteritems(self.interface.inputs) }))
def local_execute(self, **input_map): """ :param dict[Text, T] input_map: Python Std input from users. We will cast these to the appropriate Flyte literals. :rtype: dict[Text, T] :returns: The output produced by this task in Python standard format. """ return _engine_loader.get_engine('local').get_task(self).execute( _type_helpers.pack_python_std_map_to_literal_map( input_map, { k: _type_helpers.get_sdk_type_from_literal_type(v.type) for k, v in _six.iteritems(self.interface.inputs) }))
def fetch(cls, project, domain, name, version): """ This function uses the engine loader to call create a hydrated task from Admin. :param Text project: :param Text domain: :param Text name: :param Text version: :rtype: SdkTask """ task_id = _identifier.Identifier(_identifier_model.ResourceType.TASK, project, domain, name, version) admin_task = _engine_loader.get_engine().fetch_task(task_id=task_id) sdk_task = cls.promote_from_model(admin_task.closure.compiled_task.template) sdk_task._id = task_id return sdk_task
def fetch_latest(cls, project, domain, name): """ This function uses the engine loader to call create a latest hydrated task from Admin. :param Text project: :param Text domain: :param Text name: :rtype: SdkTask """ named_task = _common_model.NamedEntityIdentifier(project, domain, name) admin_task = _engine_loader.get_engine().fetch_latest_task(named_task) if not admin_task: raise _user_exceptions.FlyteEntityNotExistException("Named task {} not found".format(named_task)) sdk_task = cls.promote_from_model(admin_task.closure.compiled_task.template) sdk_task._id = admin_task.id return sdk_task
def get_child_executions(self, filters=None): """ :param list[flytekit.models.filters.Filter] filters: :rtype: dict[Text, flytekit.common.nodes.SdkNodeExecution] """ from flytekit.common import nodes as _nodes if not self.is_parent: raise _user_exceptions.FlyteAssertion( "Only task executions marked with 'is_parent' have child executions." ) models = _engine_loader.get_engine().get_task_execution( self).get_child_executions(filters=filters) return { k: _nodes.SdkNodeExecution.promote_from_model(v) for k, v in _six.iteritems(models) }
def fetch(cls, project, domain, name, version=None): """ This function uses the engine loader to call create a hydrated task from Admin. :param Text project: :param Text domain: :param Text name: :param Text version: :rtype: SdkLaunchPlan """ version = version or _internal_config.VERSION.get() launch_plan_id = _identifier.Identifier( _identifier_model.ResourceType.LAUNCH_PLAN, project, domain, name, version) lp = _engine_loader.get_engine().fetch_launch_plan(launch_plan_id) sdk_lp = cls.promote_from_model(lp.spec) sdk_lp._id = launch_plan_id return sdk_lp
def outputs(self): """ Returns the outputs to the execution in the standard Python format as dictated by the type engine. If the execution ended in error or the execution is in progress, an exception will be raised. :rtype: dict[Text, T] """ if not self.is_complete: raise _user_exceptions.FlyteAssertion( "Please what until the node execution has completed before " "requesting the outputs.") if self.error: raise _user_exceptions.FlyteAssertion( "Outputs could not be found because the execution ended in failure." ) if self._outputs is None: self._outputs = _type_helpers.unpack_literal_map_to_sdk_python_std( _engine_loader.get_engine().get_node_execution( self).get_outputs()) return self._outputs
def outputs(self): """ Returns the outputs of the task execution, if available, in the standard Python format that is produced by the type engine. If not available, perhaps due to execution being in progress or an error being produced, this will raise an exception. :rtype: dict[Text, T] """ if not self.is_complete: raise _user_exceptions.FlyteAssertion( "Please what until the task execution has completed before " "requesting the outputs.") if self.error: raise _user_exceptions.FlyteAssertion( "Outputs could not be found because the execution ended in failure." ) if self._outputs is None: self._outputs = _type_helpers.unpack_literal_map_to_sdk_python_std( _engine_loader.get_engine().get_task_execution( self).get_outputs()) return self._outputs
def execute_with_literals(self, project, domain, literal_inputs, name=None, notification_overrides=None, label_overrides=None, annotation_overrides=None): """ Executes the launch plan and returns the execution identifier. This version of execution is meant for when you already have a LiteralMap of inputs. :param Text project: :param Text domain: :param flytekit.models.literals.LiteralMap literal_inputs: Inputs to the execution. :param Text name: [Optional] If specified, an execution will be created with this name. Note: the name must be unique within the context of the project and domain. :param list[flytekit.common.notifications.Notification] notification_overrides: [Optional] If specified, these are the notifications that will be honored for this execution. An empty list signals to disable all notifications. :param flytekit.models.common.Labels label_overrides: :param flytekit.models.common.Annotations annotation_overrides: :rtype: flytekit.common.workflow_execution.SdkWorkflowExecution """ # Kubernetes requires names starting with an alphabet for some resources. name = name or "f" + _uuid.uuid4().hex[:19] execution = _engine_loader.get_engine().get_launch_plan(self).execute( project, domain, name, literal_inputs, notification_overrides=notification_overrides, label_overrides=label_overrides, annotation_overrides=annotation_overrides, ) return _workflow_execution.SdkWorkflowExecution.promote_from_model( execution)
def fetch(cls, project, domain, name, version=None): """ This function uses the engine loader to call create a hydrated task from Admin. :param Text project: :param Text domain: :param Text name: :param Text version: [Optional] If not set, the SDK will fetch the active launch plan for the given project, domain, and name. :rtype: SdkLaunchPlan """ from flytekit.common import workflow as _workflow launch_plan_id = _identifier.Identifier( _identifier_model.ResourceType.LAUNCH_PLAN, project, domain, name, version) lp = _engine_loader.get_engine().fetch_launch_plan(launch_plan_id) sdk_lp = cls.promote_from_model(lp.spec) sdk_lp._id = lp.id # TODO: Add a test for this, and this function as a whole wf_id = sdk_lp.workflow_id lp_wf = _workflow.SdkWorkflow.fetch(wf_id.project, wf_id.domain, wf_id.name, wf_id.version) sdk_lp._interface = lp_wf.interface return sdk_lp
def test_unit_load(): assert isinstance(loader.get_engine("unit"), _unit_engine.UnitTestEngineFactory)
def test_bad_load(): with pytest.raises(Exception): loader.get_engine("badname")
def terminate(self, cause): """ :param Text cause: """ _engine_loader.get_engine().get_workflow_execution(self).terminate( cause)
def sync(self): _engine_loader.get_engine().get_task_execution(self).sync()