def test_workflow_template():
task = _workflow.TaskNode(reference_id=_generic_id)
nm = _get_sample_node_metadata()
int_type = _types.LiteralType(_types.SimpleType.INTEGER)
wf_metadata = _workflow.WorkflowMetadata()
wf_metadata_defaults = _workflow.WorkflowMetadataDefaults()
typed_interface = _interface.TypedInterface(
{"a": _interface.Variable(int_type, "description1")},
{
"b": _interface.Variable(int_type, "description2"),
"c": _interface.Variable(int_type, "description3")
},
)
wf_node = _workflow.Node(
id="some:node:id",
metadata=nm,
inputs=[],
upstream_node_ids=[],
output_aliases=[],
task_node=task,
)
obj = _workflow.WorkflowTemplate(
id=_generic_id,
metadata=wf_metadata,
metadata_defaults=wf_metadata_defaults,
interface=typed_interface,
nodes=[wf_node],
outputs=[],
)
obj2 = _workflow.WorkflowTemplate.from_flyte_idl(obj.to_flyte_idl())
assert obj2 == obj
def __init__(self,
inputs,
outputs,
nodes,
id=None,
metadata=None,
metadata_defaults=None,
interface=None,
output_bindings=None):
"""
:param list[flytekit.common.promise.Input] inputs:
:param list[Output] outputs:
:param list[flytekit.common.nodes.SdkNode] nodes:
:param flytekit.models.core.identifier.Identifier id: This is an autogenerated id by the system. The id is
globally unique across Flyte.
:param WorkflowMetadata metadata: This contains information on how to run the workflow.
:param flytekit.models.interface.TypedInterface interface: Defines a strongly typed interface for the
Workflow (inputs, outputs). This can include some optional parameters.
:param list[flytekit.models.literals.Binding] output_bindings: A list of output bindings that specify how to construct
workflow outputs. Bindings can pull node outputs or specify literals. All workflow outputs specified in
the interface field must be bound
in order for the workflow to be validated. A workflow has an implicit dependency on all of its nodes
to execute successfully in order to bind final outputs.
"""
for n in nodes:
for upstream in n.upstream_nodes:
if upstream.id is None:
raise _user_exceptions.FlyteAssertion(
"Some nodes contained in the workflow were not found in the workflow description. Please "
"ensure all nodes are either assigned to attributes within the class or an element in a "
"list, dict, or tuple which is stored as an attribute in the class."
)
# Allow overrides if specified for all the arguments to the parent class constructor
id = id if id is not None else _identifier.Identifier(
_identifier_model.ResourceType.WORKFLOW,
_internal_config.PROJECT.get(), _internal_config.DOMAIN.get(),
_uuid.uuid4().hex, _internal_config.VERSION.get())
metadata = metadata if metadata is not None else _workflow_models.WorkflowMetadata(
)
interface = interface if interface is not None else _interface.TypedInterface(
{v.name: v.var
for v in inputs}, {v.name: v.var
for v in outputs})
output_bindings = output_bindings if output_bindings is not None else \
[_literal_models.Binding(v.name, v.binding_data) for v in outputs]
super(SdkWorkflow, self).__init__(
id=id,
metadata=metadata,
metadata_defaults=_workflow_models.WorkflowMetadataDefaults(),
interface=interface,
nodes=nodes,
outputs=output_bindings,
)
self._user_inputs = inputs
self._upstream_entities = set(n.executable_sdk_object for n in nodes)
def build_sdk_workflow_from_metaclass(metaclass,
on_failure=None,
disable_default_launch_plan=False,
cls=None):
"""
:param T metaclass: This is the user-defined workflow class, prior to decoration.
:param on_failure flytekit.models.core.workflow.WorkflowMetadata.OnFailurePolicy: [Optional] The execution policy
when the workflow detects a failure.
:param bool disable_default_launch_plan: Determines whether to create a default launch plan for the workflow or not.
:param cls: This is the class that will be instantiated from the inputs, outputs, and nodes. This will be used
by users extending the base Flyte programming model. If set, it must be a subclass of PythonWorkflow.
:rtype: SdkRunnableWorkflow
"""
inputs, outputs, nodes = _discover_workflow_components(metaclass)
metadata = _workflow_models.WorkflowMetadata(
on_failure=on_failure if on_failure else None)
return (cls or SdkRunnableWorkflow).construct_from_class_definition(
inputs=[i for i in sorted(inputs, key=lambda x: x.name)],
outputs=[o for o in sorted(outputs, key=lambda x: x.name)],
nodes=[n for n in sorted(nodes, key=lambda x: x.id)],
metadata=metadata,
disable_default_launch_plan=disable_default_launch_plan,
)
def __init__(self, inputs, outputs, nodes):
"""
:param list[flytekit.common.promise.Input] inputs:
:param list[Output] outputs:
:param list[flytekit.common.nodes.SdkNode] nodes:
"""
for n in nodes:
for upstream in n.upstream_nodes:
if upstream.id is None:
raise _user_exceptions.FlyteAssertion(
"Some nodes contained in the workflow were not found in the workflow description. Please "
"ensure all nodes are either assigned to attributes within the class or an element in a "
"list, dict, or tuple which is stored as an attribute in the class."
)
super(SdkWorkflow, self).__init__(
id=_identifier.Identifier(_identifier_model.ResourceType.WORKFLOW,
_internal_config.PROJECT.get(),
_internal_config.DOMAIN.get(),
_uuid.uuid4().hex,
_internal_config.VERSION.get()),
metadata=_workflow_models.WorkflowMetadata(),
interface=_interface.TypedInterface(
{v.name: v.var
for v in inputs}, {v.name: v.var
for v in outputs}),
nodes=nodes,
outputs=[
_literal_models.Binding(v.name, v.binding_data)
for v in outputs
],
)
self._user_inputs = inputs
self._upstream_entities = set(n.executable_sdk_object for n in nodes)
def test_workflow_template_with_queuing_budget():
task = _workflow.TaskNode(reference_id=_generic_id)
nm = _get_sample_node_metadata()
int_type = _types.LiteralType(_types.SimpleType.INTEGER)
wf_metadata = _workflow.WorkflowMetadata(queuing_budget=timedelta(
seconds=10))
wf_metadata_defaults = _workflow.WorkflowMetadataDefaults()
typed_interface = _interface.TypedInterface(
{'a': _interface.Variable(int_type, "description1")}, {
'b': _interface.Variable(int_type, "description2"),
'c': _interface.Variable(int_type, "description3")
})
wf_node = _workflow.Node(id='some:node:id',
metadata=nm,
inputs=[],
upstream_node_ids=[],
output_aliases=[],
task_node=task)
obj = _workflow.WorkflowTemplate(id=_generic_id,
metadata=wf_metadata,
metadata_defaults=wf_metadata_defaults,
interface=typed_interface,
nodes=[wf_node],
outputs=[])
obj2 = _workflow.WorkflowTemplate.from_flyte_idl(obj.to_flyte_idl())
assert obj2 == obj
def __init__(
self,
inputs: List[_promise.Input],
nodes: List[_nodes.SdkNode],
interface,
output_bindings,
id=None,
metadata=None,
metadata_defaults=None,
disable_default_launch_plan=False,
):
"""
:param list[flytekit.common.nodes.SdkNode] nodes:
:param flytekit.models.interface.TypedInterface interface: Defines a strongly typed interface for the
Workflow (inputs, outputs). This can include some optional parameters.
:param list[flytekit.models.literals.Binding] output_bindings: A list of output bindings that specify how to construct
workflow outputs. Bindings can pull node outputs or specify literals. All workflow outputs specified in
the interface field must be bound
in order for the workflow to be validated. A workflow has an implicit dependency on all of its nodes
to execute successfully in order to bind final outputs.
:param flytekit.models.core.identifier.Identifier id: This is an autogenerated id by the system. The id is
globally unique across Flyte.
:param WorkflowMetadata metadata: This contains information on how to run the workflow.
:param flytekit.models.core.workflow.WorkflowMetadataDefaults metadata_defaults: Defaults to be passed
to nodes contained within workflow.
:param bool disable_default_launch_plan: Determines whether to create a default launch plan for the workflow.
"""
# Save the promise.Input objects for future use.
self._user_inputs = inputs
# Set optional settings
id = (
id
if id is not None
else _identifier.Identifier(
_identifier_model.ResourceType.WORKFLOW,
_internal_config.PROJECT.get(),
_internal_config.DOMAIN.get(),
_uuid.uuid4().hex,
_internal_config.VERSION.get(),
)
)
metadata = metadata if metadata is not None else _workflow_models.WorkflowMetadata()
metadata_defaults = (
metadata_defaults if metadata_defaults is not None else _workflow_models.WorkflowMetadataDefaults()
)
super(SdkRunnableWorkflow, self).__init__(
nodes=nodes,
interface=interface,
output_bindings=output_bindings,
id=id,
metadata=metadata,
metadata_defaults=metadata_defaults,
)
# Set this last as it's set in constructor
self._upstream_entities = set(n.executable_sdk_object for n in nodes)
self._should_create_default_launch_plan = not disable_default_launch_plan
def test_workflow_metadata_failure_policy():
obj = _workflow.WorkflowMetadata(
on_failure=_workflow.WorkflowMetadata.OnFailurePolicy.
FAIL_AFTER_EXECUTABLE_NODES_COMPLETE)
obj2 = _workflow.WorkflowMetadata.from_flyte_idl(obj.to_flyte_idl())
assert obj == obj2
assert obj.on_failure == _workflow.WorkflowMetadata.OnFailurePolicy.FAIL_AFTER_EXECUTABLE_NODES_COMPLETE
assert obj2.on_failure == _workflow.WorkflowMetadata.OnFailurePolicy.FAIL_AFTER_EXECUTABLE_NODES_COMPLETE
def build_sdk_workflow_from_metaclass(metaclass, queuing_budget=None, cls=None):
"""
:param T metaclass:
:param cls: This is the class that will be instantiated from the inputs, outputs, and nodes. This will be used
by users extending the base Flyte programming model. If set, it must be a subclass of SdkWorkflow.
:param queuing_budget datetime.timedelta: [Optional] Budget that specifies the amount of time a workflow can be queued up for execution.
:rtype: SdkWorkflow
"""
inputs, outputs, nodes = _discover_workflow_components(metaclass)
metadata = _workflow_models.WorkflowMetadata(queuing_budget) if queuing_budget else None
return (cls or SdkWorkflow)(
inputs=[i for i in sorted(inputs, key=lambda x: x.name)],
outputs=[o for o in sorted(outputs, key=lambda x: x.name)],
nodes=[n for n in sorted(nodes, key=lambda x: x.id)],
metadata=metadata
)
def build_sdk_workflow_from_metaclass(metaclass, on_failure=None, cls=None):
"""
:param T metaclass:
:param cls: This is the class that will be instantiated from the inputs, outputs, and nodes. This will be used
by users extending the base Flyte programming model. If set, it must be a subclass of SdkWorkflow.
:param on_failure flytekit.models.core.workflow.WorkflowMetadata.OnFailurePolicy: [Optional] The execution policy when the workflow detects a failure.
:rtype: SdkWorkflow
"""
inputs, outputs, nodes = _discover_workflow_components(metaclass)
metadata = _workflow_models.WorkflowMetadata(on_failure=on_failure if on_failure else None)
return (cls or SdkWorkflow)(
inputs=[i for i in sorted(inputs, key=lambda x: x.name)],
outputs=[o for o in sorted(outputs, key=lambda x: x.name)],
nodes=[n for n in sorted(nodes, key=lambda x: x.id)],
metadata=metadata
)
def test_workflow_closure():
int_type = _types.LiteralType(_types.SimpleType.INTEGER)
typed_interface = _interface.TypedInterface(
{'a': _interface.Variable(int_type, "description1")}, {
'b': _interface.Variable(int_type, "description2"),
'c': _interface.Variable(int_type, "description3")
})
b0 = _literals.Binding(
'a',
_literals.BindingData(scalar=_literals.Scalar(
primitive=_literals.Primitive(integer=5))))
b1 = _literals.Binding(
'b',
_literals.BindingData(promise=_types.OutputReference('my_node', 'b')))
b2 = _literals.Binding(
'b',
_literals.BindingData(promise=_types.OutputReference('my_node', 'c')))
node_metadata = _workflow.NodeMetadata(name='node1',
timeout=timedelta(seconds=10),
retries=_literals.RetryStrategy(0))
task_metadata = _task.TaskMetadata(
True,
_task.RuntimeMetadata(_task.RuntimeMetadata.RuntimeType.FLYTE_SDK,
"1.0.0", "python"), timedelta(days=1),
_literals.RetryStrategy(3), "0.1.1b0", "This is deprecated!")
cpu_resource = _task.Resources.ResourceEntry(
_task.Resources.ResourceName.CPU, "1")
resources = _task.Resources(requests=[cpu_resource], limits=[cpu_resource])
task = _task.TaskTemplate(
_identifier.Identifier(_identifier.ResourceType.TASK, "project",
"domain", "name", "version"),
"python",
task_metadata,
typed_interface, {
'a': 1,
'b': {
'c': 2,
'd': 3
}
},
container=_task.Container("my_image", ["this", "is", "a", "cmd"],
["this", "is", "an", "arg"], resources, {},
{}))
task_node = _workflow.TaskNode(task.id)
node = _workflow.Node(id='my_node',
metadata=node_metadata,
inputs=[b0],
upstream_node_ids=[],
output_aliases=[],
task_node=task_node)
template = _workflow.WorkflowTemplate(
id=_identifier.Identifier(_identifier.ResourceType.WORKFLOW, "project",
"domain", "name", "version"),
metadata=_workflow.WorkflowMetadata(),
interface=typed_interface,
nodes=[node],
outputs=[b1, b2],
)
obj = _workflow_closure.WorkflowClosure(workflow=template, tasks=[task])
assert len(obj.tasks) == 1
obj2 = _workflow_closure.WorkflowClosure.from_flyte_idl(obj.to_flyte_idl())
assert obj == obj2
def test_workflow_metadata():
obj = _workflow.WorkflowMetadata()
obj2 = _workflow.WorkflowMetadata.from_flyte_idl(obj.to_flyte_idl())
assert obj == obj2
def to_flyte_model(self):
if self.on_failure == WorkflowFailurePolicy.FAIL_IMMEDIATELY:
on_failure = 0
else:
on_failure = 1
return _workflow_model.WorkflowMetadata(on_failure=on_failure)
def test_workflow_metadata_queuing_budget():
obj = _workflow.WorkflowMetadata(queuing_budget=timedelta(seconds=10))
obj2 = _workflow.WorkflowMetadata.from_flyte_idl(obj.to_flyte_idl())
assert obj == obj2
