def __call__(self, *args, **input_map):
"""
:param list[T] args: Do not specify. Kwargs only are supported for this function.
:param dict[Text,T] input_map: Map of inputs. Can be statically defined or OutputReference links.
:rtype: flytekit.common.nodes.SdkNode
"""
if len(args) > 0:
raise _user_exceptions.FlyteAssertion(
"When adding a launchplan as a node in a workflow, all inputs must be specified with kwargs only. We "
"detected {} positional args.".format(len(args)))
# Take the default values from the launch plan
default_inputs = {
k: v.sdk_default
for k, v in _six.iteritems(self.default_inputs.parameters)
if not v.required
}
default_inputs.update(input_map)
bindings, upstream_nodes = self.interface.create_bindings_for_inputs(
default_inputs)
return _nodes.SdkNode(
id=None,
metadata=_workflow_models.NodeMetadata(
"", _datetime.timedelta(), _literal_models.RetryStrategy(0)),
bindings=sorted(bindings, key=lambda b: b.var),
upstream_nodes=upstream_nodes,
sdk_launch_plan=self,
)
def test_non_system_nodes():
@inputs(a=primitives.Integer)
@outputs(b=primitives.Integer)
@python_task()
def my_task(wf_params, a, b):
b.set(a + 1)
my_task._id = _identifier.Identifier(_identifier.ResourceType.TASK,
"project", "domain", "my_task",
"version")
required_input = promise.Input("required", primitives.Integer)
n1 = my_task(a=required_input).assign_id_and_return("n1")
n_start = nodes.SdkNode(
"start-node",
[],
[
_literals.Binding(
"a",
interface.BindingData.from_python_std(
_types.Types.Integer.to_flyte_literal_type(), 3),
)
],
None,
sdk_task=my_task,
sdk_workflow=None,
sdk_launch_plan=None,
sdk_branch=None,
)
non_system_nodes = workflow.SdkWorkflow.get_non_system_nodes([n1, n_start])
assert len(non_system_nodes) == 1
assert non_system_nodes[0].id == "n1"
def promote_from_model(cls, base_model):
"""
:param flytekit.models.core.workflow.WorkflowTemplate base_model:
:rtype: SdkWorkflow
"""
node_map = {
n.id: _nodes.SdkNode(
n.id,
[],
n.inputs,
n.metadata,
sdk_task=None,
sdk_workflow=None,
sdk_branch=
None # TODO: Hydrate these objects by reference from the engine.
)
for n in base_model.nodes
}
for v in _six.itervalues(node_map):
v.upstream_nodes[:] = [node_map[k] for k in v.upstream_node_ids]
return cls(
metadata=base_model.metadata,
interface=_interface.TypedInterface.promote_from_model(
base_model.interface),
nodes=list(node_map.values()),
outputs=base_model.outputs,
failure_node=None # TODO: Implement failure node
)
def __call__(self, *args, **input_map):
if len(args) > 0:
raise _user_exceptions.FlyteAssertion(
"When adding a workflow as a node in a workflow, all inputs must be specified with kwargs only. We "
"detected {} positional args.".format(len(args))
)
# Take the default values from the Inputs
compiled_inputs = {
v.name: v.sdk_default
for v in self.user_inputs if not v.sdk_required
}
compiled_inputs.update(input_map)
bindings, upstream_nodes = self.interface.create_bindings_for_inputs(compiled_inputs)
node = _nodes.SdkNode(
id=None,
metadata=_workflow_models.NodeMetadata("placeholder", _datetime.timedelta(),
_literal_models.RetryStrategy(0)),
upstream_nodes=upstream_nodes,
bindings=sorted(bindings, key=lambda b: b.var),
sdk_workflow=self
)
return node
def __call__(self, *args, **input_map):
"""
:param list[T] args: Do not specify. Kwargs only are supported for this function.
:param dict[str, T] input_map: Map of inputs. Can be statically defined or OutputReference links.
:rtype: flytekit.common.nodes.SdkNode
"""
if len(args) > 0:
raise _user_exceptions.FlyteAssertion(
"When adding a task as a node in a workflow, all inputs must be specified with kwargs only. We "
"detected {} positional args.".format(len(args)))
bindings, upstream_nodes = self.interface.create_bindings_for_inputs(
input_map)
# TODO: Remove DEADBEEF
# One thing to note - this function is not overloaded at the SdkRunnableTask layer, which means 'self' here
# will sometimes refer to an object that can be executed locally, and other times will refer to something
# that cannot (ie a pure SdkTask object, fetched from Admin for instance).
return _nodes.SdkNode(
id=None,
metadata=_workflow_model.NodeMetadata(
"DEADBEEF",
self.metadata.timeout,
self.metadata.retries,
self.metadata.interruptible,
),
bindings=sorted(bindings, key=lambda b: b.var),
upstream_nodes=upstream_nodes,
sdk_task=self,
)
def test_blah():
@inputs(a=primitives.Integer)
@outputs(b=primitives.Integer)
@python_task()
def my_task(wf_params, a, b):
b.set(a + 1)
my_task._id = _identifier.Identifier(_identifier.ResourceType.TASK, 'project', 'domain', 'my_task', 'version')
required_input = promise.Input('required', primitives.Integer)
n1 = my_task(a=required_input).assign_id_and_return('n1')
n_start = nodes.SdkNode(
'start-node',
[],
[
_literals.Binding(
'a',
interface.BindingData.from_python_std(_types.Types.Integer.to_flyte_literal_type(), 3)
)
],
None,
sdk_task=my_task,
sdk_workflow=None,
sdk_launch_plan=None,
sdk_branch=None
)
non_system_nodes = workflow.SdkWorkflow.get_non_system_nodes([n1, n_start])
assert len(non_system_nodes) == 1
assert non_system_nodes[0].id == 'n1'
def _create_hive_job_node(name, hive_job, metadata):
"""
:param Text name:
:param _qubole.QuboleHiveJob hive_job: Hive job spec
:param flytekit.models.task.TaskMetadata metadata: This contains information needed at runtime to determine
behavior such as whether or not outputs are discoverable, timeouts, and retries.
:rtype: _nodes.SdkNode:
"""
return _nodes.SdkNode(id=_six.text_type(_uuid.uuid4()),
upstream_nodes=[],
bindings=[],
metadata=_workflow_model.NodeMetadata(
name, metadata.timeout,
_literal_models.RetryStrategy(0)),
sdk_task=SdkHiveJob(hive_job, metadata))
def test_sdk_node_from_lp():
@_tasks.inputs(a=_types.Types.Integer)
@_tasks.outputs(b=_types.Types.Integer)
@_tasks.python_task()
def testy_test(wf_params, a, b):
pass
@_workflow.workflow_class
class test_workflow(object):
a = _workflow.Input(_types.Types.Integer)
test = testy_test(a=a)
b = _workflow.Output(test.outputs.b, sdk_type=_types.Types.Integer)
lp = test_workflow.create_launch_plan()
n1 = _nodes.SdkNode(
"n1",
[],
[
_literals.Binding(
"a",
_interface.BindingData.from_python_std(
_types.Types.Integer.to_flyte_literal_type(), 3),
)
],
_core_workflow_models.NodeMetadata("abc",
_datetime.timedelta(minutes=15),
_literals.RetryStrategy(3)),
sdk_launch_plan=lp,
)
assert n1.id == "n1"
assert len(n1.inputs) == 1
assert n1.inputs[0].var == "a"
assert n1.inputs[0].binding.scalar.primitive.integer == 3
assert len(n1.outputs) == 1
assert "b" in n1.outputs
assert n1.outputs["b"].node_id == "n1"
assert n1.outputs["b"].var == "b"
assert n1.outputs["b"].sdk_node == n1
assert n1.outputs["b"].sdk_type == _types.Types.Integer
assert n1.metadata.name == "abc"
assert n1.metadata.retries.retries == 3
assert len(n1.upstream_nodes) == 0
assert len(n1.upstream_node_ids) == 0
assert len(n1.output_aliases) == 0
def __call__(self, *args, **input_map):
if len(args) > 0:
raise _user_exceptions.FlyteAssertion(
"When adding a workflow as a node in a workflow, all inputs must be specified with kwargs only. We "
"detected {} positional args.".format(len(args)))
bindings, upstream_nodes = self.interface.create_bindings_for_inputs(
input_map)
node = _nodes.SdkNode(
id=None,
metadata=_workflow_models.NodeMetadata(
"placeholder", _datetime.timedelta(),
_literal_models.RetryStrategy(0)),
upstream_nodes=upstream_nodes,
bindings=sorted(bindings, key=lambda b: b.var),
sdk_workflow=self,
)
return node
def __call__(self, *args, **input_map):
"""
:param list[T] args: Do not specify. Kwargs only are supported for this function.
:param dict[str, T] input_map: Map of inputs. Can be statically defined or OutputReference links.
:rtype: flytekit.common.nodes.SdkNode
"""
if len(args) > 0:
raise _user_exceptions.FlyteAssertion(
"When adding a task as a node in a workflow, all inputs must be specified with kwargs only. We "
"detected {} positional args.".format(len(args)))
bindings, upstream_nodes = self.interface.create_bindings_for_inputs(
input_map)
# TODO: Remove DEADBEEF
return _nodes.SdkNode(id=None,
metadata=_workflow_model.NodeMetadata(
"DEADBEEF", self.metadata.timeout,
self.metadata.retries),
bindings=sorted(bindings, key=lambda b: b.var),
upstream_nodes=upstream_nodes,
sdk_task=self)
def test_sdk_node_from_task():
@_tasks.inputs(a=_types.Types.Integer)
@_tasks.outputs(b=_types.Types.Integer)
@_tasks.python_task()
def testy_test(wf_params, a, b):
pass
n = _nodes.SdkNode(
"n",
[],
[
_literals.Binding(
"a",
_interface.BindingData.from_python_std(
_types.Types.Integer.to_flyte_literal_type(), 3),
)
],
_core_workflow_models.NodeMetadata("abc",
_datetime.timedelta(minutes=15),
_literals.RetryStrategy(3)),
sdk_task=testy_test,
sdk_workflow=None,
sdk_launch_plan=None,
sdk_branch=None,
)
assert n.id == "n"
assert len(n.inputs) == 1
assert n.inputs[0].var == "a"
assert n.inputs[0].binding.scalar.primitive.integer == 3
assert len(n.outputs) == 1
assert "b" in n.outputs
assert n.outputs["b"].node_id == "n"
assert n.outputs["b"].var == "b"
assert n.outputs["b"].sdk_node == n
assert n.outputs["b"].sdk_type == _types.Types.Integer
assert n.metadata.name == "abc"
assert n.metadata.retries.retries == 3
assert n.metadata.interruptible is None
assert len(n.upstream_nodes) == 0
assert len(n.upstream_node_ids) == 0
assert len(n.output_aliases) == 0
n2 = _nodes.SdkNode(
"n2",
[n],
[
_literals.Binding(
"a",
_interface.BindingData.from_python_std(
_types.Types.Integer.to_flyte_literal_type(), n.outputs.b),
)
],
_core_workflow_models.NodeMetadata("abc2",
_datetime.timedelta(minutes=15),
_literals.RetryStrategy(3)),
sdk_task=testy_test,
sdk_workflow=None,
sdk_launch_plan=None,
sdk_branch=None,
)
assert n2.id == "n2"
assert len(n2.inputs) == 1
assert n2.inputs[0].var == "a"
assert n2.inputs[0].binding.promise.var == "b"
assert n2.inputs[0].binding.promise.node_id == "n"
assert len(n2.outputs) == 1
assert "b" in n2.outputs
assert n2.outputs["b"].node_id == "n2"
assert n2.outputs["b"].var == "b"
assert n2.outputs["b"].sdk_node == n2
assert n2.outputs["b"].sdk_type == _types.Types.Integer
assert n2.metadata.name == "abc2"
assert n2.metadata.retries.retries == 3
assert "n" in n2.upstream_node_ids
assert n in n2.upstream_nodes
assert len(n2.upstream_nodes) == 1
assert len(n2.upstream_node_ids) == 1
assert len(n2.output_aliases) == 0
# Test right shift operator and late binding
n3 = _nodes.SdkNode(
"n3",
[],
[
_literals.Binding(
"a",
_interface.BindingData.from_python_std(
_types.Types.Integer.to_flyte_literal_type(), 3),
)
],
_core_workflow_models.NodeMetadata("abc3",
_datetime.timedelta(minutes=15),
_literals.RetryStrategy(3)),
sdk_task=testy_test,
sdk_workflow=None,
sdk_launch_plan=None,
sdk_branch=None,
)
n2 >> n3
n >> n2 >> n3
n3 << n2
n3 << n2 << n
assert n3.id == "n3"
assert len(n3.inputs) == 1
assert n3.inputs[0].var == "a"
assert n3.inputs[0].binding.scalar.primitive.integer == 3
assert len(n3.outputs) == 1
assert "b" in n3.outputs
assert n3.outputs["b"].node_id == "n3"
assert n3.outputs["b"].var == "b"
assert n3.outputs["b"].sdk_node == n3
assert n3.outputs["b"].sdk_type == _types.Types.Integer
assert n3.metadata.name == "abc3"
assert n3.metadata.retries.retries == 3
assert "n2" in n3.upstream_node_ids
assert n2 in n3.upstream_nodes
assert len(n3.upstream_nodes) == 1
assert len(n3.upstream_node_ids) == 1
assert len(n3.output_aliases) == 0
# Test left shift operator and late binding
n4 = _nodes.SdkNode(
"n4",
[],
[
_literals.Binding(
"a",
_interface.BindingData.from_python_std(
_types.Types.Integer.to_flyte_literal_type(), 3),
)
],
_core_workflow_models.NodeMetadata("abc4",
_datetime.timedelta(minutes=15),
_literals.RetryStrategy(3)),
sdk_task=testy_test,
sdk_workflow=None,
sdk_launch_plan=None,
sdk_branch=None,
)
n4 << n3
# Test that implicit dependencies don't cause direct dependencies
n4 << n3 << n2 << n
n >> n2 >> n3 >> n4
assert n4.id == "n4"
assert len(n4.inputs) == 1
assert n4.inputs[0].var == "a"
assert n4.inputs[0].binding.scalar.primitive.integer == 3
assert len(n4.outputs) == 1
assert "b" in n4.outputs
assert n4.outputs["b"].node_id == "n4"
assert n4.outputs["b"].var == "b"
assert n4.outputs["b"].sdk_node == n4
assert n4.outputs["b"].sdk_type == _types.Types.Integer
assert n4.metadata.name == "abc4"
assert n4.metadata.retries.retries == 3
assert "n3" in n4.upstream_node_ids
assert n3 in n4.upstream_nodes
assert len(n4.upstream_nodes) == 1
assert len(n4.upstream_node_ids) == 1
assert len(n4.output_aliases) == 0
# Add another dependency
n4 << n2
assert "n3" in n4.upstream_node_ids
assert n3 in n4.upstream_nodes
assert "n2" in n4.upstream_node_ids
assert n2 in n4.upstream_nodes
assert len(n4.upstream_nodes) == 2
assert len(n4.upstream_node_ids) == 2
