def nested_dynamic_wf_task(wf_params, task_input_num, out):
wf_params.logging.info(
"Running inner task... yielding a code generated sub workflow")
# Inner workflow
input_a = Input(Types.Integer, help="Tell me something")
node1 = sq_sub_task(in1=input_a)
MyUnregisteredWorkflowInner = workflow(
inputs={
'a': input_a,
},
outputs={
'ooo':
Output(node1.outputs.out1,
sdk_type=Types.Integer,
help='This is an integer output')
},
nodes={
'node_one': node1,
})
setattr(MyUnregisteredWorkflowInner, 'auto_assign_name',
manual_assign_name)
MyUnregisteredWorkflowInner._platform_valid_name = 'unregistered'
# Output workflow
input_a = Input(Types.Integer, help="Tell me something")
node1 = MyUnregisteredWorkflowInner(a=task_input_num)
MyUnregisteredWorkflowOuter = workflow(
inputs={
'a': input_a,
},
outputs={
'ooo':
Output(node1.outputs.ooo,
sdk_type=Types.Integer,
help='This is an integer output')
},
nodes={
'node_one': node1,
})
setattr(MyUnregisteredWorkflowOuter, 'auto_assign_name',
manual_assign_name)
MyUnregisteredWorkflowOuter._platform_valid_name = 'unregistered'
unregistered_workflow_execution = MyUnregisteredWorkflowOuter(
a=task_input_num)
out.set(unregistered_workflow_execution.outputs.ooo)
def test_serialize():
workflow_to_test = _workflow.workflow(
{},
inputs={
"required_input": _workflow.Input(_types.Types.Integer),
"default_input": _workflow.Input(_types.Types.Integer, default=5),
},
)
workflow_to_test.id = _identifier.Identifier(
_identifier.ResourceType.WORKFLOW, "p", "d", "n", "v")
lp = workflow_to_test.create_launch_plan(
fixed_inputs={"required_input": 5},
role="iam_role",
)
with _configuration.TemporaryConfiguration(
_os.path.join(
_os.path.dirname(_os.path.realpath(__file__)),
"../../common/configs/local.config",
),
internal_overrides={
"image": "myflyteimage:v123",
"project": "myflyteproject",
"domain": "development"
},
):
s = lp.serialize()
assert s.workflow_id == _identifier.Identifier(
_identifier.ResourceType.WORKFLOW, "p", "d", "n", "v").to_flyte_idl()
assert s.auth_role.assumable_iam_role == "iam_role"
assert s.default_inputs.parameters[
"default_input"].default.scalar.primitive.integer == 5
def test_serialize():
workflow_to_test = _workflow.workflow(
{},
inputs={
'required_input': _workflow.Input(_types.Types.Integer),
'default_input': _workflow.Input(_types.Types.Integer, default=5)
})
workflow_to_test._id = _identifier.Identifier(
_identifier.ResourceType.WORKFLOW, "p", "d", "n", "v")
lp = workflow_to_test.create_launch_plan(
fixed_inputs={'required_input': 5},
role='iam_role',
)
with _configuration.TemporaryConfiguration(_os.path.join(
_os.path.dirname(_os.path.realpath(__file__)),
'../../common/configs/local.config'),
internal_overrides={
'image':
'myflyteimage:v123',
'project': 'myflyteproject',
'domain': 'development'
}):
s = lp.serialize()
assert s.workflow_id == _identifier.Identifier(
_identifier.ResourceType.WORKFLOW, "p", "d", "n", "v").to_flyte_idl()
assert s.auth.assumable_iam_role == 'iam_role'
assert s.default_inputs.parameters[
'default_input'].default.scalar.primitive.integer == 5
def test_workflow_no_node_dependencies_or_outputs():
@inputs(a=Types.Integer)
@outputs(b=Types.Integer)
@python_task
def my_task(wf_params, a, b):
b.set(a + 1)
i1 = Input(Types.Integer)
i2 = Input(Types.Integer, default=5, help="Not required.")
input_dict = {"input_1": i1, "input_2": i2}
nodes = {
"a": my_task(a=input_dict["input_1"]),
"b": my_task(a=input_dict["input_2"]),
"c": my_task(a=100),
}
w = workflow(inputs=input_dict, outputs={}, nodes=nodes)
assert w.interface.inputs[
"input_1"].type == Types.Integer.to_flyte_literal_type()
assert w.interface.inputs[
"input_2"].type == Types.Integer.to_flyte_literal_type()
assert _get_node_by_id(w, "a").inputs[0].var == "a"
assert _get_node_by_id(
w, "a"
).inputs[0].binding.promise.node_id == constants.GLOBAL_INPUT_NODE_ID
assert _get_node_by_id(w, "a").inputs[0].binding.promise.var == "input_1"
assert _get_node_by_id(
w, "b"
).inputs[0].binding.promise.node_id == constants.GLOBAL_INPUT_NODE_ID
assert _get_node_by_id(w, "b").inputs[0].binding.promise.var == "input_2"
assert _get_node_by_id(
w, "c").inputs[0].binding.scalar.primitive.integer == 100
def workflow_builder(wf_params, task_input_num, decider, out):
wf_params.logging.info(
"Running inner task... yielding a code generated sub workflow")
input_a = Input(Types.Integer, help="Tell me something")
if decider:
node1 = inverse_inner_task(num=input_a)
else:
node1 = inner_task(num=input_a)
MyUnregisteredWorkflow = workflow(inputs={
'a': input_a,
},
outputs={
'ooo':
Output(
node1.outputs.out,
sdk_type=Types.Integer,
help='This is an integer output')
},
nodes={
'node_one': node1,
})
# This is an unfortunate setting that will hopefully not be necessary in the future.
setattr(MyUnregisteredWorkflow, 'auto_assign_name', manual_assign_name)
MyUnregisteredWorkflow._platform_valid_name = 'unregistered'
unregistered_workflow_execution = MyUnregisteredWorkflow(a=task_input_num)
yield unregistered_workflow_execution
out.set(unregistered_workflow_execution.outputs.ooo)
def test_workflow_no_node_dependencies_or_outputs():
@inputs(a=Types.Integer)
@outputs(b=Types.Integer)
@python_task
def my_task(wf_params, a, b):
b.set(a + 1)
i1 = Input(Types.Integer)
i2 = Input(Types.Integer, default=5, help='Not required.')
input_dict = {
'input_1': i1,
'input_2': i2
}
nodes = {
'a': my_task(a=input_dict['input_1']),
'b': my_task(a=input_dict['input_2']),
'c': my_task(a=100)
}
w = workflow(inputs=input_dict, outputs={}, nodes=nodes)
assert w.interface.inputs['input_1'].type == Types.Integer.to_flyte_literal_type()
assert w.interface.inputs['input_2'].type == Types.Integer.to_flyte_literal_type()
assert _get_node_by_id(w, 'a').inputs[0].var == 'a'
assert _get_node_by_id(w, 'a').inputs[0].binding.promise.node_id == constants.GLOBAL_INPUT_NODE_ID
assert _get_node_by_id(w, 'a').inputs[0].binding.promise.var == 'input_1'
assert _get_node_by_id(w, 'b').inputs[0].binding.promise.node_id == constants.GLOBAL_INPUT_NODE_ID
assert _get_node_by_id(w, 'b').inputs[0].binding.promise.var == 'input_2'
assert _get_node_by_id(w, 'c').inputs[0].binding.scalar.primitive.integer == 100
def test_hard_coded_deprecated_role():
workflow_to_test = _workflow.workflow(
{},
inputs={
'required_input': _workflow.Input(_types.Types.Integer),
'default_input': _workflow.Input(_types.Types.Integer, default=5)
})
lp = workflow_to_test.create_launch_plan(role='override')
assert lp.auth.assumable_iam_role == 'override'
def test_no_notifications():
workflow_to_test = _workflow.workflow(
{},
inputs={
'required_input': _workflow.Input(_types.Types.Integer),
'default_input': _workflow.Input(_types.Types.Integer, default=5)
})
lp = workflow_to_test.create_launch_plan()
assert len(lp.entity_metadata.notifications) == 0
def test_kubernetes_service_account():
workflow_to_test = _workflow.workflow(
{},
inputs={
'required_input': _workflow.Input(_types.Types.Integer),
'default_input': _workflow.Input(_types.Types.Integer, default=5)
})
lp = workflow_to_test.create_launch_plan(
kubernetes_service_account='kube-service-acct')
assert lp.auth.kubernetes_service_account == 'kube-service-acct'
def test_hard_coded_deprecated_role():
workflow_to_test = _workflow.workflow(
{},
inputs={
"required_input": _workflow.Input(_types.Types.Integer),
"default_input": _workflow.Input(_types.Types.Integer, default=5),
},
)
lp = workflow_to_test.create_launch_plan(role="override")
assert lp.auth_role.assumable_iam_role == "override"
def test_launch_plan_node():
workflow_to_test = _workflow.workflow(
{},
inputs={
"required_input": _workflow.Input(_types.Types.Integer),
"default_input": _workflow.Input(_types.Types.Integer, default=5),
},
outputs={
"out": _workflow.Output([1, 2, 3], sdk_type=[_types.Types.Integer])
},
)
lp = workflow_to_test.create_launch_plan()
# Test that required input isn't set
with _pytest.raises(_user_exceptions.FlyteAssertion):
lp()
# Test that positional args are rejected
with _pytest.raises(_user_exceptions.FlyteAssertion):
lp(1, 2)
# Test that type checking works
with _pytest.raises(_user_exceptions.FlyteTypeException):
lp(required_input="abc", default_input=1)
# Test that bad arg name is detected
with _pytest.raises(_user_exceptions.FlyteAssertion):
lp(required_input=1, bad_arg=1)
# Test default input is accounted for
n = lp(required_input=10)
assert n.inputs[0].var == "default_input"
assert n.inputs[0].binding.scalar.primitive.integer == 5
assert n.inputs[1].var == "required_input"
assert n.inputs[1].binding.scalar.primitive.integer == 10
# Test default input is overridden
n = lp(required_input=10, default_input=50)
assert n.inputs[0].var == "default_input"
assert n.inputs[0].binding.scalar.primitive.integer == 50
assert n.inputs[1].var == "required_input"
assert n.inputs[1].binding.scalar.primitive.integer == 10
# Test that launch plan ID ref is flexible
lp._id = "fake"
assert n.workflow_node.launchplan_ref == "fake"
lp._id = None
# Test that outputs are promised
n.assign_id_and_return("node-id")
assert n.outputs["out"].sdk_type.to_flyte_literal_type(
).collection_type.simple == _type_models.SimpleType.INTEGER
assert n.outputs["out"].var == "out"
assert n.outputs["out"].node_id == "node-id"
def test_no_schedule():
workflow_to_test = _workflow.workflow(
{},
inputs={
'required_input': _workflow.Input(_types.Types.Integer),
'default_input': _workflow.Input(_types.Types.Integer, default=5)
})
lp = workflow_to_test.create_launch_plan()
assert lp.entity_metadata.schedule.kickoff_time_input_arg == ''
assert lp.entity_metadata.schedule.schedule_expression is None
assert not lp.is_scheduled
def test_kubernetes_service_account():
workflow_to_test = _workflow.workflow(
{},
inputs={
"required_input": _workflow.Input(_types.Types.Integer),
"default_input": _workflow.Input(_types.Types.Integer, default=5),
},
)
lp = workflow_to_test.create_launch_plan(
kubernetes_service_account="kube-service-acct")
assert lp.auth_role.kubernetes_service_account == "kube-service-acct"
def test_schedule_pointing_to_datetime():
workflow_to_test = _workflow.workflow(
{},
inputs={
'required_input': _workflow.Input(_types.Types.Datetime),
'default_input': _workflow.Input(_types.Types.Integer, default=5)
})
lp = workflow_to_test.create_launch_plan(schedule=_schedules.CronSchedule(
"* * ? * * *", kickoff_time_input_arg='required_input'),
role='what')
assert lp.entity_metadata.schedule.kickoff_time_input_arg == 'required_input'
assert lp.entity_metadata.schedule.cron_expression == "* * ? * * *"
def test_no_additional_inputs():
workflow_to_test = _workflow.workflow(
{},
inputs={
'required_input': _workflow.Input(_types.Types.Integer),
'default_input': _workflow.Input(_types.Types.Integer, default=5)
})
lp = workflow_to_test.create_launch_plan()
assert len(lp.fixed_inputs.literals) == 0
assert lp.default_inputs.parameters[
'default_input'].default.scalar.primitive.integer == 5
assert lp.default_inputs.parameters['required_input'].required is True
def test_annotations():
workflow_to_test = _workflow.workflow(
{},
inputs={
'required_input': _workflow.Input(_types.Types.Integer),
'default_input': _workflow.Input(_types.Types.Integer, default=5)
})
lp = workflow_to_test.create_launch_plan(
fixed_inputs={'required_input': 5},
schedule=_schedules.CronSchedule("* * ? * * *"),
role='what',
annotations=_common_models.Annotations({"my": "annotation"}))
assert lp.annotations.values == {"my": "annotation"}
def test_default_deprecated_role():
with _configuration.TemporaryConfiguration(
_os.path.join(_os.path.dirname(_os.path.realpath(__file__)),
'../../common/configs/deprecated_local.config')):
workflow_to_test = _workflow.workflow(
{},
inputs={
'required_input': _workflow.Input(_types.Types.Integer),
'default_input': _workflow.Input(_types.Types.Integer,
default=5)
})
lp = workflow_to_test.create_launch_plan()
assert lp.auth.assumable_iam_role == 'arn:aws:iam::ABC123:role/my-flyte-role'
def test_schedule():
workflow_to_test = _workflow.workflow(
{},
inputs={
'required_input': _workflow.Input(_types.Types.Integer),
'default_input': _workflow.Input(_types.Types.Integer, default=5)
})
lp = workflow_to_test.create_launch_plan(
fixed_inputs={'required_input': 5},
schedule=_schedules.CronSchedule("* * ? * * *"),
role='what')
assert lp.entity_metadata.schedule.kickoff_time_input_arg is None
assert lp.entity_metadata.schedule.cron_expression == "* * ? * * *"
assert lp.is_scheduled
def test_redefining_inputs_good():
workflow_to_test = _workflow.workflow(
{},
inputs={
'required_input': _workflow.Input(_types.Types.Integer),
'default_input': _workflow.Input(_types.Types.Integer, default=5)
}
)
lp = workflow_to_test.create_launch_plan(
default_inputs={'required_input': _workflow.Input(_types.Types.Integer, default=900)}
)
assert len(lp.fixed_inputs.literals) == 0
assert len(lp.default_inputs.parameters) == 2
assert lp.default_inputs.parameters['required_input'].default.scalar.primitive.integer == 900
assert lp.default_inputs.parameters['default_input'].default.scalar.primitive.integer == 5
def test_labels():
workflow_to_test = _workflow.workflow(
{},
inputs={
"required_input": _workflow.Input(_types.Types.Integer),
"default_input": _workflow.Input(_types.Types.Integer, default=5),
},
)
lp = workflow_to_test.create_launch_plan(
fixed_inputs={"required_input": 5},
schedule=_schedules.CronSchedule("* * ? * * *"),
role="what",
labels=_common_models.Labels({"my": "label"}),
)
assert lp.labels.values == {"my": "label"}
def test_notifications():
workflow_to_test = _workflow.workflow(
{},
inputs={
'required_input': _workflow.Input(_types.Types.Integer),
'default_input': _workflow.Input(_types.Types.Integer, default=5)
}
)
lp = workflow_to_test.create_launch_plan(
notifications=[
_notifications.PagerDuty([_execution.WorkflowExecutionPhase.FAILED], ['*****@*****.**'])
]
)
assert len(lp.entity_metadata.notifications) == 1
assert lp.entity_metadata.notifications[0].pager_duty.recipients_email == ['*****@*****.**']
assert lp.entity_metadata.notifications[0].phases == [_execution.WorkflowExecutionPhase.FAILED]
def test_fixed_inputs():
workflow_to_test = _workflow.workflow(
{},
inputs={
"required_input": _workflow.Input(_types.Types.Integer),
"default_input": _workflow.Input(_types.Types.Integer, default=5),
},
)
lp = workflow_to_test.create_launch_plan(
fixed_inputs={"required_input": 4})
assert len(lp.fixed_inputs.literals) == 1
assert lp.fixed_inputs.literals[
"required_input"].scalar.primitive.integer == 4
assert len(lp.default_inputs.parameters) == 1
assert lp.default_inputs.parameters[
"default_input"].default.scalar.primitive.integer == 5
def dynamic_wf_no_outputs_task(wf_params, task_input_num):
wf_params.logging.info(
"Running inner task... yielding a code generated sub workflow")
input_a = Input(Types.Integer, help="Tell me something")
node1 = sq_sub_task(in1=input_a)
MyUnregisteredWorkflow = workflow(inputs={"a": input_a},
outputs={},
nodes={"node_one": node1})
setattr(MyUnregisteredWorkflow, "auto_assign_name", manual_assign_name)
MyUnregisteredWorkflow._platform_valid_name = "unregistered"
unregistered_workflow_execution = MyUnregisteredWorkflow(a=task_input_num)
yield unregistered_workflow_execution
def test_raw_data_output_prefix():
workflow_to_test = _workflow.workflow(
{},
inputs={
"required_input": _workflow.Input(_types.Types.Integer),
"default_input": _workflow.Input(_types.Types.Integer, default=5),
},
)
lp = workflow_to_test.create_launch_plan(
fixed_inputs={"required_input": 5},
raw_output_data_prefix="s3://bucket-name",
)
assert lp.raw_output_data_config.output_location_prefix == "s3://bucket-name"
lp2 = workflow_to_test.create_launch_plan(
fixed_inputs={"required_input": 5}, )
assert lp2.raw_output_data_config.output_location_prefix == ""
def test_schedule(schedule, cron_expression, cron_schedule):
workflow_to_test = _workflow.workflow(
{},
inputs={
"required_input": _workflow.Input(_types.Types.Integer),
"default_input": _workflow.Input(_types.Types.Integer, default=5),
},
)
lp = workflow_to_test.create_launch_plan(
fixed_inputs={"required_input": 5},
schedule=schedule,
role="what",
)
assert lp.entity_metadata.schedule.kickoff_time_input_arg is None
assert lp.entity_metadata.schedule.cron_expression == cron_expression
assert lp.entity_metadata.schedule.cron_schedule == cron_schedule
assert lp.is_scheduled
def test_promote_from_model():
workflow_to_test = _workflow.workflow(
{},
inputs={
'required_input': _workflow.Input(_types.Types.Integer),
'default_input': _workflow.Input(_types.Types.Integer, default=5)
})
workflow_to_test._id = _identifier.Identifier(
_identifier.ResourceType.WORKFLOW, "p", "d", "n", "v")
lp = workflow_to_test.create_launch_plan(
fixed_inputs={'required_input': 5},
schedule=_schedules.CronSchedule("* * ? * * *"),
role='what',
labels=_common_models.Labels({"my": "label"}))
with _pytest.raises(_user_exceptions.FlyteAssertion):
_launch_plan.SdkRunnableLaunchPlan.from_flyte_idl(lp.to_flyte_idl())
lp_from_spec = _launch_plan.SdkLaunchPlan.from_flyte_idl(lp.to_flyte_idl())
assert not isinstance(lp_from_spec, _launch_plan.SdkRunnableLaunchPlan)
assert isinstance(lp_from_spec, _launch_plan.SdkLaunchPlan)
assert lp_from_spec == lp
def dynamic_wf_task(wf_params, task_input_num, out):
wf_params.logging.info(
"Running inner task... yielding a code generated sub workflow")
input_a = Input(Types.Integer, help="Tell me something")
node1 = sq_sub_task(in1=input_a)
MyUnregisteredWorkflow = workflow(
inputs={"a": input_a},
outputs={
"ooo":
Output(node1.outputs.out1,
sdk_type=Types.Integer,
help="This is an integer output")
},
nodes={"node_one": node1},
)
setattr(MyUnregisteredWorkflow, "auto_assign_name", manual_assign_name)
MyUnregisteredWorkflow._platform_valid_name = "unregistered"
unregistered_workflow_execution = MyUnregisteredWorkflow(a=task_input_num)
out.set(unregistered_workflow_execution.outputs.ooo)
