def exec_while(condition, inner_while):
"""
Generate the Argo recursive logic. For example
https://github.com/argoproj/argo/blob/master/examples/README.md#recursion.
"""
# _while_lock means 'exec_while' operation begins to work
# _while_steps stores logic steps inside the recursion logic
states._while_lock = True
# Enforce inner function of the while-loop to run
if callable(inner_while):
branch = inner_while()
if branch is None:
raise SyntaxError("require function return value")
else:
raise TypeError("condition to run would be a function")
branch_dict = output.extract_step_return(branch)
recursive_name = "exec-while-" + branch_dict["name"]
recursive_id = "exec-while-" + branch_dict["id"]
if states.workflow.get_template(recursive_name) is None:
template = Steps(name=recursive_name)
else:
raise SyntaxError("Recursive function can not be called twice ")
# Generate leaving point for recursive
step_out_name = "%s-%s" % (recursive_name, "exit")
pre = condition["pre"]
pre_dict = output.extract_step_return(pre)
condition_suffix = condition["condition"]
# Generate the recursive go to step
when_prefix = "{{steps.%s.%s}} %s %s" % (
branch_dict["id"],
branch_dict["output"],
condition_suffix,
pre_dict["value"],
)
step_out_template = OrderedDict({
"name": step_out_name,
"template": recursive_name,
"when": when_prefix,
})
step_out_id = utils.invocation_name(step_out_name, recursive_id)
states._while_steps[step_out_id] = [step_out_template]
# Add steps inside the recursive logic to recursive template
template.steps = list(states._while_steps.values())
# Add this recursive logic to the templates
states.workflow.add_template(template)
# Add recursive logic to global _steps
recursive_out_step = Step(name=recursive_id, template=recursive_name)
states.workflow.add_step(name=recursive_id, step=recursive_out_step)
states._while_lock = False
states._while_steps = OrderedDict()
def when(condition, function):
"""Generates an Argo conditional step.
For example, the coinflip example in
https://github.com/argoproj/argo/blob/master/examples/coinflip.yaml.
"""
pre = condition["pre"]
post = condition["post"]
if pre is None or post is None:
raise SyntaxError("Output of function can not be null")
condition_suffix = condition["condition"]
pre_dict = output.extract_step_return(pre)
post_dict = output.extract_step_return(post)
if "name" in pre_dict:
left_function_id = pre_dict["id"]
if states.workflow.get_step(left_function_id) is None:
states.workflow.add_step(
name=left_function_id,
step=Step(name=left_function_id, template=pre_dict["name"]),
)
else:
# TODO: fixed if left branch is a variable rather than function
pre_dict["value"]
post_value = post_dict["value"]
if states._upstream_dag_task is not None:
step_type = "tasks"
states._when_task = pre_dict["id"]
else:
step_type = "steps"
states._when_prefix = "{{%s.%s.%s}} %s %s" % (
step_type,
pre_dict["id"],
pre_dict["output"],
condition_suffix,
post_value,
)
states._condition_id = "%s.%s" % (pre_dict["id"], pre_dict["output"])
# Enforce the function to run and lock to add into step
if callable(function):
function()
else:
raise TypeError("condition to run would be a function")
states._when_prefix = None
states._condition_id = None
def _update_dag_tasks(
function_name,
caller_line,
dependencies,
depends_logic,
args=None,
template_name=None,
step_name=None,
):
"""
A task in DAG of Argo YAML contains name, related template and parameters.
Here we insert a single task into the global tasks.
"""
if step_name is None:
function_id = utils.invocation_name(function_name, caller_line)
else:
function_id = step_name
task_template = states.workflow.get_dag_task(function_id)
if task_template is None:
task_template = OrderedDict({"name": function_id})
if dependencies is not None and isinstance(dependencies, list):
if "dependencies" in task_template:
task_template["dependencies"].extend(dependencies)
else:
task_template["dependencies"] = dependencies
if depends_logic is not None:
task_template["depends"] = depends_logic
if template_name is None:
task_template["template"] = function_name
else:
task_template["template"] = template_name
# configure the args
if args is not None:
parameters, artifacts = _get_params_and_artifacts_from_args(
args, function_name, prefix="tasks")
if len(parameters) > 0:
task_template["arguments"] = OrderedDict()
task_template["arguments"]["parameters"] = parameters
if len(artifacts) > 0:
if "arguments" not in task_template:
task_template["arguments"] = OrderedDict()
task_template["arguments"]["artifacts"] = artifacts
else:
# step exist on the dag, thus, we update its dependency
if dependencies is not None:
if "dependencies" in task_template:
task_template["dependencies"].extend(dependencies)
else:
task_template["dependencies"] = [dependencies]
if depends_logic is not None:
task_template["depends"] = depends_logic
t_name = function_name if template_name is None else template_name
step = Step(name=function_id, template=t_name)
if states._exit_handler_enable:
if states._when_prefix is not None:
step.when = states._when_prefix
if function_id in states.workflow.exit_handler_step:
states.workflow.exit_handler_step.get(function_id).append(
step.to_dict())
else:
states.workflow.exit_handler_step[function_id] = [step.to_dict()]
elif states._when_prefix is not None:
step.when = states._when_prefix
if step.name not in states.workflow.dag_tasks.keys():
step_spec = step.to_dict()
step_spec["dependencies"] = [states._when_task]
states.workflow.dag_tasks[step.name] = step_spec
else:
states.workflow.update_dag_task(function_id, task_template)
# return the current task name
return function_id
def _update_steps(function_name, caller_line, args=None, template_name=None):
"""
A step in Argo YAML contains name, related template and parameters.
Here we insert a single step into the global steps.
"""
function_id = utils.invocation_name(function_name, caller_line)
# Update `steps` only if needed
if states._update_steps_lock:
name = function_id
if states._run_concurrent_lock:
_id = utils.invocation_name(template_name, caller_line)
name = "%s-%s" % (_id, states._concurrent_func_id)
if states._sub_steps is not None:
states._concurrent_func_id = states._concurrent_func_id + 1
t_name = function_name if template_name is None else template_name
step = Step(name=name, template=t_name)
if states._when_prefix is not None:
step.when = states._when_prefix
if args is not None:
parameters, artifacts = _get_params_and_artifacts_from_args(
args,
template_name
if states._run_concurrent_lock else function_name,
prefix="steps",
)
if len(parameters) > 0:
step.arguments = OrderedDict()
step.arguments["parameters"] = parameters
if len(artifacts) > 0:
if step.arguments is None:
step.arguments = OrderedDict()
step.arguments["artifacts"] = artifacts
if states._condition_id is not None:
function_id = states._condition_id
if states._while_lock:
if function_id in states._while_steps:
states._while_steps.get(function_id).append(step.to_dict())
else:
states._while_steps[function_id] = [step.to_dict()]
else:
if states._sub_steps is not None:
if function_id in states._sub_steps:
states._sub_steps.get(function_id).append(step.to_dict())
else:
states._sub_steps[function_id] = [step.to_dict()]
elif states._exit_handler_enable is True:
if function_id in states.workflow.exit_handler_step:
states.workflow.exit_handler_step.get(function_id).append(
step.to_dict())
else:
states.workflow.exit_handler_step[function_id] = [
step.to_dict()
]
else:
states.workflow.add_step(function_id, step)
return step.name
else:
return function_id
def map(function, input_list):
"""
map operation of Couler
"""
# Enforce the function to run and lock to add into step
if callable(function):
states._update_steps_lock = False
# TODO (terrytangyuan): Support functions with multiple arguments.
para = input_list[0]
inner = function(para)
if inner is None:
raise SyntaxError("require function return value")
states._update_steps_lock = True
else:
raise TypeError("require loop over a function to run")
inner_dict = output.extract_step_return(inner)
template_name = inner_dict["name"]
inner_step = Step(name=inner_dict["id"], template=template_name)
parameters = []
items_param_name = "%s-para-name" % template_name
items_param_dict = {"name": items_param_name}
function_template = states.workflow.get_template(template_name)
function_template_dict = function_template.to_dict()
if "resource" in function_template_dict:
# Update the template with the new dynamic `metadata.name`.
manifest_dict = yaml.safe_load(
function_template_dict["resource"]["manifest"]
)
manifest_dict["metadata"]["name"] = (
"'{{inputs.parameters.%s}}'" % items_param_name
)
function_template = states.workflow.get_template(template_name)
function_template.manifest = pyaml.dump(manifest_dict)
# Append this items parameter to input parameters in the template
function_template.args.append(items_param_dict)
states.workflow.add_template(function_template)
input_parameters = [items_param_dict]
else:
input_parameters = function_template_dict["inputs"]["parameters"]
for para_name in input_parameters:
parameters.append(
{
"name": para_name["name"],
"value": '"{{item.%s}}"' % para_name["name"],
}
)
inner_step.arguments = {"parameters": parameters}
with_items = []
for para_values in input_list:
item = {}
if not isinstance(para_values, list):
para_values = [para_values]
for j in range(len(input_parameters)):
para_name = input_parameters[j]["name"]
item[para_name] = para_values[j]
with_items.append(item)
inner_step.with_items = with_items
states.workflow.add_step(inner_dict["id"], inner_step)
return inner_step
def map(function, *arguments):
"""
map operation of Couler
"""
# Enforce the function to run and lock to add into step
# Checks the correct syntax
if callable(function):
states._update_steps_lock = False
para = []
x = 0
while x < len(arguments):
para.append(arguments[x][0])
x += 1
inner = function(*para)
if inner is None:
raise SyntaxError("require function return value")
states._update_steps_lock = True
else:
raise TypeError("require loop over a function to run")
inner_dict = output.extract_step_return(inner)
template_name = inner_dict["name"]
inner_step = Step(name=inner_dict["id"], template=template_name)
parameters = []
items_param_name = "%s-para-name" % template_name
items_param_dict = {"name": items_param_name}
function_template = states.workflow.get_template(template_name)
function_template_dict = function_template.to_dict()
if "resource" in function_template_dict:
# Update the template with the new dynamic `metadata.name`.
manifest_dict = yaml.safe_load(
function_template_dict["resource"]["manifest"])
manifest_dict["metadata"]["name"] = ("'{{inputs.parameters.%s}}'" %
items_param_name)
function_template = states.workflow.get_template(template_name)
function_template.manifest = pyaml.dump(manifest_dict)
# Append this items parameter to input parameters in the template
function_template.args.append(items_param_dict)
states.workflow.add_template(function_template)
input_parameters = [items_param_dict]
else:
input_parameters = function_template_dict["inputs"]["parameters"]
for para_name in input_parameters:
parameters.append({
"name": para_name["name"],
"value": '"{{item.%s}}"' % para_name["name"],
})
inner_step.arguments = {"parameters": parameters}
# the following part of the code
# Adds values to parameters (with items) while it goes
# through the *arguments-variable with two loops.
# inner loop:
# arguments[ind_of_func_param][0], arguments[ind_of_func_param][0]...
# Outer loop:
# arguments[0][ind_of_func_call], arguments[0][ind_of_func_call]...
# With two lists in *arguments
# result would be:
# 1. pair of items for the function:
# arguments[0][0], arguments[1][0]
# 2. pair of items for the function:
# arguments[0][1], arguments[1][1]
# and so on...
with_items = []
ind_of_func_call = 0
# the number of calls to be made to function
while ind_of_func_call < len(arguments[0]):
ind_of_func_param = 0
item = {}
# the number of parameters function takes.
while ind_of_func_param < len(arguments):
# checks if arguments[ind_of_func_param] is a list if not makes it
if not isinstance(arguments[ind_of_func_param], list):
arguments[ind_of_func_param] = [arguments[ind_of_func_param]]
# items are created for the with items part in .yaml
para_name = input_parameters[ind_of_func_param]["name"]
item[para_name] = arguments[ind_of_func_param][ind_of_func_call]
ind_of_func_param += 1
with_items.append(item)
ind_of_func_call += 1
# all the created items are added to the step and
# then the step is added to .yaml
inner_step.with_items = with_items
states.workflow.add_step(inner_dict["id"], inner_step)
return inner_step