def exec_while(condition, inner_while): """ Generate the Argo recursive logic. For example https://github.com/argoproj/argo/blob/master/examples/README.md#recursion. """ # _while_lock means 'exec_while' operation begins to work # _while_steps stores logic steps inside the recursion logic states._while_lock = True # Enforce inner function of the while-loop to run if callable(inner_while): branch = inner_while() if branch is None: raise SyntaxError("require function return value") else: raise TypeError("condition to run would be a function") branch_dict = output.extract_step_return(branch) recursive_name = "exec-while-" + branch_dict["name"] recursive_id = "exec-while-" + branch_dict["id"] if states.workflow.get_template(recursive_name) is None: template = Steps(name=recursive_name) else: raise SyntaxError("Recursive function can not be called twice ") # Generate leaving point for recursive step_out_name = "%s-%s" % (recursive_name, "exit") pre = condition["pre"] pre_dict = output.extract_step_return(pre) condition_suffix = condition["condition"] # Generate the recursive go to step when_prefix = "{{steps.%s.%s}} %s %s" % ( branch_dict["id"], branch_dict["output"], condition_suffix, pre_dict["value"], ) step_out_template = OrderedDict({ "name": step_out_name, "template": recursive_name, "when": when_prefix, }) step_out_id = utils.invocation_name(step_out_name, recursive_id) states._while_steps[step_out_id] = [step_out_template] # Add steps inside the recursive logic to recursive template template.steps = list(states._while_steps.values()) # Add this recursive logic to the templates states.workflow.add_template(template) # Add recursive logic to global _steps recursive_out_step = Step(name=recursive_id, template=recursive_name) states.workflow.add_step(name=recursive_id, step=recursive_out_step) states._while_lock = False states._while_steps = OrderedDict()
def when(condition, function): """Generates an Argo conditional step. For example, the coinflip example in https://github.com/argoproj/argo/blob/master/examples/coinflip.yaml. """ pre = condition["pre"] post = condition["post"] if pre is None or post is None: raise SyntaxError("Output of function can not be null") condition_suffix = condition["condition"] pre_dict = output.extract_step_return(pre) post_dict = output.extract_step_return(post) if "name" in pre_dict: left_function_id = pre_dict["id"] if states.workflow.get_step(left_function_id) is None: states.workflow.add_step( name=left_function_id, step=Step(name=left_function_id, template=pre_dict["name"]), ) else: # TODO: fixed if left branch is a variable rather than function pre_dict["value"] post_value = post_dict["value"] if states._upstream_dag_task is not None: step_type = "tasks" states._when_task = pre_dict["id"] else: step_type = "steps" states._when_prefix = "{{%s.%s.%s}} %s %s" % ( step_type, pre_dict["id"], pre_dict["output"], condition_suffix, post_value, ) states._condition_id = "%s.%s" % (pre_dict["id"], pre_dict["output"]) # Enforce the function to run and lock to add into step if callable(function): function() else: raise TypeError("condition to run would be a function") states._when_prefix = None states._condition_id = None
def _update_dag_tasks( function_name, caller_line, dependencies, depends_logic, args=None, template_name=None, step_name=None, ): """ A task in DAG of Argo YAML contains name, related template and parameters. Here we insert a single task into the global tasks. """ if step_name is None: function_id = utils.invocation_name(function_name, caller_line) else: function_id = step_name task_template = states.workflow.get_dag_task(function_id) if task_template is None: task_template = OrderedDict({"name": function_id}) if dependencies is not None and isinstance(dependencies, list): if "dependencies" in task_template: task_template["dependencies"].extend(dependencies) else: task_template["dependencies"] = dependencies if depends_logic is not None: task_template["depends"] = depends_logic if template_name is None: task_template["template"] = function_name else: task_template["template"] = template_name # configure the args if args is not None: parameters, artifacts = _get_params_and_artifacts_from_args( args, function_name, prefix="tasks") if len(parameters) > 0: task_template["arguments"] = OrderedDict() task_template["arguments"]["parameters"] = parameters if len(artifacts) > 0: if "arguments" not in task_template: task_template["arguments"] = OrderedDict() task_template["arguments"]["artifacts"] = artifacts else: # step exist on the dag, thus, we update its dependency if dependencies is not None: if "dependencies" in task_template: task_template["dependencies"].extend(dependencies) else: task_template["dependencies"] = [dependencies] if depends_logic is not None: task_template["depends"] = depends_logic t_name = function_name if template_name is None else template_name step = Step(name=function_id, template=t_name) if states._exit_handler_enable: if states._when_prefix is not None: step.when = states._when_prefix if function_id in states.workflow.exit_handler_step: states.workflow.exit_handler_step.get(function_id).append( step.to_dict()) else: states.workflow.exit_handler_step[function_id] = [step.to_dict()] elif states._when_prefix is not None: step.when = states._when_prefix if step.name not in states.workflow.dag_tasks.keys(): step_spec = step.to_dict() step_spec["dependencies"] = [states._when_task] states.workflow.dag_tasks[step.name] = step_spec else: states.workflow.update_dag_task(function_id, task_template) # return the current task name return function_id
def _update_steps(function_name, caller_line, args=None, template_name=None): """ A step in Argo YAML contains name, related template and parameters. Here we insert a single step into the global steps. """ function_id = utils.invocation_name(function_name, caller_line) # Update `steps` only if needed if states._update_steps_lock: name = function_id if states._run_concurrent_lock: _id = utils.invocation_name(template_name, caller_line) name = "%s-%s" % (_id, states._concurrent_func_id) if states._sub_steps is not None: states._concurrent_func_id = states._concurrent_func_id + 1 t_name = function_name if template_name is None else template_name step = Step(name=name, template=t_name) if states._when_prefix is not None: step.when = states._when_prefix if args is not None: parameters, artifacts = _get_params_and_artifacts_from_args( args, template_name if states._run_concurrent_lock else function_name, prefix="steps", ) if len(parameters) > 0: step.arguments = OrderedDict() step.arguments["parameters"] = parameters if len(artifacts) > 0: if step.arguments is None: step.arguments = OrderedDict() step.arguments["artifacts"] = artifacts if states._condition_id is not None: function_id = states._condition_id if states._while_lock: if function_id in states._while_steps: states._while_steps.get(function_id).append(step.to_dict()) else: states._while_steps[function_id] = [step.to_dict()] else: if states._sub_steps is not None: if function_id in states._sub_steps: states._sub_steps.get(function_id).append(step.to_dict()) else: states._sub_steps[function_id] = [step.to_dict()] elif states._exit_handler_enable is True: if function_id in states.workflow.exit_handler_step: states.workflow.exit_handler_step.get(function_id).append( step.to_dict()) else: states.workflow.exit_handler_step[function_id] = [ step.to_dict() ] else: states.workflow.add_step(function_id, step) return step.name else: return function_id
def map(function, input_list): """ map operation of Couler """ # Enforce the function to run and lock to add into step if callable(function): states._update_steps_lock = False # TODO (terrytangyuan): Support functions with multiple arguments. para = input_list[0] inner = function(para) if inner is None: raise SyntaxError("require function return value") states._update_steps_lock = True else: raise TypeError("require loop over a function to run") inner_dict = output.extract_step_return(inner) template_name = inner_dict["name"] inner_step = Step(name=inner_dict["id"], template=template_name) parameters = [] items_param_name = "%s-para-name" % template_name items_param_dict = {"name": items_param_name} function_template = states.workflow.get_template(template_name) function_template_dict = function_template.to_dict() if "resource" in function_template_dict: # Update the template with the new dynamic `metadata.name`. manifest_dict = yaml.safe_load( function_template_dict["resource"]["manifest"] ) manifest_dict["metadata"]["name"] = ( "'{{inputs.parameters.%s}}'" % items_param_name ) function_template = states.workflow.get_template(template_name) function_template.manifest = pyaml.dump(manifest_dict) # Append this items parameter to input parameters in the template function_template.args.append(items_param_dict) states.workflow.add_template(function_template) input_parameters = [items_param_dict] else: input_parameters = function_template_dict["inputs"]["parameters"] for para_name in input_parameters: parameters.append( { "name": para_name["name"], "value": '"{{item.%s}}"' % para_name["name"], } ) inner_step.arguments = {"parameters": parameters} with_items = [] for para_values in input_list: item = {} if not isinstance(para_values, list): para_values = [para_values] for j in range(len(input_parameters)): para_name = input_parameters[j]["name"] item[para_name] = para_values[j] with_items.append(item) inner_step.with_items = with_items states.workflow.add_step(inner_dict["id"], inner_step) return inner_step
def map(function, *arguments): """ map operation of Couler """ # Enforce the function to run and lock to add into step # Checks the correct syntax if callable(function): states._update_steps_lock = False para = [] x = 0 while x < len(arguments): para.append(arguments[x][0]) x += 1 inner = function(*para) if inner is None: raise SyntaxError("require function return value") states._update_steps_lock = True else: raise TypeError("require loop over a function to run") inner_dict = output.extract_step_return(inner) template_name = inner_dict["name"] inner_step = Step(name=inner_dict["id"], template=template_name) parameters = [] items_param_name = "%s-para-name" % template_name items_param_dict = {"name": items_param_name} function_template = states.workflow.get_template(template_name) function_template_dict = function_template.to_dict() if "resource" in function_template_dict: # Update the template with the new dynamic `metadata.name`. manifest_dict = yaml.safe_load( function_template_dict["resource"]["manifest"]) manifest_dict["metadata"]["name"] = ("'{{inputs.parameters.%s}}'" % items_param_name) function_template = states.workflow.get_template(template_name) function_template.manifest = pyaml.dump(manifest_dict) # Append this items parameter to input parameters in the template function_template.args.append(items_param_dict) states.workflow.add_template(function_template) input_parameters = [items_param_dict] else: input_parameters = function_template_dict["inputs"]["parameters"] for para_name in input_parameters: parameters.append({ "name": para_name["name"], "value": '"{{item.%s}}"' % para_name["name"], }) inner_step.arguments = {"parameters": parameters} # the following part of the code # Adds values to parameters (with items) while it goes # through the *arguments-variable with two loops. # inner loop: # arguments[ind_of_func_param][0], arguments[ind_of_func_param][0]... # Outer loop: # arguments[0][ind_of_func_call], arguments[0][ind_of_func_call]... # With two lists in *arguments # result would be: # 1. pair of items for the function: # arguments[0][0], arguments[1][0] # 2. pair of items for the function: # arguments[0][1], arguments[1][1] # and so on... with_items = [] ind_of_func_call = 0 # the number of calls to be made to function while ind_of_func_call < len(arguments[0]): ind_of_func_param = 0 item = {} # the number of parameters function takes. while ind_of_func_param < len(arguments): # checks if arguments[ind_of_func_param] is a list if not makes it if not isinstance(arguments[ind_of_func_param], list): arguments[ind_of_func_param] = [arguments[ind_of_func_param]] # items are created for the with items part in .yaml para_name = input_parameters[ind_of_func_param]["name"] item[para_name] = arguments[ind_of_func_param][ind_of_func_call] ind_of_func_param += 1 with_items.append(item) ind_of_func_call += 1 # all the created items are added to the step and # then the step is added to .yaml inner_step.with_items = with_items states.workflow.add_step(inner_dict["id"], inner_step) return inner_step