def _update_steps(function_name, caller_line, args=None, template_name=None): """ Step of Argo Yaml contains name, related template and parameters. thus, we insert these information into couler.step. """ function_id = pyfunc.invocation_name(function_name, caller_line) # Update `steps` only if needed if _update_steps_lock: step_template = OrderedDict() if _run_concurrent_lock: _id = pyfunc.invocation_name(template_name, caller_line) step_template["name"] = _id else: step_template["name"] = function_id if template_name is None: step_template["template"] = function_name else: step_template["template"] = template_name if _when_prefix is not None: step_template["when"] = _when_prefix if args is not None: parameters = [] for i in range(len(args)): value = args[i] if "couler" in value: tmp = args[i].split(".") if len(tmp) < 3: raise ValueError( "wrong container return representation") # To avoid duplicate map function # value = ".".join(map(str, tmp[2:])) value = tmp[2] for item in tmp[3:]: value = value + "." + item value = '"{{steps.%s}}"' % value if _run_concurrent_lock: parameters.append({ "name": pyfunc.input_parameter(template_name, i), "value": value, }) else: parameters.append({ "name": pyfunc.input_parameter(function_name, i), "value": value, }) step_template["arguments"] = {"parameters": parameters} if _condition_id is not None: function_id = _condition_id if _while_lock: if function_id in _while_steps: _while_steps.get(function_id).append(step_template) else: _while_steps[function_id] = [step_template] else: if function_id in _steps: _steps.get(function_id).append(step_template) else: _steps[function_id] = [step_template]
def run_container( image, command=None, args=None, output=None, env=None, secret=None, resources=None, ): """Generate an Argo container template. For example, the template whalesay in https://github.com/argoproj/argo/tree/master/examples#hello-world """ function_name, caller_line = pyfunc.invocation_location() output_id = None if function_name not in _templates: template = OrderedDict({"name": function_name}) # Generate the inputs parameter for the template if args is not None: parameters = [] for i in range(len(args)): para_name = pyfunc.input_parameter(function_name, i) parameters.append({"name": para_name}) inputs = OrderedDict({"parameters": parameters}) template["inputs"] = inputs # Generate the container template container = OrderedDict() if image is not None: container["image"] = image container["command"] = ["bash", "-c"] if isinstance(command, list): container["command"].extend(command) elif command is not None: container["command"].extend([command]) if args is not None: # Rewrite the args into yaml format container["args"] = [] for i in range(len(args)): para_name = pyfunc.input_parameter(function_name, i) arg_yaml = '"{{inputs.parameters.%s}}"' % para_name container["args"].append(arg_yaml) if env is not None: container["env"] = _convert_dict_to_env_list(env) if secret is not None: env_secrets = _convert_secret_to_list(secret) if "env" not in container.keys(): container["env"] = env_secrets else: container["env"].extend(env_secrets) if resources is not None: container["resources"] = _resources(resources) template["container"] = container # Generate the output if output is not None and isinstance(output, Artifact): output_id = output.id path = output.path _output = OrderedDict() _output["parameters"] = [{ "name": output_id, "valueFrom": { "path": path } }] template["outputs"] = _output # else TODO, when container does not output anything # Update the pod with cluster specific config template = _update_pod_config(template) _templates[function_name] = template if _run_concurrent_lock: _update_steps("concurrent_func_name", _concurrent_func_line, args, function_name) else: _update_steps(function_name, caller_line, args) if output_id is None: output_id = "output-id-%s" % caller_line return pyfunc.container_output(function_name, caller_line, output_id)