def _update_steps(function_name, caller_line, args=None, template_name=None):
"""
Step of Argo Yaml contains name, related template and parameters.
thus, we insert these information into couler.step.
"""
function_id = pyfunc.invocation_name(function_name, caller_line)
# Update `steps` only if needed
if _update_steps_lock:
step_template = OrderedDict()
if _run_concurrent_lock:
_id = pyfunc.invocation_name(template_name, caller_line)
step_template["name"] = _id
else:
step_template["name"] = function_id
if template_name is None:
step_template["template"] = function_name
else:
step_template["template"] = template_name
if _when_prefix is not None:
step_template["when"] = _when_prefix
if args is not None:
parameters = []
for i in range(len(args)):
value = args[i]
if "couler" in value:
tmp = args[i].split(".")
if len(tmp) < 3:
raise ValueError(
"wrong container return representation")
# To avoid duplicate map function
# value = ".".join(map(str, tmp[2:]))
value = tmp[2]
for item in tmp[3:]:
value = value + "." + item
value = '"{{steps.%s}}"' % value
if _run_concurrent_lock:
parameters.append({
"name":
pyfunc.input_parameter(template_name, i),
"value":
value,
})
else:
parameters.append({
"name":
pyfunc.input_parameter(function_name, i),
"value":
value,
})
step_template["arguments"] = {"parameters": parameters}
if _condition_id is not None:
function_id = _condition_id
if _while_lock:
if function_id in _while_steps:
_while_steps.get(function_id).append(step_template)
else:
_while_steps[function_id] = [step_template]
else:
if function_id in _steps:
_steps.get(function_id).append(step_template)
else:
_steps[function_id] = [step_template]
def run_container(
image,
command=None,
args=None,
output=None,
env=None,
secret=None,
resources=None,
):
"""Generate an Argo container template. For example, the template whalesay
in https://github.com/argoproj/argo/tree/master/examples#hello-world
"""
function_name, caller_line = pyfunc.invocation_location()
output_id = None
if function_name not in _templates:
template = OrderedDict({"name": function_name})
# Generate the inputs parameter for the template
if args is not None:
parameters = []
for i in range(len(args)):
para_name = pyfunc.input_parameter(function_name, i)
parameters.append({"name": para_name})
inputs = OrderedDict({"parameters": parameters})
template["inputs"] = inputs
# Generate the container template
container = OrderedDict()
if image is not None:
container["image"] = image
container["command"] = ["bash", "-c"]
if isinstance(command, list):
container["command"].extend(command)
elif command is not None:
container["command"].extend([command])
if args is not None:
# Rewrite the args into yaml format
container["args"] = []
for i in range(len(args)):
para_name = pyfunc.input_parameter(function_name, i)
arg_yaml = '"{{inputs.parameters.%s}}"' % para_name
container["args"].append(arg_yaml)
if env is not None:
container["env"] = _convert_dict_to_env_list(env)
if secret is not None:
env_secrets = _convert_secret_to_list(secret)
if "env" not in container.keys():
container["env"] = env_secrets
else:
container["env"].extend(env_secrets)
if resources is not None:
container["resources"] = _resources(resources)
template["container"] = container
# Generate the output
if output is not None and isinstance(output, Artifact):
output_id = output.id
path = output.path
_output = OrderedDict()
_output["parameters"] = [{
"name": output_id,
"valueFrom": {
"path": path
}
}]
template["outputs"] = _output
# else TODO, when container does not output anything
# Update the pod with cluster specific config
template = _update_pod_config(template)
_templates[function_name] = template
if _run_concurrent_lock:
_update_steps("concurrent_func_name", _concurrent_func_line, args,
function_name)
else:
_update_steps(function_name, caller_line, args)
if output_id is None:
output_id = "output-id-%s" % caller_line
return pyfunc.container_output(function_name, caller_line, output_id)