def dag_phase(self, data, last_builder): for step in sb_step.global_steps: if (step.type == tool_invocation_step): step.container = get_container_with_tool(step.tool_to_call) else: step.container = self.simple_container last_elem = None # Keys: Step nmes. Values a list with tuples: # A tuple contains the step object and the dependency step_dependencies = defaultdict(list) for step in sb_step.global_steps: if (step.type == step_type.initial): #print (last_builder, type(last_builder)) couler.set_dependencies(lambda: self.sb_step_call(step), dependencies=last_builder) continue if len(step.dependencies) == 0: key = len(list(couler.workflow.dag_tasks.keys())) - 1 if key >= 0: last_elem = list(couler.workflow.dag_tasks.keys())[key] sb_step_name = utils.argo_safe_name(last_elem) couler.set_dependencies(lambda: self.sb_step_call(step), dependencies=sb_step_name) else: for dep in step.dependencies: key = None initial = None for tmp in sb_step.global_steps: if tmp.name == dep.name: key = tmp if tmp.type == step_type.initial: initial = tmp if (key.type == step_type.tool_installation): sb_step_name = utils.argo_safe_name(initial.name) couler.set_dependencies( lambda: self.sb_step_call(step), dependencies=sb_step_name) else: sb_step_name = utils.argo_safe_name(dep.name) # Do not set dependencies here. This dependenciy might belong to more than on step #couler.set_dependencies(lambda: sb_step_call(wfl, step), dependencies=sb_step_name) step_dependencies[step.name].append( (step, sb_step_name)) # print(step.name, " depends on ", dep.name) # Add step dependencies for k, v in step_dependencies.items(): couler.set_dependencies(lambda: self.sb_step_call(v[0][0]), dependencies=' && '.join(x[1] for x in v))
def run_script( image, command=None, source=None, env=None, resources=None, secret=None, timeout=None, retry=None, step_name=None, image_pull_policy=None, pool=None, daemon=False, ): """Generate an Argo script template. For example, https://github.com/argoproj/argo/tree/master/examples#scripts--results. Step_name is only used for annotating step while developing step zoo. """ func_name, caller_line = utils.invocation_location() func_name = (utils.argo_safe_name(step_name) if step_name is not None else func_name) if states.workflow.get_template(func_name) is None: if source is None: raise ValueError("Input script can not be null") template = Script( name=func_name, image=image, command=command, source=source, env=env, secret=states.get_secret(secret), resources=resources, timeout=timeout, retry=retry, image_pull_policy=image_pull_policy, pool=pool, daemon=daemon, ) states.workflow.add_template(template) step_name = step_update_utils.update_step(func_name, args=None, step_name=step_name, caller_line=caller_line) rets = _script_output(step_name, func_name) states._steps_outputs[step_name] = rets # TODO(typhoonzero): return pb_step when using a couler server. pb_step = proto_repr.step_repr( # noqa: F841 step_name=step_name, tmpl_name=func_name, image=image, command=command, source=source, script_output=rets, ) return rets
def run_canned_step(name, args, inputs=None, outputs=None, step_name=None, cache=None): func_name, caller_line = utils.invocation_location() func_name = (utils.argo_safe_name(step_name) if step_name is not None else func_name) step_name = step_update_utils.update_step(func_name, args, step_name, caller_line) tmpl_args = [] if states._outputs_tmp is not None: tmpl_args.extend(states._outputs_tmp) pb_step = None if proto_repr: pb_step = proto_repr.step_repr( # noqa: F841 input=inputs, output=outputs, canned_step_name=name, canned_step_args=args, step_name=step_name, tmpl_name=step_name + "-tmpl", args=tmpl_args, cache=cache, ) proto_repr.add_deps_to_step(step_name) return pb_step
def sb_step_call(self, step: sb_step): c = self.simple_container if (step.type == step_type.tool_invocation): conts = get_container_with_tool(step.tool_to_call) c = conts[0] elif (step.type == step_type.tool_installation): return artifact = rawArtifact("/root/step_bash.sh", step.bash) sb_step_name = utils.argo_safe_name(step.name) obc_env = { "OBC_WORK_PATH": self.work_path, "OBC_TOOL_PATH": self.work_path, "OBC_DATA_PATH": self.work_path } return couler.run_container(c.image, command=["/bin/bash"], args="/root/step_bash.sh", input=[artifact], step_name=sb_step_name, env=obc_env)
def test_argo_safe_name(self): self.assertIsNone(utils.argo_safe_name(None)) self.assertEqual(utils.argo_safe_name("a_b"), "a-b") self.assertEqual(utils.argo_safe_name("a.b"), "a-b") self.assertEqual(utils.argo_safe_name("a_.b"), "a--b") self.assertEqual(utils.argo_safe_name("_abc."), "-abc-")
def run_container( image, command=None, args=None, output=None, input=None, env=None, secret=None, resources=None, timeout=None, retry=None, step_name=None, image_pull_policy=None, pool=None, enable_ulogfs=True, daemon=False, volume_mounts=None, working_dir=None, node_selector=None, ): """ Generate an Argo container template. For example, the template whalesay in https://github.com/argoproj/argo/tree/master/examples#hello-world. :param image: :param command: :param args: :param output: output artifact for container output :param input: input artifact for container input :param env: environmental variable :param secret: :param resources: CPU or memory resource config dict :param timeout: in seconds :param retry: retry policy :param step_name: used for annotating step . :param image_pull_policy: :param pool: :param enable_ulogfs: :param daemon: :return: """ func_name, caller_line = utils.invocation_location() func_name = (utils.argo_safe_name(step_name) if step_name is not None else func_name) if states.workflow.get_template(func_name) is None: # Generate the inputs parameter for the template if input is None: input = [] if args is None and states._outputs_tmp is not None: args = [] if args is not None: if not isinstance(args, list): args = [args] # Handle case where args is a list of list type # For example, [[Output, ]] if (isinstance(args, list) and len(args) > 0 and isinstance(args[0], list) and len(args[0]) > 0 and isinstance(args[0][0], Output)): args = args[0] if states._outputs_tmp is not None: args.extend(states._outputs_tmp) # In case, the args include output artifact # Place output artifact into the input for arg in args: if isinstance(arg, (OutputArtifact, OutputJob)): input.append(arg) # Automatically append emptyDir volume and volume mount to work with # Argo k8sapi executor. # More info: https://argoproj.github.io/argo/empty-dir/ if output is not None: if not isinstance(output, list): output = [output] if volume_mounts is None: volume_mounts = [] mounted_path = [] for i, out in enumerate(output): if "/tmp" in out.path: raise ValueError("Mounting to /tmp is not supported") path_to_mount = os.path.dirname(out.path) # Avoid duplicate mount paths if path_to_mount not in mounted_path: volume_mounts.append( VolumeMount("couler-out-dir-%s" % i, path_to_mount)) mounted_path.append(path_to_mount) # Generate container and template template = Container( name=func_name, image=image, command=command, args=args, env=env, secret=states.get_secret(secret), resources=resources, image_pull_policy=image_pull_policy, retry=retry, timeout=timeout, output=output, input=input, pool=pool, enable_ulogfs=enable_ulogfs, daemon=daemon, volume_mounts=volume_mounts, working_dir=working_dir, node_selector=node_selector, ) states.workflow.add_template(template) step_name = step_update_utils.update_step(func_name, args, step_name, caller_line) # TODO: need to switch to use field `output` directly step_templ = states.workflow.get_template(func_name) _output = step_templ.to_dict().get("outputs", None) _input = step_templ.to_dict().get("inputs", None) rets = _container_output(step_name, func_name, _output) states._steps_outputs[step_name] = rets pb_step = proto_repr.step_repr( # noqa: F841 step_name=step_name, tmpl_name=func_name, image=image, command=command, source=None, script_output=None, args=args, input=_input, output=_output, ) return rets
def run_job( manifest, success_condition, failure_condition, timeout=None, retry=None, step_name=None, pool=None, env=None, set_owner_reference=True, ): """ Create a k8s job. For example, the pi-tmpl template in https://github.com/argoproj/argo/blob/master/examples/k8s-jobs.yaml :param manifest: YAML specification of the job to be created. :param success_condition: expression for verifying job success. :param failure_condition: expression for verifying job failure. :param timeout: To limit the elapsed time for a workflow in seconds. :param step_name: is only used while developing functions of step zoo. :param env: environmental parameter with a dict types, e.g., {"OS_ENV_1": "OS_ENV_value"} # noqa: E501 :param set_owner_reference: Whether to set the workflow as the job's owner reference. If `True`, the job will be deleted once the workflow is deleted. :return: output """ if manifest is None: raise ValueError("Input manifest can not be null") func_name, caller_line = utils.invocation_location() func_name = (utils.argo_safe_name(step_name) if step_name is not None else func_name) args = [] if states.workflow.get_template(func_name) is None: if states._outputs_tmp is not None and env is not None: env["inferred_outputs"] = states._outputs_tmp # Generate the inputs for the manifest template envs, parameters, args = utils.generate_parameters_run_job(env) # update the env if env is not None: manifest_dict = yaml.safe_load(manifest) manifest_dict["spec"]["env"] = envs # TODO this is used to pass the test cases, # should be fixed in a better way if ("labels" in manifest_dict["metadata"] and "argo.step.owner" in manifest_dict["metadata"]["labels"]): manifest_dict["metadata"]["labels"][ "argo.step.owner"] = "'{{pod.name}}'" manifest = pyaml.dump(manifest_dict) template = Job( name=func_name, args=args, action="create", manifest=manifest, set_owner_reference=set_owner_reference, success_condition=success_condition, failure_condition=failure_condition, timeout=timeout, retry=retry, pool=pool, ) states.workflow.add_template(template) step_name = step_update_utils.update_step(func_name, args, step_name, caller_line) # return job name and job uid for reference rets = _job_output(step_name, func_name) states._steps_outputs[step_name] = rets pb_step = proto_repr.step_repr( # noqa: F841 step_name=step_name, tmpl_name=func_name, image=None, source=None, script_output=None, input=None, output=rets, manifest=manifest, success_cond=success_condition, failure_cond=failure_condition, ) return rets
def run_container( image, command=None, args=None, output=None, input=None, env=None, secret=None, resources=None, timeout=None, retry=None, step_name=None, image_pull_policy=None, pool=None, enable_ulogfs=True, daemon=False, volume_mounts=None, working_dir=None, node_selector=None, ): """ Generate an Argo container template. For example, the template whalesay in https://github.com/argoproj/argo/tree/master/examples#hello-world. :param image: :param command: :param args: :param output: output artifact for container output :param input: input artifact for container input :param env: environmental variable :param secret: :param resources: CPU or memory resource config dict :param timeout: in seconds :param retry: retry policy :param step_name: used for annotating step . :param image_pull_policy: :param pool: :param enable_ulogfs: :param daemon: :return: """ func_name, caller_line = utils.invocation_location() func_name = (utils.argo_safe_name(step_name) if step_name is not None else func_name) if states.workflow.get_template(func_name) is None: # Generate the inputs parameter for the template if input is None: input = [] if args is None and states._outputs_tmp is not None: args = [] if args is not None: if not isinstance(args, list): args = [args] # Handle case where args is a list of list type # For example, [[Output, ]] if (isinstance(args, list) and len(args) > 0 and isinstance(args[0], list) and len(args[0]) > 0 and isinstance(args[0][0], Output)): args = args[0] if states._outputs_tmp is not None: args.extend(states._outputs_tmp) # In case, the args include output artifact # Place output artifact into the input for arg in args: if isinstance(arg, (OutputArtifact, OutputJob)): input.append(arg) # Generate container and template template = Container( name=func_name, image=image, command=command, args=args, env=env, secret=states.get_secret(secret), resources=resources, image_pull_policy=image_pull_policy, retry=retry, timeout=timeout, output=output, input=input, pool=pool, enable_ulogfs=enable_ulogfs, daemon=daemon, volume_mounts=volume_mounts, working_dir=working_dir, node_selector=node_selector, ) states.workflow.add_template(template) step_name = step_update_utils.update_step(func_name, args, step_name, caller_line) # TODO: need to switch to use field `output` directly _output = (states.workflow.get_template(func_name).to_dict().get( "outputs", None)) rets = _container_output(step_name, func_name, _output) states._steps_outputs[step_name] = rets return rets