def test_run_container_with_workflow_volume(self):
pvc = VolumeClaimTemplate("workdir")
volume_mount = VolumeMount("workdir", "/mnt/vol")
couler.create_workflow_volume(pvc)
couler.run_container(
image="docker/whalesay:latest",
args=["echo -n hello world"],
command=["bash", "-c"],
step_name="A",
volume_mounts=[volume_mount],
)
volume_mount = VolumeMount("workdir", "/mnt/vol")
couler.run_container(
image="docker/whalesay:latest",
args=["echo -n hello world"],
command=["bash", "-c"],
step_name="A",
volume_mounts=[volume_mount],
)
wf = couler.workflow_yaml()
self.assertEqual(len(wf["spec"]["volumeClaimTemplates"]), 1)
self.assertEqual(wf["spec"]["volumeClaimTemplates"][0], pvc.to_dict())
self.assertEqual(
wf["spec"]["templates"][1]["container"]["volumeMounts"][0],
volume_mount.to_dict(),
)
couler._cleanup()
def test_run_container_with_volume(self):
volume = Volume("workdir", "my-existing-volume")
volume_mount = VolumeMount("workdir", "/mnt/vol")
couler.add_volume(volume)
couler.run_container(
image="docker/whalesay:latest",
args=["echo -n hello world"],
command=["bash", "-c"],
step_name="A",
volume_mounts=[volume_mount],
)
wf = couler.workflow_yaml()
self.assertEqual(wf["spec"]["volumes"][0], volume.to_dict())
self.assertEqual(
wf["spec"]["templates"][1]["container"]["volumeMounts"][0],
volume_mount.to_dict(),
)
couler._cleanup()
def run_container(
image,
command=None,
args=None,
output=None,
input=None,
env=None,
secret=None,
resources=None,
timeout=None,
retry=None,
step_name=None,
image_pull_policy=None,
pool=None,
enable_ulogfs=True,
daemon=False,
volume_mounts=None,
working_dir=None,
node_selector=None,
):
"""
Generate an Argo container template. For example, the template whalesay
in https://github.com/argoproj/argo/tree/master/examples#hello-world.
:param image:
:param command:
:param args:
:param output: output artifact for container output
:param input: input artifact for container input
:param env: environmental variable
:param secret:
:param resources: CPU or memory resource config dict
:param timeout: in seconds
:param retry: retry policy
:param step_name: used for annotating step .
:param image_pull_policy:
:param pool:
:param enable_ulogfs:
:param daemon:
:return:
"""
func_name, caller_line = utils.invocation_location()
func_name = (utils.argo_safe_name(step_name)
if step_name is not None else func_name)
if states.workflow.get_template(func_name) is None:
# Generate the inputs parameter for the template
if input is None:
input = []
if args is None and states._outputs_tmp is not None:
args = []
if args is not None:
if not isinstance(args, list):
args = [args]
# Handle case where args is a list of list type
# For example, [[Output, ]]
if (isinstance(args, list) and len(args) > 0
and isinstance(args[0], list) and len(args[0]) > 0
and isinstance(args[0][0], Output)):
args = args[0]
if states._outputs_tmp is not None:
args.extend(states._outputs_tmp)
# In case, the args include output artifact
# Place output artifact into the input
for arg in args:
if isinstance(arg, (OutputArtifact, OutputJob)):
input.append(arg)
# Automatically append emptyDir volume and volume mount to work with
# Argo k8sapi executor.
# More info: https://argoproj.github.io/argo/empty-dir/
if output is not None:
if not isinstance(output, list):
output = [output]
if volume_mounts is None:
volume_mounts = []
mounted_path = []
for i, out in enumerate(output):
if "/tmp" in out.path:
raise ValueError("Mounting to /tmp is not supported")
path_to_mount = os.path.dirname(out.path)
# Avoid duplicate mount paths
if path_to_mount not in mounted_path:
volume_mounts.append(
VolumeMount("couler-out-dir-%s" % i, path_to_mount))
mounted_path.append(path_to_mount)
# Generate container and template
template = Container(
name=func_name,
image=image,
command=command,
args=args,
env=env,
secret=states.get_secret(secret),
resources=resources,
image_pull_policy=image_pull_policy,
retry=retry,
timeout=timeout,
output=output,
input=input,
pool=pool,
enable_ulogfs=enable_ulogfs,
daemon=daemon,
volume_mounts=volume_mounts,
working_dir=working_dir,
node_selector=node_selector,
)
states.workflow.add_template(template)
step_name = step_update_utils.update_step(func_name, args, step_name,
caller_line)
# TODO: need to switch to use field `output` directly
step_templ = states.workflow.get_template(func_name)
_output = step_templ.to_dict().get("outputs", None)
_input = step_templ.to_dict().get("inputs", None)
rets = _container_output(step_name, func_name, _output)
states._steps_outputs[step_name] = rets
pb_step = proto_repr.step_repr( # noqa: F841
step_name=step_name,
tmpl_name=func_name,
image=image,
command=command,
source=None,
script_output=None,
args=args,
input=_input,
output=_output,
)
return rets
_SUBMITTER_IMPL_ENV_VAR_KEY,
ArgoSubmitter,
_SubmitterImplTypes,
)
from couler.core.templates.volume import VolumeMount
if __name__ == "__main__":
for impl_type in [_SubmitterImplTypes.PYTHON]:
os.environ[_SUBMITTER_IMPL_ENV_VAR_KEY] = impl_type
print("Submitting volume example workflow via %s implementation" %
impl_type)
couler.config_workflow(
name="volume-%s" % impl_type.lower(),
timeout=3600,
time_to_clean=3600 * 1.5,
)
# 2) Add a container to the workflow.
couler.run_container(
image="debian:latest",
command=["/bin/bash", "-c"],
args=[
' vol_found=`mount | grep /tmp` && \
if [[ -n $vol_found ]]; \
then echo "Volume mounted and found"; \
else exit -1; fi '
],
volume_mounts=[VolumeMount("apppath", "/tmp")],
)
submitter = ArgoSubmitter(namespace="argo")
couler.run(submitter=submitter)