def pipeline_with_sidecar(sleep_sec: int = 120):
# sidecar with sevice that reply "hello world" to any GET request
echo = dsl.Sidecar(
name="echo",
image="hashicorp/http-echo:latest",
args=['-text="hello world"'],
)
# container op with sidecar
op1 = dsl.ContainerOp(
name="download",
image="busybox:latest",
command=["sh", "-c"],
arguments=[
"sleep %s; wget localhost:5678 -O /tmp/results.txt" % sleep_sec
], # sleep for X sec and call the sidecar and save results to output
sidecars=[echo],
file_outputs={"downloaded": "/tmp/results.txt"},
)
op2 = dsl.ContainerOp(
name="echo",
image="library/bash",
command=["sh", "-c"],
arguments=["echo %s" % op1.output], # print out content of op1 output
)
def pipelineparams_pipeline(tag: str = 'latest', sleep_ms: int = 10):
echo = dsl.Sidecar(
name='echo',
image='hashicorp/http-echo:%s' % tag,
args=['-text="hello world"'],
)
op1 = dsl.ContainerOp(
name='download',
image='busybox:%s' % tag,
command=['sh', '-c'],
arguments=[
'sleep %s; wget localhost:5678 -O /tmp/results.txt' % sleep_ms
],
sidecars=[echo],
file_outputs={'downloaded': '/tmp/results.txt'})
op2 = dsl.ContainerOp(name='echo',
image='library/bash',
command=['sh', '-c'],
arguments=['echo $MSG %s' % op1.output])
op2.container.add_env_variable(
V1EnvVar(name='MSG', value='pipelineParams: '))
def serve_sidecar():
"""Serves tensorflow model as sidecar to testing container."""
return dsl.Sidecar(
name='tensorflow-serve',
image='tensorflow/serving:1.14.0',
command='/usr/bin/tensorflow_model_server',
args=[
'--model_name=mnist',
'--model_base_path=/output/mnist',
'--port=9000',
'--rest_api_port=9001',
],
mirror_volume_mounts=True,
)
def pipelineparams_pipeline(tag: str = 'latest', sleep_ms: int = 10):
echo = dsl.Sidecar(
name='echo',
image='hashicorp/http-echo:%s' % tag,
args=['-text="hello world"'],
)
op1 = components.load_component_from_text("""
name: download
description: download
inputs:
- {name: sleep_ms, type: Integer}
outputs:
- {name: data, type: String}
implementation:
container:
image: busy:placeholder
command:
- sh
- -c
args:
- |
sleep $0; wget localhost:5678 -O $1
- {inputValue: sleep_ms}
- {outputPath: data}
""")(sleep_ms)
op1.container.image = "busy:%s" % tag
op2 = components.load_component_from_text("""
name: echo
description: echo
inputs:
- {name: message, type: String}
implementation:
container:
image: library/bash
command:
- sh
- -c
args:
- |
echo $MSG $0
- {inputValue: message}
""")(op1.output)
op2.container.add_env_variable(V1EnvVar(name='MSG', value='pipelineParams: '))
def sidecar_pipeline():
echo = dsl.Sidecar(name='echo',
image='hashicorp/http-echo',
args=['-text="hello world"'])
op1 = dsl.ContainerOp(
name='download',
image='busybox',
command=['sh', '-c'],
arguments=['sleep 10; wget localhost:5678 -O /tmp/results.txt'],
sidecars=[echo],
file_outputs={'downloaded': '/tmp/results.txt'})
op2 = dsl.ContainerOp(name='echo',
image='library/bash',
command=['sh', '-c'],
arguments=['echo %s' % op1.output])
def pipeline_with_sidecar():
# sidecar with sevice that reply "hello world" to any GET request
echo = dsl.Sidecar(
name="echo",
image="nginx:1.13",
command=["nginx", "-g", "daemon off;"],
)
# container op with sidecar
op1 = dsl.ContainerOp(
name="download",
image="busybox:latest",
command=["sh", "-c"],
arguments=[
"until wget http://localhost:80 -O /tmp/results.txt; do sleep 5; done && cat /tmp/results.txt"
],
sidecars=[echo],
file_outputs={"downloaded": "/tmp/results.txt"},
)
def sidecar_pipeline():
echo = dsl.Sidecar(name='echo',
image='hashicorp/http-echo',
args=['-text="hello world"'])
op1 = components.load_component_from_text("""
name: download
description: download
outputs:
- {name: download, type: String}
implementation:
container:
image: busybox
command:
- sh
- -c
args:
- |
sleep 10; wget localhost:5678 -O $0
- {outputPath: download}
""")()
op2 = components.load_component_from_text("""
name: echo
description: echo
inputs:
- {name: msg, type: String}
implementation:
container:
image: library/bash
command:
- sh
- -c
args:
- echo
- {inputValue: msg}
""")(msg=op1.output)
def serve_sidecar():
"""Serves tensorflow model as sidecar to testing container."""
# See https://github.com/argoproj/argo/issues/1570 for why we
# need the wrapper script
code = """\
import os
import signal
import subprocess
import time
p = subprocess.Popen([
'/usr/bin/tensorflow_model_server',
'--model_name=mnist',
'--model_base_path=/output/mnist',
'--port=9000',
'--rest_api_port=9001',
])
start = time.time()
while p.poll() is None:
# Kill process after 30 seconds due to https://github.com/argoproj/argo/issues/1570
if time.time() - start >= 30:
print('Waited long enough for tests to complete.')
os.kill(p.pid, signal.SIGKILL)
break
time.sleep(0.1)
"""
return dsl.Sidecar(
name='tensorflow-serve',
image='tensorflow/serving:1.14.0',
command='sh',
args=['-c', f'apt update && apt install -y python3 && python3 -c "{code}"'],
mirror_volume_mounts=True,
)
