def pipeline(my_pipe_param: int = 10):
loop_args = [{'a': 1, 'b': 2}, {'a': 10, 'b': 20}]
with dsl.ParallelFor(loop_args) as item:
op1 = dsl.ContainerOp(
name="my-in-coop1",
image="library/bash:4.4.23",
command=["sh", "-c"],
arguments=["echo op1 %s %s" % (item.a, my_pipe_param)],
)
with dsl.ParallelFor([100, 200, 300]) as inner_item:
op11 = dsl.ContainerOp(
name="my-inner-inner-coop",
image="library/bash:4.4.23",
command=["sh", "-c"],
arguments=[
"echo op1 %s %s %s" % (item.a, inner_item, my_pipe_param)
],
)
op2 = dsl.ContainerOp(
name="my-in-coop2",
image="library/bash:4.4.23",
command=["sh", "-c"],
arguments=["echo op2 %s" % item.b],
)
op_out = dsl.ContainerOp(
name="my-out-cop",
image="library/bash:4.4.23",
command=["sh", "-c"],
arguments=["echo %s" % my_pipe_param],
)
def pipeline():
dump_loop_args_op = dump_loop_args()
with dsl.SubGraph(parallelism=2):
with dsl.ParallelFor(dump_loop_args_op.output) as item:
print_op(item)
print_op(item.A_a)
print_op(item.B_b)
def pipeline(loopidy_doop: dict = [{'a': 1, 'b': 2}, {'a': 10, 'b': 20}]):
op0 = dsl.ContainerOp(
name="my-out-cop0",
image='python:alpine3.6',
command=["sh", "-c"],
arguments=[
'python -c "import json; import sys; json.dump([i for i in range(20, 31)], open(\'/tmp/out.json\', \'w\'))"'
],
file_outputs={'out': '/tmp/out.json'},
)
with dsl.ParallelFor(loopidy_doop) as item:
op1 = dsl.ContainerOp(
name="my-in-cop1",
image="library/bash:4.4.23",
command=["sh", "-c"],
arguments=["echo no output global op1, item.a: %s" % item.a],
).after(op0)
op_out = dsl.ContainerOp(
name="my-out-cop2",
image="library/bash:4.4.23",
command=["sh", "-c"],
arguments=["echo no output global op2, outp: %s" % op0.output],
)
def pipeline():
op0 = dsl.ContainerOp(
name="my-out-cop0",
image='python:alpine3.6',
command=["sh", "-c"],
arguments=[
'python -c "import json; import sys; json.dump([{\'a\': 1, \'b\': 2}, {\'a\': 10, \'b\': 20}], open(\'/tmp/out.json\', \'w\'))"'
],
file_outputs={'out': '/tmp/out.json'},
)
with dsl.ParallelFor(op0.output) as item:
op1 = dsl.ContainerOp(
name="my-in-cop1",
image="library/bash:4.4.23",
command=["sh", "-c"],
arguments=["echo do output op1 item.a: %s" % item.a],
)
op_out = dsl.ContainerOp(
name="my-out-cop2",
image="library/bash:4.4.23",
command=["sh", "-c"],
arguments=["echo do output op2, outp: %s" % op0.output],
)
def pipeline():
loop_args = [{'A_a': 1, 'B_b': 2}, {'A_a': 10, 'B_b': 20}]
with dsl.SubGraph(parallelism=2):
with dsl.ParallelFor(loop_args) as item:
print_op(item)
print_op(item.A_a)
print_op(item.B_b)
def my_pipeline(
greeting: str = 'this is a test for looping through parameters'):
print_task = print_op(text=greeting)
generate_task = generate_op()
with dsl.ParallelFor(generate_task.output) as item:
concat_task = concat_op(a=item.a, b=item.b)
concat_task.after(print_task)
print_task_2 = print_op(concat_task.output)
def my_pipeline(
static_loop_arguments: List[dict] = _DEFAULT_LOOP_ARGUMENTS,
greeting: str = 'this is a test for looping through parameters',
):
print_task = print_op(text=greeting)
with dsl.ParallelFor(static_loop_arguments) as item:
concat_task = concat_op(a=item.a, b=item.b)
concat_task.after(print_task)
print_task_2 = print_op(concat_task.output)
def uri_artifact(text: str = 'Hello world!'):
task_1 = write_to_gcs(text=text)
task_2 = read_from_gcs(input_gcs_path=task_1.outputs['output_gcs_path'])
# Test use URI within ParFor loop.
loop_args = [1, 2, 3, 4]
with dsl.ParallelFor(loop_args) as loop_arg:
loop_task_2 = read_from_gcs(
input_gcs_path=task_1.outputs['output_gcs_path'])
# Test use URI within condition.
flip = flip_coin_op()
with dsl.Condition(flip.output == 'heads'):
condition_task_2 = read_from_gcs(
input_gcs_path=task_1.outputs['output_gcs_path'])
def my_pipeline():
args_generator = args_generator_op()
with dsl.ParallelFor(args_generator.output) as item:
print_op(item)
def pipeline():
source_task = produce_op()
with dsl.ParallelFor(source_task.output) as item:
consume_op(item)