def run(submitter=None):
"""To submit the workflow using user-provided submitter implementation.
Note that, the provided submitter must have a submit function which
takes the workflow YAML as input.
"""
states._enable_print_yaml = False
if submitter is None and ArgoSubmitter._default_submitter is None:
raise ValueError(
"The input submitter is None and default submitter was not set."
)
wf = workflow_yaml()
secrets = states._secrets.values()
validate_workflow_yaml(wf)
if submitter is not None:
if isinstance(submitter, ArgoSubmitter):
res = submitter.submit(wf, secrets=secrets)
elif issubclass(submitter, ArgoSubmitter):
submitter = ArgoSubmitter()
res = submitter.submit(wf, secrets=secrets)
else:
raise ValueError("Only ArgoSubmitter is supported currently.")
else:
res = ArgoSubmitter._default_submitter.submit(wf, secrets=secrets)
# Clean up the saved states of the workflow since we made a copy of
# the workflow above and no longer need the original reference. This
# would also allow users to define a new workflow after submission.
_cleanup()
return res
def run(submitter=ArgoSubmitter):
"""To submit the workflow using user-provided submitter implementation.
Note that, the provided submitter must have a submit function which
takes the workflow YAML as input.
"""
states._enable_print_yaml = False
if submitter is None:
raise ValueError("The input submitter is None")
wf = workflow_yaml()
secrets = states._secrets.values()
validate_workflow_yaml(wf)
if isinstance(submitter, ArgoSubmitter):
return submitter.submit(wf, secrets=secrets)
elif issubclass(submitter, ArgoSubmitter):
submitter = ArgoSubmitter()
return submitter.submit(wf, secrets=secrets)
else:
raise ValueError("Only ArgoSubmitter is supported currently.")
def run(submitter=ArgoSubmitter):
"""To submit the workflow using user-provided submitter implementation.
Note that, the provided submitter must have a submit function which
takes the workflow YAML as input.
"""
states._enable_print_yaml = False
if submitter is None:
raise ValueError("The input submitter is None")
wf = workflow_yaml()
# TODO (terrytangyuan): Decouple with Argo and then uncomment this
# validate_workflow_yaml(wf)
if isinstance(submitter, ArgoSubmitter):
return submitter.submit(wf)
elif issubclass(submitter, ArgoSubmitter):
submitter = ArgoSubmitter()
return submitter.submit(wf)
else:
raise ValueError("Only ArgoSubmitter is supported currently.")
def dispatch(
*,
nice_id,
client_parameters,
workflow_object,
server_url,
):
executor_parameters = {
'workflow_name': 'openbio-' + nice_id,
'image_registry': client_parameters['image_registry'],
'work_path': os.path.join(client_parameters['work_path'], nice_id)
}
namespace = client_parameters['namespace']
# Setup bash scripts
args = type('A', (), {
'server': server_url,
'insecure': False,
})
setup_bash_patterns(args)
# Parse cytoscape workflow
w = Workflow(workflow_object=workflow_object,
askinput='NO',
obc_server=server_url,
workflow_id=None)
# Create argo scripts
e = ArgoExecutor(w, executor_parameters)
e.build(output=None,
output_format='argo',
workflow_id=None,
obc_client=server_url)
# Submit with couler
submitter = ArgoSubmitter(namespace=namespace)
result = couler.run(submitter=submitter)
# Get visualization url
visualization_url = urllib.parse.urlparse(
client_parameters['argo_url'])._replace(
path='/workflows/%s/%s' %
(namespace, result['metadata']['name'])).geturl()
return {
'visualization_url': visualization_url,
}
def flip_coin():
return couler.run_script(image="python:alpine3.6", source=random_code)
def heads():
return couler.run_container(image="alpine:3.6",
command=["sh", "-c", 'echo "it was heads"'])
def tails():
return couler.run_container(image="alpine:3.6",
command=["sh", "-c", 'echo "it was tails"'])
if __name__ == "__main__":
for impl_type in [_SubmitterImplTypes.GO, _SubmitterImplTypes.PYTHON]:
os.environ[_SUBMITTER_IMPL_ENV_VAR_KEY] = impl_type
print("Submitting flip coin example workflow via %s implementation" %
impl_type)
couler.config_workflow(
name="flip-coin-%s" % impl_type.lower(),
timeout=3600,
time_to_clean=3600 * 1.5,
)
result = flip_coin()
couler.when(couler.equal(result, "heads"), lambda: heads())
couler.when(couler.equal(result, "tails"), lambda: tails())
submitter = ArgoSubmitter(namespace="argo")
couler.run(submitter=submitter)
# A
# / \
# B C
# /
# D
def linear():
couler.set_dependencies(lambda: job_a(message="A"), dependencies=None)
couler.set_dependencies(lambda: job_b(message="B"), dependencies=["A"])
couler.set_dependencies(lambda: job_c(message="C"), dependencies=["A"])
couler.set_dependencies(lambda: job_d(message="D"), dependencies=["B"])
# A
# / \
# B C
# \ /
# D
def diamond():
couler.dag([
[lambda: job_a(message="A")],
[lambda: job_a(message="A"), lambda: job_b(message="B")], # A -> B
[lambda: job_a(message="A"), lambda: job_c(message="C")], # A -> C
[lambda: job_b(message="B"), lambda: job_d(message="D")], # B -> D
[lambda: job_c(message="C"), lambda: job_d(message="D")], # C -> D
])
linear()
submitter = ArgoSubmitter()
couler.run(submitter=submitter)
import couler.argo as couler
from couler.argo_submitter import ArgoSubmitter
submitter = ArgoSubmitter(namespace="my_namespace")
# Instead of setting up and passing a submitter each time,
# you can just set it as default!
couler.set_default_submitter(submitter)
couler.run_container(
image="docker/whalesay",
command=["cowsay"],
args=["hello world with default submitter"],
)
# no submitter need be passed now!
couler.run()
couler.run_container(
image="docker/whalesay",
command=["cowsay"],
args=["yet another hello world with default submitter"],
)
# You can still pass a submitter to use,
# overriding the default submitter if you want to
couler.run(ArgoSubmitter(namespace="other_namespace"))
