class ToolB(Tool):
def cmd(self):
return NOOP
class ToolC(Tool):
def cmd(self):
return NOOP
cosmos = Cosmos()
cosmos.initdb()
workflow = cosmos.start("One2One", "/tmp", check_output_dir=False)
stageA_tasks = workflow.add(ToolA(params=dict(i=i)) for i in [1, 2])
stageB_tasks = workflow.add(
ToolB(params=task.params, parents=[task]) for task in stageA_tasks)
draw_task_graph(workflow.task_graph(), "one2one.png", format="png")
workflow = cosmos.start("One2Many", "/tmp", check_output_dir=False)
stageA_tasks = workflow.add(ToolA(params=dict(i=i)) for i in [1, 2])
stageB_tasks = workflow.add(
ToolB(params=dict(j=j, **task.params), parents=[task])
for task in stageA_tasks for j in ["a", "b"])
draw_task_graph(workflow.task_graph(), "one2many.png", format="png")
workflow = cosmos.start("Many2One", "/tmp", check_output_dir=False)
stageA_tasks = workflow.add(
ToolA(params=dict(i=i, j=j)) for i in [1, 2] for j in ["a", "b"])
class ToolB(Tool):
def cmd(self):
return NOOP
class ToolC(Tool):
def cmd(self):
return NOOP
cosmos = Cosmos()
cosmos.initdb()
execution = cosmos.start('One2One', '/tmp', check_output_dir=False)
stageA_tasks = execution.add(ToolA(tags=dict(i=i)) for i in [1, 2])
stageB_tasks = execution.add(
ToolB(tags=task.tags, parents=[task]) for task in stageA_tasks)
draw_task_graph(execution.task_graph(), 'one2one.png', format='png')
execution = cosmos.start('One2Many', '/tmp', check_output_dir=False)
stageA_tasks = execution.add(ToolA(tags=dict(i=i)) for i in [1, 2])
stageB_tasks = execution.add(
ToolB(tags=dict(j=j, **task.tags), parents=[task]) for task in stageA_tasks
for j in [1, 2])
draw_task_graph(execution.task_graph(), 'one2many.png', format='png')
execution = cosmos.start('Many2One', '/tmp', check_output_dir=False)
stageA_tasks = execution.add(
ToolA(tags=dict(i=i, j=j)) for i in [1, 2] for j in [1, 2])
from ex1 import run_ex1
import os
from cosmos.util.helpers import mkdir
def run_ex3(execution):
@signal_execution_status_change.connect
def sig(ex):
msg = "%s %s" % (ex, ex.status)
if ex.status in [ExecutionStatus.successful, ExecutionStatus.failed, ExecutionStatus.killed]:
text_message(msg)
ex.log.info('Sent a text message')
def text_message(message):
from twilio.rest import TwilioRestClient
account = "XYZ"
token = "XYZ"
client = TwilioRestClient(account, token)
message = client.messages.create(to="+1231231234", from_="+1231231234", body=message)
run_ex1(execution)
if __name__ == '__main__':
cosmos = Cosmos('sqlite:///%s/sqlite.db' % os.path.dirname(os.path.abspath(__file__)))
cosmos.initdb()
mkdir('out')
execution = cosmos.start('Example1', 'out/ex1', max_attempts=2, restart=True, skip_confirm=True)
run_ex1(execution)
class ToolB(Tool):
def cmd(self):
return NOOP
class ToolC(Tool):
def cmd(self):
return NOOP
cosmos = Cosmos()
cosmos.initdb()
workflow = cosmos.start('One2One', '/tmp', check_output_dir=False)
stageA_tasks = workflow.add(ToolA(params=dict(i=i))
for i in [1, 2])
stageB_tasks = workflow.add(ToolB(params=task.params, parents=[task])
for task in stageA_tasks)
draw_task_graph(workflow.task_graph(), 'one2one.png', format='png')
workflow = cosmos.start('One2Many', '/tmp', check_output_dir=False)
stageA_tasks = workflow.add(ToolA(params=dict(i=i))
for i in [1, 2])
stageB_tasks = workflow.add(ToolB(params=dict(j=j, **task.params), parents=[task])
for task in stageA_tasks
for j in ['a','b'])
draw_task_graph(workflow.task_graph(), 'one2many.png', format='png')
workflow = cosmos.start('Many2One', '/tmp', check_output_dir=False)
from cosmos import Cosmos, Input
import tools
from cosmos.util.helpers import mkdir
def run_ex2(execution):
# TODO update this
pass
# # These tasks have no dependencies
# inpts = execution.add([Input('/tmp', 'tmp_dir', 'dir', dict(test='tag'))])
# echos = execution.add([tools.Echo(dict(word='hello')), tools.Echo(tags=dict(word='world'))])
#
# # This task always fails
# fails = execution.add(tools.Fail, inpts)
#
# # Not dependent on the task that failed, will be executed
# sleeps = execution.add(tools.Sleep(dict(time=5), [inp]) for inp in inpts)
#
# # This will not run, because it depends on a task that failed
# cats = execution.add(tools.Cat(parents=[echos[0], fails[0]]))
#
# execution.run()
if __name__ == '__main__':
cosmos = Cosmos('sqlite:///sqlite.db')
cosmos.initdb()
mkdir('out_dir')
ex = cosmos.start('Failed_Task', 'out_dir/failed_task', max_attempts=2, restart=True, skip_confirm=True)
run_ex2(ex)
class ToolB(Tool):
def cmd(self):
return NOOP
class ToolC(Tool):
def cmd(self):
return NOOP
cosmos = Cosmos()
cosmos.initdb()
execution = cosmos.start('One2One', '/tmp', check_output_dir=False)
stageA_tasks = execution.add(ToolA(tags=dict(i=i))
for i in [1, 2])
stageB_tasks = execution.add(ToolB(tags=task.tags, parents=[task])
for task in stageA_tasks)
draw_task_graph(execution.task_graph(), 'one2one.png', format='png')
execution = cosmos.start('One2Many', '/tmp', check_output_dir=False)
stageA_tasks = execution.add(ToolA(tags=dict(i=i))
for i in [1, 2])
stageB_tasks = execution.add(ToolB(tags=dict(j=j, **task.tags), parents=[task])
for task in stageA_tasks
for j in ['a','b'])
draw_task_graph(execution.task_graph(), 'one2many.png', format='png')
execution = cosmos.start('Many2One', '/tmp', check_output_dir=False)
ExecutionStatus.killed
]:
text_message(msg)
ex.log.info('Sent a text message')
def text_message(message):
from twilio.rest import TwilioRestClient
account = "XYZ"
token = "XYZ"
client = TwilioRestClient(account, token)
message = client.messages.create(to="+1231231234",
from_="+1231231234",
body=message)
main(execution)
if __name__ == '__main__':
cosmos = Cosmos('sqlite:///%s/sqlite.db' %
os.path.dirname(os.path.abspath(__file__)))
cosmos.initdb()
mkdir('out')
execution = cosmos.start('Example1',
'out/ex1',
max_attempts=2,
restart=True,
skip_confirm=True)
main(execution)
growl = kwargs.pop("growl")
debug = kwargs.pop("debug")
if growl:
from cosmos.util import growl
from cosmos import signal_execution_status_change, ExecutionStatus
@signal_execution_status_change.connect
def growl_signal(execution):
if execution.status != ExecutionStatus.running:
growl.send("%s %s" % (execution, execution.status))
if func.__module__.startswith("ex"):
execution_params = {
n: kwargs.pop(n, None)
for n in ["name", "restart", "skip_confirm", "max_cpus", "max_attempts", "output_dir"]
}
if not execution_params["output_dir"]:
mkdir(os.path.join(root_path, "out"))
execution_params["output_dir"] = os.path.join(root_path, "out", execution_params["name"])
ex = cosmos.start(**execution_params)
kwargs["execution"] = ex
if debug:
import ipdb
with ipdb.launch_ipdb_on_exception():
func(**kwargs)
else:
func(**kwargs)
if growl:
from cosmos.util import growl
from cosmos import signal_execution_status_change, ExecutionStatus
@signal_execution_status_change.connect
def growl_signal(execution):
if execution.status != ExecutionStatus.running:
growl.send('%s %s' % (execution, execution.status))
if func.__name__.startswith('ex'):
execution_params = {
n: kwargs.pop(n, None)
for n in [
'name', 'restart', 'skip_confirm', 'max_cpus', 'max_attempts',
'output_dir'
]
}
if not execution_params['output_dir']:
mkdir(os.path.join(root_path, 'out'))
execution_params['output_dir'] = os.path.join(
root_path, 'out', execution_params['name'])
ex = cosmos.start(**execution_params)
kwargs['execution'] = ex
if debug:
import ipdb
with ipdb.launch_ipdb_on_exception():
func(**kwargs)
else:
func(**kwargs)
def main(execution):
# These tasks have no dependencies
inpts = execution.add([Input('/tmp', 'tmp_dir', 'dir', dict(test='tag'))])
echos = execution.add(
[tools.Echo(dict(word='hello')),
tools.Echo(tags=dict(word='world'))])
# This task always fails
fails = execution.add(tools.Fail, inpts)
# Not dependent on the task that failed, will be executed
sleeps = execution.add(tools.Sleep(dict(time=5), [inp]) for inp in inpts)
# This will not run, because it depends on a task that failed
cats = execution.add(tools.Cat(parents=[echos[0], fails[0]]))
execution.run()
if __name__ == '__main__':
cosmos = Cosmos('sqlite:///sqlite.db')
cosmos.initdb()
mkdir('out')
ex = cosmos.start('Failed_Task',
'out/failed_task',
max_attempts=2,
restart=True,
skip_confirm=True)
main(ex)
