class ToolB(Tool): def cmd(self): return NOOP class ToolC(Tool): def cmd(self): return NOOP cosmos = Cosmos() cosmos.initdb() workflow = cosmos.start("One2One", "/tmp", check_output_dir=False) stageA_tasks = workflow.add(ToolA(params=dict(i=i)) for i in [1, 2]) stageB_tasks = workflow.add( ToolB(params=task.params, parents=[task]) for task in stageA_tasks) draw_task_graph(workflow.task_graph(), "one2one.png", format="png") workflow = cosmos.start("One2Many", "/tmp", check_output_dir=False) stageA_tasks = workflow.add(ToolA(params=dict(i=i)) for i in [1, 2]) stageB_tasks = workflow.add( ToolB(params=dict(j=j, **task.params), parents=[task]) for task in stageA_tasks for j in ["a", "b"]) draw_task_graph(workflow.task_graph(), "one2many.png", format="png") workflow = cosmos.start("Many2One", "/tmp", check_output_dir=False) stageA_tasks = workflow.add( ToolA(params=dict(i=i, j=j)) for i in [1, 2] for j in ["a", "b"])
class ToolB(Tool): def cmd(self): return NOOP class ToolC(Tool): def cmd(self): return NOOP cosmos = Cosmos() cosmos.initdb() execution = cosmos.start('One2One', '/tmp', check_output_dir=False) stageA_tasks = execution.add(ToolA(tags=dict(i=i)) for i in [1, 2]) stageB_tasks = execution.add( ToolB(tags=task.tags, parents=[task]) for task in stageA_tasks) draw_task_graph(execution.task_graph(), 'one2one.png', format='png') execution = cosmos.start('One2Many', '/tmp', check_output_dir=False) stageA_tasks = execution.add(ToolA(tags=dict(i=i)) for i in [1, 2]) stageB_tasks = execution.add( ToolB(tags=dict(j=j, **task.tags), parents=[task]) for task in stageA_tasks for j in [1, 2]) draw_task_graph(execution.task_graph(), 'one2many.png', format='png') execution = cosmos.start('Many2One', '/tmp', check_output_dir=False) stageA_tasks = execution.add( ToolA(tags=dict(i=i, j=j)) for i in [1, 2] for j in [1, 2])
from ex1 import run_ex1 import os from cosmos.util.helpers import mkdir def run_ex3(execution): @signal_execution_status_change.connect def sig(ex): msg = "%s %s" % (ex, ex.status) if ex.status in [ExecutionStatus.successful, ExecutionStatus.failed, ExecutionStatus.killed]: text_message(msg) ex.log.info('Sent a text message') def text_message(message): from twilio.rest import TwilioRestClient account = "XYZ" token = "XYZ" client = TwilioRestClient(account, token) message = client.messages.create(to="+1231231234", from_="+1231231234", body=message) run_ex1(execution) if __name__ == '__main__': cosmos = Cosmos('sqlite:///%s/sqlite.db' % os.path.dirname(os.path.abspath(__file__))) cosmos.initdb() mkdir('out') execution = cosmos.start('Example1', 'out/ex1', max_attempts=2, restart=True, skip_confirm=True) run_ex1(execution)
class ToolB(Tool): def cmd(self): return NOOP class ToolC(Tool): def cmd(self): return NOOP cosmos = Cosmos() cosmos.initdb() workflow = cosmos.start('One2One', '/tmp', check_output_dir=False) stageA_tasks = workflow.add(ToolA(params=dict(i=i)) for i in [1, 2]) stageB_tasks = workflow.add(ToolB(params=task.params, parents=[task]) for task in stageA_tasks) draw_task_graph(workflow.task_graph(), 'one2one.png', format='png') workflow = cosmos.start('One2Many', '/tmp', check_output_dir=False) stageA_tasks = workflow.add(ToolA(params=dict(i=i)) for i in [1, 2]) stageB_tasks = workflow.add(ToolB(params=dict(j=j, **task.params), parents=[task]) for task in stageA_tasks for j in ['a','b']) draw_task_graph(workflow.task_graph(), 'one2many.png', format='png') workflow = cosmos.start('Many2One', '/tmp', check_output_dir=False)
from cosmos import Cosmos, Input import tools from cosmos.util.helpers import mkdir def run_ex2(execution): # TODO update this pass # # These tasks have no dependencies # inpts = execution.add([Input('/tmp', 'tmp_dir', 'dir', dict(test='tag'))]) # echos = execution.add([tools.Echo(dict(word='hello')), tools.Echo(tags=dict(word='world'))]) # # # This task always fails # fails = execution.add(tools.Fail, inpts) # # # Not dependent on the task that failed, will be executed # sleeps = execution.add(tools.Sleep(dict(time=5), [inp]) for inp in inpts) # # # This will not run, because it depends on a task that failed # cats = execution.add(tools.Cat(parents=[echos[0], fails[0]])) # # execution.run() if __name__ == '__main__': cosmos = Cosmos('sqlite:///sqlite.db') cosmos.initdb() mkdir('out_dir') ex = cosmos.start('Failed_Task', 'out_dir/failed_task', max_attempts=2, restart=True, skip_confirm=True) run_ex2(ex)
class ToolB(Tool): def cmd(self): return NOOP class ToolC(Tool): def cmd(self): return NOOP cosmos = Cosmos() cosmos.initdb() execution = cosmos.start('One2One', '/tmp', check_output_dir=False) stageA_tasks = execution.add(ToolA(tags=dict(i=i)) for i in [1, 2]) stageB_tasks = execution.add(ToolB(tags=task.tags, parents=[task]) for task in stageA_tasks) draw_task_graph(execution.task_graph(), 'one2one.png', format='png') execution = cosmos.start('One2Many', '/tmp', check_output_dir=False) stageA_tasks = execution.add(ToolA(tags=dict(i=i)) for i in [1, 2]) stageB_tasks = execution.add(ToolB(tags=dict(j=j, **task.tags), parents=[task]) for task in stageA_tasks for j in ['a','b']) draw_task_graph(execution.task_graph(), 'one2many.png', format='png') execution = cosmos.start('Many2One', '/tmp', check_output_dir=False)
ExecutionStatus.killed ]: text_message(msg) ex.log.info('Sent a text message') def text_message(message): from twilio.rest import TwilioRestClient account = "XYZ" token = "XYZ" client = TwilioRestClient(account, token) message = client.messages.create(to="+1231231234", from_="+1231231234", body=message) main(execution) if __name__ == '__main__': cosmos = Cosmos('sqlite:///%s/sqlite.db' % os.path.dirname(os.path.abspath(__file__))) cosmos.initdb() mkdir('out') execution = cosmos.start('Example1', 'out/ex1', max_attempts=2, restart=True, skip_confirm=True) main(execution)
growl = kwargs.pop("growl") debug = kwargs.pop("debug") if growl: from cosmos.util import growl from cosmos import signal_execution_status_change, ExecutionStatus @signal_execution_status_change.connect def growl_signal(execution): if execution.status != ExecutionStatus.running: growl.send("%s %s" % (execution, execution.status)) if func.__module__.startswith("ex"): execution_params = { n: kwargs.pop(n, None) for n in ["name", "restart", "skip_confirm", "max_cpus", "max_attempts", "output_dir"] } if not execution_params["output_dir"]: mkdir(os.path.join(root_path, "out")) execution_params["output_dir"] = os.path.join(root_path, "out", execution_params["name"]) ex = cosmos.start(**execution_params) kwargs["execution"] = ex if debug: import ipdb with ipdb.launch_ipdb_on_exception(): func(**kwargs) else: func(**kwargs)
if growl: from cosmos.util import growl from cosmos import signal_execution_status_change, ExecutionStatus @signal_execution_status_change.connect def growl_signal(execution): if execution.status != ExecutionStatus.running: growl.send('%s %s' % (execution, execution.status)) if func.__name__.startswith('ex'): execution_params = { n: kwargs.pop(n, None) for n in [ 'name', 'restart', 'skip_confirm', 'max_cpus', 'max_attempts', 'output_dir' ] } if not execution_params['output_dir']: mkdir(os.path.join(root_path, 'out')) execution_params['output_dir'] = os.path.join( root_path, 'out', execution_params['name']) ex = cosmos.start(**execution_params) kwargs['execution'] = ex if debug: import ipdb with ipdb.launch_ipdb_on_exception(): func(**kwargs) else: func(**kwargs)
def main(execution): # These tasks have no dependencies inpts = execution.add([Input('/tmp', 'tmp_dir', 'dir', dict(test='tag'))]) echos = execution.add( [tools.Echo(dict(word='hello')), tools.Echo(tags=dict(word='world'))]) # This task always fails fails = execution.add(tools.Fail, inpts) # Not dependent on the task that failed, will be executed sleeps = execution.add(tools.Sleep(dict(time=5), [inp]) for inp in inpts) # This will not run, because it depends on a task that failed cats = execution.add(tools.Cat(parents=[echos[0], fails[0]])) execution.run() if __name__ == '__main__': cosmos = Cosmos('sqlite:///sqlite.db') cosmos.initdb() mkdir('out') ex = cosmos.start('Failed_Task', 'out/failed_task', max_attempts=2, restart=True, skip_confirm=True) main(ex)