def make_output_dirs(self): """ Create directory paths of all output files """ dirs = set() for task in self.tasks: for out_name, v in task.output_map.items(): dirname = lambda p: p if out_name.endswith('dir') or p is None else os.path.dirname(p) if isinstance(v, (tuple, list)): dirs.update(map(dirname, v)) elif isinstance(v, dict): raise NotImplemented() else: dirs.add(dirname(v)) for d in dirs: if d is not None and '://' not in d: mkdir(d)
def make_output_dirs(self): """ Create directory paths of all output files """ dirs = set() for task in self.tasks: for out_name, v in task.output_map.iteritems(): dirname = lambda p: p if out_name.endswith('dir') or p is None else os.path.dirname(p) if isinstance(v, (tuple, list)): dirs.update(map(dirname, v)) elif isinstance(v, dict): raise NotImplemented() else: dirs.add(dirname(v)) for d in dirs: if d is not None and '://' not in d: mkdir(d)
def submit_task(self, task, command): task.log_dir = self.log_out_dir_func(task) for p in [task.output_stdout_path, task.output_stderr_path, task.output_command_script_path]: if os.path.exists(p): os.unlink(p) if command is NOOP: task.NOOP = True if task.NOOP: task.status = TaskStatus.submitted return else: mkdir(task.log_dir) _create_command_sh(task, command) task.drm_native_specification = self.get_submit_args(task) assert task.drm is not None, 'task has no drm set' self.get_drm(task.drm).submit_job(task)
def prepare_task_for_submission(self, task, command): task.log_dir = self.log_out_dir_func(task) for p in [ task.output_stdout_path, task.output_stderr_path, task.output_command_script_path, ]: if os.path.exists(p): os.unlink(p) if command is NOOP: task.NOOP = True if task.NOOP: task.status = TaskStatus.submitted return else: mkdir(task.log_dir) _create_command_sh(task, command) task.drm_native_specification = self.get_submit_args(task) assert task.drm is not None, "task has no drm set"
from ex1 import run_ex1 import os from cosmos.util.helpers import mkdir def run_ex3(execution): @signal_execution_status_change.connect def sig(ex): msg = "%s %s" % (ex, ex.status) if ex.status in [ExecutionStatus.successful, ExecutionStatus.failed, ExecutionStatus.killed]: text_message(msg) ex.log.info('Sent a text message') def text_message(message): from twilio.rest import TwilioRestClient account = "XYZ" token = "XYZ" client = TwilioRestClient(account, token) message = client.messages.create(to="+1231231234", from_="+1231231234", body=message) run_ex1(execution) if __name__ == '__main__': cosmos = Cosmos('sqlite:///%s/sqlite.db' % os.path.dirname(os.path.abspath(__file__))) cosmos.initdb() mkdir('out') execution = cosmos.start('Example1', 'out/ex1', max_attempts=2, restart=True, skip_confirm=True) run_ex1(execution)
from cosmos import Cosmos, Input import tools from cosmos.util.helpers import mkdir def run_ex2(execution): # TODO update this pass # # These tasks have no dependencies # inpts = execution.add([Input('/tmp', 'tmp_dir', 'dir', dict(test='tag'))]) # echos = execution.add([tools.Echo(dict(word='hello')), tools.Echo(tags=dict(word='world'))]) # # # This task always fails # fails = execution.add(tools.Fail, inpts) # # # Not dependent on the task that failed, will be executed # sleeps = execution.add(tools.Sleep(dict(time=5), [inp]) for inp in inpts) # # # This will not run, because it depends on a task that failed # cats = execution.add(tools.Cat(parents=[echos[0], fails[0]])) # # execution.run() if __name__ == '__main__': cosmos = Cosmos('sqlite:///sqlite.db') cosmos.initdb() mkdir('out_dir') ex = cosmos.start('Failed_Task', 'out_dir/failed_task', max_attempts=2, restart=True, skip_confirm=True) run_ex2(ex)
args = parser.parse_args() kwargs = dict(args._get_kwargs()) func = kwargs.pop('func') growl = kwargs.pop('growl') debug = kwargs.pop('debug') if growl: from cosmos.util import growl from cosmos.api import signal_execution_status_change, ExecutionStatus @signal_execution_status_change.connect def growl_signal(execution): if execution.status != ExecutionStatus.running: growl.send('%s %s' % (execution, execution.status)) if func.__module__.startswith('t'): execution_params = {n: kwargs.pop(n, None) for n in ['name', 'restart', 'skip_confirm', 'max_cores', 'max_attempts', 'output_dir']} if not execution_params['output_dir']: mkdir(os.path.join(root_path, 'out_dir')) execution_params['output_dir'] = os.path.join(root_path, 'out_dir', execution_params['name']) ex = cosmos.start(**execution_params) kwargs['execution'] = ex if debug: import ipdb with ipdb.launch_ipdb_on_exception(): func(**kwargs) else: func(**kwargs)
from cosmos.util import growl from cosmos.api import signal_execution_status_change, ExecutionStatus @signal_execution_status_change.connect def growl_signal(execution): if execution.status != ExecutionStatus.running: growl.send('%s %s' % (execution, execution.status)) if func.__module__.startswith('ex'): execution_params = { n: kwargs.pop(n, None) for n in [ 'name', 'restart', 'skip_confirm', 'max_cpus', 'max_attempts', 'output_dir' ] } if not execution_params['output_dir']: mkdir(os.path.join(root_path, 'out_dir')) execution_params['output_dir'] = os.path.join( root_path, 'out_dir', execution_params['name']) ex = cosmos.start(**execution_params) kwargs['execution'] = ex if debug: import ipdb with ipdb.launch_ipdb_on_exception(): func(**kwargs) else: func(**kwargs)
ExecutionStatus.killed ]: text_message(msg) ex.log.info('Sent a text message') def text_message(message): from twilio.rest import TwilioRestClient account = "XYZ" token = "XYZ" client = TwilioRestClient(account, token) message = client.messages.create(to="+1231231234", from_="+1231231234", body=message) main(execution) if __name__ == '__main__': cosmos = Cosmos('sqlite:///%s/sqlite.db' % os.path.dirname(os.path.abspath(__file__))) cosmos.initdb() mkdir('out') execution = cosmos.start('Example1', 'out/ex1', max_attempts=2, restart=True, skip_confirm=True) main(execution)
growl = kwargs.pop("growl") debug = kwargs.pop("debug") if growl: from cosmos.util import growl from cosmos import signal_execution_status_change, ExecutionStatus @signal_execution_status_change.connect def growl_signal(execution): if execution.status != ExecutionStatus.running: growl.send("%s %s" % (execution, execution.status)) if func.__module__.startswith("ex"): execution_params = { n: kwargs.pop(n, None) for n in ["name", "restart", "skip_confirm", "max_cpus", "max_attempts", "output_dir"] } if not execution_params["output_dir"]: mkdir(os.path.join(root_path, "out")) execution_params["output_dir"] = os.path.join(root_path, "out", execution_params["name"]) ex = cosmos.start(**execution_params) kwargs["execution"] = ex if debug: import ipdb with ipdb.launch_ipdb_on_exception(): func(**kwargs) else: func(**kwargs)