def make_output_dirs(self):
"""
Create directory paths of all output files
"""
dirs = set()
for task in self.tasks:
for out_name, v in task.output_map.items():
dirname = lambda p: p if out_name.endswith('dir') or p is None else os.path.dirname(p)
if isinstance(v, (tuple, list)):
dirs.update(map(dirname, v))
elif isinstance(v, dict):
raise NotImplemented()
else:
dirs.add(dirname(v))
for d in dirs:
if d is not None and '://' not in d:
mkdir(d)
def make_output_dirs(self):
"""
Create directory paths of all output files
"""
dirs = set()
for task in self.tasks:
for out_name, v in task.output_map.iteritems():
dirname = lambda p: p if out_name.endswith('dir') or p is None else os.path.dirname(p)
if isinstance(v, (tuple, list)):
dirs.update(map(dirname, v))
elif isinstance(v, dict):
raise NotImplemented()
else:
dirs.add(dirname(v))
for d in dirs:
if d is not None and '://' not in d:
mkdir(d)
def submit_task(self, task, command):
task.log_dir = self.log_out_dir_func(task)
for p in [task.output_stdout_path, task.output_stderr_path, task.output_command_script_path]:
if os.path.exists(p):
os.unlink(p)
if command is NOOP:
task.NOOP = True
if task.NOOP:
task.status = TaskStatus.submitted
return
else:
mkdir(task.log_dir)
_create_command_sh(task, command)
task.drm_native_specification = self.get_submit_args(task)
assert task.drm is not None, 'task has no drm set'
self.get_drm(task.drm).submit_job(task)
def prepare_task_for_submission(self, task, command):
task.log_dir = self.log_out_dir_func(task)
for p in [
task.output_stdout_path,
task.output_stderr_path,
task.output_command_script_path,
]:
if os.path.exists(p):
os.unlink(p)
if command is NOOP:
task.NOOP = True
if task.NOOP:
task.status = TaskStatus.submitted
return
else:
mkdir(task.log_dir)
_create_command_sh(task, command)
task.drm_native_specification = self.get_submit_args(task)
assert task.drm is not None, "task has no drm set"
from ex1 import run_ex1
import os
from cosmos.util.helpers import mkdir
def run_ex3(execution):
@signal_execution_status_change.connect
def sig(ex):
msg = "%s %s" % (ex, ex.status)
if ex.status in [ExecutionStatus.successful, ExecutionStatus.failed, ExecutionStatus.killed]:
text_message(msg)
ex.log.info('Sent a text message')
def text_message(message):
from twilio.rest import TwilioRestClient
account = "XYZ"
token = "XYZ"
client = TwilioRestClient(account, token)
message = client.messages.create(to="+1231231234", from_="+1231231234", body=message)
run_ex1(execution)
if __name__ == '__main__':
cosmos = Cosmos('sqlite:///%s/sqlite.db' % os.path.dirname(os.path.abspath(__file__)))
cosmos.initdb()
mkdir('out')
execution = cosmos.start('Example1', 'out/ex1', max_attempts=2, restart=True, skip_confirm=True)
run_ex1(execution)
from cosmos import Cosmos, Input
import tools
from cosmos.util.helpers import mkdir
def run_ex2(execution):
# TODO update this
pass
# # These tasks have no dependencies
# inpts = execution.add([Input('/tmp', 'tmp_dir', 'dir', dict(test='tag'))])
# echos = execution.add([tools.Echo(dict(word='hello')), tools.Echo(tags=dict(word='world'))])
#
# # This task always fails
# fails = execution.add(tools.Fail, inpts)
#
# # Not dependent on the task that failed, will be executed
# sleeps = execution.add(tools.Sleep(dict(time=5), [inp]) for inp in inpts)
#
# # This will not run, because it depends on a task that failed
# cats = execution.add(tools.Cat(parents=[echos[0], fails[0]]))
#
# execution.run()
if __name__ == '__main__':
cosmos = Cosmos('sqlite:///sqlite.db')
cosmos.initdb()
mkdir('out_dir')
ex = cosmos.start('Failed_Task', 'out_dir/failed_task', max_attempts=2, restart=True, skip_confirm=True)
run_ex2(ex)
args = parser.parse_args()
kwargs = dict(args._get_kwargs())
func = kwargs.pop('func')
growl = kwargs.pop('growl')
debug = kwargs.pop('debug')
if growl:
from cosmos.util import growl
from cosmos.api import signal_execution_status_change, ExecutionStatus
@signal_execution_status_change.connect
def growl_signal(execution):
if execution.status != ExecutionStatus.running:
growl.send('%s %s' % (execution, execution.status))
if func.__module__.startswith('t'):
execution_params = {n: kwargs.pop(n, None) for n in
['name', 'restart', 'skip_confirm', 'max_cores', 'max_attempts', 'output_dir']}
if not execution_params['output_dir']:
mkdir(os.path.join(root_path, 'out_dir'))
execution_params['output_dir'] = os.path.join(root_path, 'out_dir', execution_params['name'])
ex = cosmos.start(**execution_params)
kwargs['execution'] = ex
if debug:
import ipdb
with ipdb.launch_ipdb_on_exception():
func(**kwargs)
else:
func(**kwargs)
from cosmos.util import growl
from cosmos.api import signal_execution_status_change, ExecutionStatus
@signal_execution_status_change.connect
def growl_signal(execution):
if execution.status != ExecutionStatus.running:
growl.send('%s %s' % (execution, execution.status))
if func.__module__.startswith('ex'):
execution_params = {
n: kwargs.pop(n, None)
for n in [
'name', 'restart', 'skip_confirm', 'max_cpus', 'max_attempts',
'output_dir'
]
}
if not execution_params['output_dir']:
mkdir(os.path.join(root_path, 'out_dir'))
execution_params['output_dir'] = os.path.join(
root_path, 'out_dir', execution_params['name'])
ex = cosmos.start(**execution_params)
kwargs['execution'] = ex
if debug:
import ipdb
with ipdb.launch_ipdb_on_exception():
func(**kwargs)
else:
func(**kwargs)
ExecutionStatus.killed
]:
text_message(msg)
ex.log.info('Sent a text message')
def text_message(message):
from twilio.rest import TwilioRestClient
account = "XYZ"
token = "XYZ"
client = TwilioRestClient(account, token)
message = client.messages.create(to="+1231231234",
from_="+1231231234",
body=message)
main(execution)
if __name__ == '__main__':
cosmos = Cosmos('sqlite:///%s/sqlite.db' %
os.path.dirname(os.path.abspath(__file__)))
cosmos.initdb()
mkdir('out')
execution = cosmos.start('Example1',
'out/ex1',
max_attempts=2,
restart=True,
skip_confirm=True)
main(execution)
growl = kwargs.pop("growl")
debug = kwargs.pop("debug")
if growl:
from cosmos.util import growl
from cosmos import signal_execution_status_change, ExecutionStatus
@signal_execution_status_change.connect
def growl_signal(execution):
if execution.status != ExecutionStatus.running:
growl.send("%s %s" % (execution, execution.status))
if func.__module__.startswith("ex"):
execution_params = {
n: kwargs.pop(n, None)
for n in ["name", "restart", "skip_confirm", "max_cpus", "max_attempts", "output_dir"]
}
if not execution_params["output_dir"]:
mkdir(os.path.join(root_path, "out"))
execution_params["output_dir"] = os.path.join(root_path, "out", execution_params["name"])
ex = cosmos.start(**execution_params)
kwargs["execution"] = ex
if debug:
import ipdb
with ipdb.launch_ipdb_on_exception():
func(**kwargs)
else:
func(**kwargs)