def testTaskSignature(self):
'''Test re-execution of tasks'''
with cd_new('temp_signature'):
with open('test.sos', 'w') as tst:
tst.write('''
task:
sh:
sleep 2
''')
subprocess.call('sos run test -s force', shell=True)
tasks = get_tasks()
tf = TaskFile(tasks[0])
self.assertTrue(tf.has_signature())
self.assertEqual(tf.status, 'completed')
info = tf.tags_created_start_and_duration()
#
subprocess.call('sos run test', shell=True)
self.assertLess(tf.tags_created_start_and_duration()[3], 1)
def test_task_signature(self):
"""Test re-execution of tasks"""
with cd_new("temp_signature"):
with open("test.sos", "w") as tst:
tst.write(
"""
task:
sh:
sleep 2
"""
)
subprocess.call("sos run test -s force -q localhost", shell=True)
tasks = get_tasks()
tf = TaskFile(tasks[0])
self.assertTrue(tf.has_signature())
self.assertEqual(tf.status, "completed")
tf.tags_created_start_and_duration()
#
subprocess.call("sos run test -q localhost", shell=True)
self.assertLess(tf.tags_created_start_and_duration()[3], 1)
def test_task_file(self):
"""Test task file handling"""
for ext in (".pulse", ".out", ".err", ".task", ".sh"):
filename = os.path.join(
os.path.expanduser("~"), ".sos", "tasks", "ffffffffffffffff" + ext
)
if os.path.isfile(filename):
os.remove(filename)
params = TaskParams(
name="ffffffffffffffff",
global_def={},
task="b=a",
sos_dict={"a": 1},
tags=["b", "a"],
)
a = TaskFile("ffffffffffffffff")
a.save(params)
self.assertEqual(a.tags, "a b")
for ext in (".pulse", ".out", ".err", ".sh"):
with open(
os.path.join(
os.path.expanduser("~"), ".sos", "tasks", "ffffffffffffffff" + ext
),
"w",
) as fh:
fh.write(ext)
self.assertFalse(a.has_stdout())
self.assertFalse(a.has_stderr())
a.add_outputs()
#
self.assertEqual(a.params.sos_dict["a"], 1)
self.assertEqual(a.status, "new")
a.status = "completed"
self.assertLess(time.time() - a.last_updated, 2)
self.assertEqual(a.status, "completed")
#
# get and reset info
info = a.info
a.status = "running"
self.assertEqual(a.status, "running")
a.info = info
self.assertEqual(a.status, "completed")
self.assertTrue(a.has_stdout())
#
a.add_result({"ret_code": 5})
#
a.tags = ["ee", "d"]
self.assertEqual(a.tags, "d ee")
# a.add_tags(['kk'])
# self.assertEqual(a.tags.split(), ['d', 'ee', 'kk'])
#
self.assertEqual(a.params.sos_dict["a"], 1)
self.assertEqual(a.params.task, "b=a")
#
self.assertEqual(a.stdout, ".out")
self.assertEqual(a.stderr, ".err")
self.assertEqual(a.pulse, ".pulse")
self.assertEqual(a.shell, ".sh")
self.assertTrue(a.has_stdout())
self.assertTrue(a.has_stderr())
self.assertTrue(a.has_pulse())
self.assertTrue(a.has_shell())
#
#
a.reset()
self.assertEqual(a.status, "new")
self.assertEqual(a.stdout, "")
self.assertEqual(a.stderr, "")
self.assertEqual(a.signature, {})
a.add_outputs()
a.add_result({"ret_code": 5})
self.assertEqual(a.result["ret_code"], 5)
def testTaskFile(self):
'''Test task file handling'''
for ext in ('.pulse', '.out', '.err', '.task', '.sh'):
filename = os.path.join(os.path.expanduser('~'), '.sos', 'tasks',
'ffffffffffffffff' + ext)
if os.path.isfile(filename):
os.remove(filename)
params = TaskParams(name='ffffffffffffffff',
global_def={},
task='b=a',
sos_dict={'a': 1},
tags=['b', 'a'])
a = TaskFile('ffffffffffffffff')
a.save(params)
self.assertEqual(a.tags, 'a b')
for ext in ('.pulse', '.out', '.err', '.sh'):
with open(
os.path.join(os.path.expanduser('~'), '.sos', 'tasks',
'ffffffffffffffff' + ext), 'w') as fh:
fh.write(ext)
self.assertFalse(a.has_stdout())
self.assertFalse(a.has_stderr())
a.add_outputs()
#
self.assertEqual(a.params.sos_dict['a'], 1)
self.assertEqual(a.status, 'new')
a.status = 'completed'
self.assertLess(time.time() - a.last_updated, 2)
self.assertEqual(a.status, 'completed')
#
# get and reset info
info = a.info
a.status = 'running'
self.assertEqual(a.status, 'running')
a.info = info
self.assertEqual(a.status, 'completed')
self.assertTrue(a.has_stdout())
#
a.add_result({'ret_code': 5})
#
a.tags = ['ee', 'd']
self.assertEqual(a.tags, 'd ee')
#a.add_tags(['kk'])
#self.assertEqual(a.tags.split(), ['d', 'ee', 'kk'])
#
self.assertEqual(a.params.sos_dict['a'], 1)
self.assertEqual(a.params.task, 'b=a')
#
self.assertEqual(a.stdout, '.out')
self.assertEqual(a.stderr, '.err')
self.assertEqual(a.pulse, '.pulse')
self.assertEqual(a.shell, '.sh')
self.assertTrue(a.has_stdout())
self.assertTrue(a.has_stderr())
self.assertTrue(a.has_pulse())
self.assertTrue(a.has_shell())
#
#
a.reset()
self.assertEqual(a.status, 'new')
self.assertEqual(a.stdout, '')
self.assertEqual(a.stderr, '')
self.assertEqual(a.signature, {})
a.add_outputs()
a.add_result({'ret_code': 5})
self.assertEqual(a.result['ret_code'], 5)
def _prepare_script(self, task_id):
# read the task file and look for runtime info
#
params = TaskFile(task_id).params
sos_dict = params.sos_dict
# for this task, we will need walltime, nodes, cores, mem
# however, these could be fixed in the job template and we do not need to have them all in the runtime
runtime = self.config
# we also use saved verbosity and sig_mode because the current sig_mode might have been changed
# (e.g. in Jupyter) after the job is saved.
runtime.update({
x: sos_dict['_runtime'][x]
for x in ('nodes', 'cores', 'mem', 'walltime', 'cur_dir',
'home_dir', 'verbosity', 'sig_mode', 'run_mode')
if x in sos_dict['_runtime']
})
if 'name' in sos_dict['_runtime']:
env.logger.warning(
"Runtime option name is deprecated. Please use tags to keep track of task names."
)
runtime['task'] = task_id
# this is also deprecated
runtime['job_name'] = task_id
if 'nodes' not in runtime:
runtime['nodes'] = 1
if 'cores' not in runtime:
runtime['cores'] = 1
# for backward compatibility
runtime['job_file'] = f'~/.sos/tasks/{task_id}.sh'
# let us first prepare a task file
try:
job_text = cfg_interpolate(self.job_template, runtime)
except Exception as e:
raise ValueError(
f'Failed to generate job file for task {task_id}: {e}')
# now we need to write a job file
job_file = os.path.join(os.path.expanduser('~'), '.sos', 'tasks',
task_id + '.sh')
# do not translate newline under windows because the script will be executed
# under linux/mac
with open(job_file, 'w', newline='') as job:
job.write(job_text)
# then copy the job file to remote host if necessary
self.agent.send_task_file(job_file)
if runtime['run_mode'] == 'dryrun':
try:
cmd = f'bash ~/.sos/tasks/{task_id}.sh'
print(self.agent.check_output(cmd))
except Exception as e:
raise RuntimeError(f'Failed to submit task {task_id}: {e}')
else:
#
# now we need to figure out a command to submit the task
try:
cmd = cfg_interpolate(self.submit_cmd, runtime)
except Exception as e:
raise ValueError(
f'Failed to generate job submission command from template "{self.submit_cmd}": {e}'
)
env.logger.debug(f'submit {task_id}: {cmd}')
try:
try:
cmd_output = self.agent.check_output(cmd).strip()
except Exception as e:
raise RuntimeError(f'Failed to submit task {task_id}: {e}')
if not cmd_output:
raise RuntimeError(
f'Failed to submit task {task_id} with command {cmd}. No output returned.'
)
if 'submit_cmd_output' not in self.config:
submit_cmd_output = '{job_id}'
else:
submit_cmd_output = self.config['submit_cmd_output']
#
if not '{job_id}' in submit_cmd_output:
raise ValueError(
f'Option submit_cmd_output should have at least a pattern for job_id, "{submit_cmd_output}" specified.'
)
#
# try to extract job_id from command output
# let us write an job_id file so that we can check status of tasks more easily
job_id_file = os.path.join(os.path.expanduser('~'), '.sos',
'tasks', task_id + '.job_id')
with open(job_id_file, 'w') as job:
res = extract_pattern(submit_cmd_output,
[cmd_output.strip()])
if 'job_id' not in res or len(
res['job_id']) != 1 or res['job_id'][0] is None:
raise RuntimeError(
f'Failed to extract job_id from "{cmd_output.strip()}" using pattern "{submit_cmd_output}"'
)
else:
job_id = res['job_id'][0]
# other variables
for k, v in res.items():
job.write(f'{k}: {v[0]}\n')
# Send job id files to remote host so that
# 1. the job could be properly killed (with job_id) on remote host (not remotely)
# 2. the job status could be perperly probed in case the job was not properly submitted (#911)
self.agent.send_task_file(job_id_file)
# output job id to stdout
env.logger.info(
f'{task_id} ``submitted`` to {self.alias} with job id {job_id}'
)
return True
except Exception as e:
raise RuntimeError(f'Failed to submit task {task_id}: {e}')