def test_run_command_returns_1(self):
"""Test basic usage, command returns 1"""
ctx = SubProcContext('lies', ['false'])
SubProcPool.run_command(ctx)
self.assertEqual(ctx.err, '')
self.assertEqual(ctx.out, '')
self.assertEqual(ctx.ret_code, 1)
def test_run_command_returns_0(self):
"""Test basic usage, command returns 0"""
ctx = SubProcContext('truth', ['true'])
SubProcPool.run_command(ctx)
self.assertEqual(ctx.err, '')
self.assertEqual(ctx.out, '')
self.assertEqual(ctx.ret_code, 0)
def test_run_command_with_stdin_from_unicode(self):
"""Test STDIN from string with Unicode"""
ctx = SubProcContext('meow', ['cat'], stdin_str='喵\n')
SubProcPool.run_command(ctx)
self.assertEqual(ctx.err, '')
self.assertEqual(ctx.out, '喵\n')
self.assertEqual(ctx.ret_code, 0)
def test_run_command_writes_to_out(self):
"""Test basic usage, command writes to STDOUT"""
ctx = SubProcContext('parrot', ['echo', 'pirate', 'urrrr'])
SubProcPool.run_command(ctx)
self.assertEqual(ctx.err, '')
self.assertEqual(ctx.out, 'pirate urrrr\n')
self.assertEqual(ctx.ret_code, 0)
def test_run_command_writes_to_out(self):
"""Test basic usage, command writes to STDOUT"""
ctx = SubProcContext('parrot', ['echo', 'pirate', 'urrrr'])
SubProcPool.run_command(ctx)
self.assertEqual(ctx.err, '')
self.assertEqual(ctx.out, 'pirate urrrr\n')
self.assertEqual(ctx.ret_code, 0)
def test_run_command_returns_1(self):
"""Test basic usage, command returns 1"""
ctx = SubProcContext('lies', ['false'])
SubProcPool.run_command(ctx)
self.assertEqual(ctx.err, '')
self.assertEqual(ctx.out, '')
self.assertEqual(ctx.ret_code, 1)
def test_run_command_returns_0(self):
"""Test basic usage, command returns 0"""
ctx = SubProcContext('truth', ['true'])
SubProcPool.run_command(ctx)
self.assertEqual(ctx.err, '')
self.assertEqual(ctx.out, '')
self.assertEqual(ctx.ret_code, 0)
def test_run_command_with_stdin_from_unicode(self):
"""Test STDIN from string with Unicode"""
ctx = SubProcContext('meow', ['cat'], stdin_str='喵\n')
SubProcPool.run_command(ctx)
self.assertEqual(ctx.err, '')
self.assertEqual(ctx.out, '喵\n')
self.assertEqual(ctx.ret_code, 0)
def test_run_command_writes_to_err(self):
"""Test basic usage, command writes to STDERR"""
ctx = SubProcContext('parrot2',
['bash', '-c', 'echo pirate errrr >&2'])
SubProcPool.run_command(ctx)
self.assertEqual(ctx.err, 'pirate errrr\n')
self.assertEqual(ctx.out, '')
self.assertEqual(ctx.ret_code, 0)
def test_run_command_writes_to_err(self):
"""Test basic usage, command writes to STDERR"""
ctx = SubProcContext(
'parrot2', ['bash', '-c', 'echo pirate errrr >&2'])
SubProcPool.run_command(ctx)
self.assertEqual(ctx.err, 'pirate errrr\n')
self.assertEqual(ctx.out, '')
self.assertEqual(ctx.ret_code, 0)
def test__run_command_exit_add_to_badhosts(mock_ctx):
"""It updates the list of badhosts
"""
badhosts = {'foo', 'bar'}
SubProcPool._run_command_exit(mock_ctx(cmd=['ssh']),
bad_hosts=badhosts,
callback=print,
callback_args=['Welcome to Magrathea'])
assert badhosts == {'foo', 'bar', 'mouse'}
def test__run_command_exit_no_255_args(caplog, mock_ctx):
"""It runs the 255 callback with the args of the callback if no
callback 255 args provided.
"""
SubProcPool._run_command_exit(mock_ctx(cmd=['ssh', 'Zaphod']),
callback=_test_callback,
callback_args=['Zaphod'],
callback_255=_test_callback_255)
assert '255' in caplog.records[1].msg
def test__run_command_exit(caplog, mock_ctx, expect, ret_code, cmd_key):
"""It runs a callback
"""
ctx = mock_ctx(ret_code=ret_code, cmd_key=cmd_key, cmd=['ssh'])
SubProcPool._run_command_exit(ctx,
callback=_test_callback,
callback_255=_test_callback_255)
assert expect in caplog.records[0].msg
if ret_code == 255:
assert f'255 callback called.' in caplog.records[1].msg
def test_run_command_with_stdin_from_path(self):
"""Test STDIN from a single file path"""
handle = NamedTemporaryFile()
handle.write('catches mice.\n'.encode('UTF-8'))
handle.seek(0)
ctx = SubProcContext('meow', ['cat'], stdin_files=[handle.name])
SubProcPool.run_command(ctx)
self.assertEqual(ctx.err, '')
self.assertEqual(ctx.out, 'catches mice.\n')
self.assertEqual(ctx.ret_code, 0)
handle.close()
def test_run_command_with_stdin_from_path(self):
"""Test STDIN from a single file path"""
handle = NamedTemporaryFile()
handle.write('catches mice.\n'.encode('UTF-8'))
handle.seek(0)
ctx = SubProcContext('meow', ['cat'], stdin_files=[handle.name])
SubProcPool.run_command(ctx)
self.assertEqual(ctx.err, '')
self.assertEqual(ctx.out, 'catches mice.\n')
self.assertEqual(ctx.ret_code, 0)
handle.close()
def test__run_command_exit_rsync_fails(mock_ctx):
"""It updates the list of badhosts
"""
badhosts = {'foo', 'bar'}
ctx = mock_ctx(cmd=['rsync'], ret_code=42, cmd_key='file-install')
SubProcPool._run_command_exit(
ctx=ctx,
bad_hosts=badhosts,
callback=print,
callback_args=['Welcome to Magrathea', {'ssh command': 'ssh'}]
)
assert badhosts == {'foo', 'bar', 'mouse'}
def test_run_command_with_stdin_from_handles(self):
"""Test STDIN from multiple file handles"""
handles = []
for txt in ['catches mice.\n', 'eat fish.\n']:
handle = TemporaryFile()
handle.write(txt.encode('UTF-8'))
handle.seek(0)
handles.append(handle)
ctx = SubProcContext('meow', ['cat'], stdin_files=handles)
SubProcPool.run_command(ctx)
self.assertEqual(ctx.err, '')
self.assertEqual(ctx.out, 'catches mice.\neat fish.\n')
self.assertEqual(ctx.ret_code, 0)
for handle in handles:
handle.close()
def test_run_command_with_stdin_from_handles(self):
"""Test STDIN from multiple file handles"""
handles = []
for txt in ['catches mice.\n', 'eat fish.\n']:
handle = TemporaryFile()
handle.write(txt.encode('UTF-8'))
handle.seek(0)
handles.append(handle)
ctx = SubProcContext('meow', ['cat'], stdin_files=handles)
SubProcPool.run_command(ctx)
self.assertEqual(ctx.err, '')
self.assertEqual(ctx.out, 'catches mice.\neat fish.\n')
self.assertEqual(ctx.ret_code, 0)
for handle in handles:
handle.close()
def test_rsync_255_fail(mock_ctx, expect, ctx_kwargs):
"""It knows when a ctx has failed
"""
output = SubProcPool.rsync_255_fail(
mock_ctx(**ctx_kwargs),
{'ssh command': 'ssh'}
)
assert output == expect
def main(_, options, *args):
# suite name or file path
suite, suiterc = parse_suite_arg(options, args[0])
# extract task host accounts from the suite
config = SuiteConfig(
suite, suiterc, options,
load_template_vars(options.templatevars, options.templatevars_file))
account_set = set()
for name in config.get_namespace_list('all tasks'):
account_set.add(
(config.get_config(['runtime', name, 'remote', 'owner']),
config.get_config(['runtime', name, 'remote', 'host'])))
task_remote_mgr = TaskRemoteMgr(suite, SubProcPool())
for _, host_str in account_set:
task_remote_mgr.remote_host_select(host_str)
accounts = []
while account_set:
for user, host_str in account_set.copy():
res = task_remote_mgr.remote_host_select(host_str)
if res:
account_set.remove((user, host_str))
accounts.append((user, res))
if account_set:
task_remote_mgr.proc_pool.process()
sleep(1.0)
# Interrogate the each remote account with CYLC_VERSION set to our version.
# Post backward compatibility concerns to do this we can just run:
# cylc version --host=HOST --user=USER
# but this command only exists for version > 6.3.0.
# So for the moment generate an actual remote invocation command string for
# "cylc --version".
# (save verbose flag as gets reset in remrun)
verbose = cylc.flow.flags.verbose
warn = {}
contacted = 0
for user, host in sorted(accounts):
argv = ["cylc", "version"]
if user and host:
argv += ["--user=%s" % user, "--host=%s" % host]
user_at_host = "%s@%s" % (user, host)
elif user:
argv += ["--user=%s" % user]
user_at_host = "%s@localhost" % user
elif host:
argv += ["--host=%s" % host]
user_at_host = host
if verbose:
print("%s: %s" % (user_at_host, ' '.join(argv)))
proc = procopen(argv,
stdin=open(os.devnull),
stdoutpipe=True,
stderrpipe=True)
out, err = proc.communicate()
out = out.decode()
err = err.decode()
if proc.wait() == 0:
if verbose:
print(" %s" % out)
contacted += 1
out = out.strip()
if out != CYLC_VERSION:
warn[user_at_host] = out
else:
print('ERROR ' + user_at_host + ':', file=sys.stderr)
print(err, file=sys.stderr)
# report results
if not warn:
if contacted:
print("All", contacted, "accounts have cylc-" + CYLC_VERSION)
else:
print("WARNING: failed to invoke cylc-%s on %d accounts:" %
(CYLC_VERSION, len(warn)))
m = max(len(ac) for ac in warn)
for ac, warning in warn.items():
print(' ', ac.ljust(m), warning)
if options.error:
sys.exit(1)
def test_get_temporary_file(self):
"""Test SubProcPool.get_temporary_file."""
self.assertIsInstance(
SubProcPool.get_temporary_file(), SpooledTemporaryFile)
def main(parser, options, suite, *task_ids):
"""cylc submit CLI.
No TASK EVENT HOOKS are set for the submit command because there is
no scheduler instance watching for task failure etc.
Note: a suite contact env file is not written by this command (it
would overwrite the real one if the suite is running).
"""
if not options.verbose and not options.debug:
LOG.setLevel(WARNING)
for task_id in task_ids:
if not TaskID.is_valid_id(task_id):
raise UserInputError("Invalid task ID %s" % task_id)
suiterc = get_suite_rc(suite)
suite_dir = os.path.dirname(suiterc)
# For user-defined batch system handlers
sys.path.append(os.path.join(suite_dir, 'python'))
# Load suite config and tasks
config = SuiteConfig(
suite, suiterc, options,
load_template_vars(options.templatevars, options.templatevars_file))
itasks = []
for task_id in task_ids:
name_str, point_str = TaskID.split(task_id)
taskdefs = config.find_taskdefs(name_str)
if not taskdefs:
raise UserInputError("No task found for %s" % task_id)
for taskdef in taskdefs:
itasks.append(
TaskProxy(taskdef,
get_point(point_str).standardise(),
is_startup=True))
# Initialise job submit environment
make_suite_run_tree(suite)
# Extract job.sh from library, for use in job scripts.
extract_resources(get_suite_srv_dir(suite), ['etc/job.sh'])
pool = SubProcPool()
owner = get_user()
job_pool = JobPool(suite, owner)
db_mgr = SuiteDatabaseManager()
task_job_mgr = TaskJobManager(
suite, pool, db_mgr,
TaskEventsManager(suite, pool, db_mgr, BroadcastMgr(db_mgr), job_pool),
job_pool)
task_job_mgr.task_remote_mgr.single_task_mode = True
task_job_mgr.job_file_writer.set_suite_env({
'CYLC_UTC':
str(config.cfg['cylc']['UTC mode']),
'CYLC_DEBUG':
str(cylc.flow.flags.debug).lower(),
'CYLC_VERBOSE':
str(cylc.flow.flags.verbose).lower(),
'CYLC_SUITE_NAME':
suite,
'CYLC_CYCLING_MODE':
str(config.cfg['scheduling']['cycling mode']),
'CYLC_SUITE_INITIAL_CYCLE_POINT':
str(config.cfg['scheduling']['initial cycle point']),
'CYLC_SUITE_FINAL_CYCLE_POINT':
str(config.cfg['scheduling']['final cycle point']),
})
ret_code = 0
waiting_tasks = list(itasks)
if options.dry_run:
while waiting_tasks:
prep_tasks, bad_tasks = task_job_mgr.prep_submit_task_jobs(
suite, waiting_tasks, dry_run=True)
for itask in prep_tasks + bad_tasks:
waiting_tasks.remove(itask)
if waiting_tasks:
task_job_mgr.proc_pool.process()
sleep(1.0)
for itask in itasks:
if itask.local_job_file_path:
print(('JOB SCRIPT=%s' % itask.local_job_file_path))
else:
print(('Unable to prepare job file for %s' % itask.identity),
file=sys.stderr)
ret_code = 1
else:
while waiting_tasks:
for itask in task_job_mgr.submit_task_jobs(suite, waiting_tasks):
waiting_tasks.remove(itask)
if waiting_tasks:
task_job_mgr.proc_pool.process()
sleep(1.0)
while task_job_mgr.proc_pool.is_not_done():
task_job_mgr.proc_pool.process()
for itask in itasks:
if itask.summary.get('submit_method_id') is not None:
print(('[%s] Job ID: %s' %
(itask.identity, itask.summary['submit_method_id'])))
if itask.state(TASK_STATUS_SUBMIT_FAILED):
ret_code = 1
sys.exit(ret_code)
def test_get_temporary_file(self):
"""Test SubProcPool.get_temporary_file."""
self.assertIsInstance(SubProcPool.get_temporary_file(),
SpooledTemporaryFile)
def test_ssh_255_fail(mock_ctx, expect, ctx_kwargs):
"""It knows when a ctx has failed
"""
output = SubProcPool.ssh_255_fail(mock_ctx(**ctx_kwargs))
assert output == expect
def test__run_command_exit_no_255_callback(caplog, mock_ctx):
"""It runs the vanilla callback if no 255 callback provided"""
SubProcPool._run_command_exit(mock_ctx(), callback=_test_callback)
assert 'callback called' in caplog.records[0].msg
