def test_run_command_returns_1(self): """Test basic usage, command returns 1""" ctx = SubProcContext('lies', ['false']) SubProcPool.run_command(ctx) self.assertEqual(ctx.err, '') self.assertEqual(ctx.out, '') self.assertEqual(ctx.ret_code, 1)
def test_run_command_returns_0(self): """Test basic usage, command returns 0""" ctx = SubProcContext('truth', ['true']) SubProcPool.run_command(ctx) self.assertEqual(ctx.err, '') self.assertEqual(ctx.out, '') self.assertEqual(ctx.ret_code, 0)
def test_run_command_with_stdin_from_unicode(self): """Test STDIN from string with Unicode""" ctx = SubProcContext('meow', ['cat'], stdin_str='ĺ–µ\n') SubProcPool.run_command(ctx) self.assertEqual(ctx.err, '') self.assertEqual(ctx.out, 'ĺ–µ\n') self.assertEqual(ctx.ret_code, 0)
def test_run_command_writes_to_out(self): """Test basic usage, command writes to STDOUT""" ctx = SubProcContext('parrot', ['echo', 'pirate', 'urrrr']) SubProcPool.run_command(ctx) self.assertEqual(ctx.err, '') self.assertEqual(ctx.out, 'pirate urrrr\n') self.assertEqual(ctx.ret_code, 0)
def test_run_command_writes_to_out(self): """Test basic usage, command writes to STDOUT""" ctx = SubProcContext('parrot', ['echo', 'pirate', 'urrrr']) SubProcPool.run_command(ctx) self.assertEqual(ctx.err, '') self.assertEqual(ctx.out, 'pirate urrrr\n') self.assertEqual(ctx.ret_code, 0)
def test_run_command_returns_1(self): """Test basic usage, command returns 1""" ctx = SubProcContext('lies', ['false']) SubProcPool.run_command(ctx) self.assertEqual(ctx.err, '') self.assertEqual(ctx.out, '') self.assertEqual(ctx.ret_code, 1)
def test_run_command_returns_0(self): """Test basic usage, command returns 0""" ctx = SubProcContext('truth', ['true']) SubProcPool.run_command(ctx) self.assertEqual(ctx.err, '') self.assertEqual(ctx.out, '') self.assertEqual(ctx.ret_code, 0)
def test_run_command_with_stdin_from_unicode(self): """Test STDIN from string with Unicode""" ctx = SubProcContext('meow', ['cat'], stdin_str='ĺ–µ\n') SubProcPool.run_command(ctx) self.assertEqual(ctx.err, '') self.assertEqual(ctx.out, 'ĺ–µ\n') self.assertEqual(ctx.ret_code, 0)
def test_run_command_writes_to_err(self): """Test basic usage, command writes to STDERR""" ctx = SubProcContext('parrot2', ['bash', '-c', 'echo pirate errrr >&2']) SubProcPool.run_command(ctx) self.assertEqual(ctx.err, 'pirate errrr\n') self.assertEqual(ctx.out, '') self.assertEqual(ctx.ret_code, 0)
def test_run_command_writes_to_err(self): """Test basic usage, command writes to STDERR""" ctx = SubProcContext( 'parrot2', ['bash', '-c', 'echo pirate errrr >&2']) SubProcPool.run_command(ctx) self.assertEqual(ctx.err, 'pirate errrr\n') self.assertEqual(ctx.out, '') self.assertEqual(ctx.ret_code, 0)
def test__run_command_exit_add_to_badhosts(mock_ctx): """It updates the list of badhosts """ badhosts = {'foo', 'bar'} SubProcPool._run_command_exit(mock_ctx(cmd=['ssh']), bad_hosts=badhosts, callback=print, callback_args=['Welcome to Magrathea']) assert badhosts == {'foo', 'bar', 'mouse'}
def test__run_command_exit_no_255_args(caplog, mock_ctx): """It runs the 255 callback with the args of the callback if no callback 255 args provided. """ SubProcPool._run_command_exit(mock_ctx(cmd=['ssh', 'Zaphod']), callback=_test_callback, callback_args=['Zaphod'], callback_255=_test_callback_255) assert '255' in caplog.records[1].msg
def test__run_command_exit(caplog, mock_ctx, expect, ret_code, cmd_key): """It runs a callback """ ctx = mock_ctx(ret_code=ret_code, cmd_key=cmd_key, cmd=['ssh']) SubProcPool._run_command_exit(ctx, callback=_test_callback, callback_255=_test_callback_255) assert expect in caplog.records[0].msg if ret_code == 255: assert f'255 callback called.' in caplog.records[1].msg
def test_run_command_with_stdin_from_path(self): """Test STDIN from a single file path""" handle = NamedTemporaryFile() handle.write('catches mice.\n'.encode('UTF-8')) handle.seek(0) ctx = SubProcContext('meow', ['cat'], stdin_files=[handle.name]) SubProcPool.run_command(ctx) self.assertEqual(ctx.err, '') self.assertEqual(ctx.out, 'catches mice.\n') self.assertEqual(ctx.ret_code, 0) handle.close()
def test_run_command_with_stdin_from_path(self): """Test STDIN from a single file path""" handle = NamedTemporaryFile() handle.write('catches mice.\n'.encode('UTF-8')) handle.seek(0) ctx = SubProcContext('meow', ['cat'], stdin_files=[handle.name]) SubProcPool.run_command(ctx) self.assertEqual(ctx.err, '') self.assertEqual(ctx.out, 'catches mice.\n') self.assertEqual(ctx.ret_code, 0) handle.close()
def test__run_command_exit_rsync_fails(mock_ctx): """It updates the list of badhosts """ badhosts = {'foo', 'bar'} ctx = mock_ctx(cmd=['rsync'], ret_code=42, cmd_key='file-install') SubProcPool._run_command_exit( ctx=ctx, bad_hosts=badhosts, callback=print, callback_args=['Welcome to Magrathea', {'ssh command': 'ssh'}] ) assert badhosts == {'foo', 'bar', 'mouse'}
def test_run_command_with_stdin_from_handles(self): """Test STDIN from multiple file handles""" handles = [] for txt in ['catches mice.\n', 'eat fish.\n']: handle = TemporaryFile() handle.write(txt.encode('UTF-8')) handle.seek(0) handles.append(handle) ctx = SubProcContext('meow', ['cat'], stdin_files=handles) SubProcPool.run_command(ctx) self.assertEqual(ctx.err, '') self.assertEqual(ctx.out, 'catches mice.\neat fish.\n') self.assertEqual(ctx.ret_code, 0) for handle in handles: handle.close()
def test_run_command_with_stdin_from_handles(self): """Test STDIN from multiple file handles""" handles = [] for txt in ['catches mice.\n', 'eat fish.\n']: handle = TemporaryFile() handle.write(txt.encode('UTF-8')) handle.seek(0) handles.append(handle) ctx = SubProcContext('meow', ['cat'], stdin_files=handles) SubProcPool.run_command(ctx) self.assertEqual(ctx.err, '') self.assertEqual(ctx.out, 'catches mice.\neat fish.\n') self.assertEqual(ctx.ret_code, 0) for handle in handles: handle.close()
def test_rsync_255_fail(mock_ctx, expect, ctx_kwargs): """It knows when a ctx has failed """ output = SubProcPool.rsync_255_fail( mock_ctx(**ctx_kwargs), {'ssh command': 'ssh'} ) assert output == expect
def main(_, options, *args): # suite name or file path suite, suiterc = parse_suite_arg(options, args[0]) # extract task host accounts from the suite config = SuiteConfig( suite, suiterc, options, load_template_vars(options.templatevars, options.templatevars_file)) account_set = set() for name in config.get_namespace_list('all tasks'): account_set.add( (config.get_config(['runtime', name, 'remote', 'owner']), config.get_config(['runtime', name, 'remote', 'host']))) task_remote_mgr = TaskRemoteMgr(suite, SubProcPool()) for _, host_str in account_set: task_remote_mgr.remote_host_select(host_str) accounts = [] while account_set: for user, host_str in account_set.copy(): res = task_remote_mgr.remote_host_select(host_str) if res: account_set.remove((user, host_str)) accounts.append((user, res)) if account_set: task_remote_mgr.proc_pool.process() sleep(1.0) # Interrogate the each remote account with CYLC_VERSION set to our version. # Post backward compatibility concerns to do this we can just run: # cylc version --host=HOST --user=USER # but this command only exists for version > 6.3.0. # So for the moment generate an actual remote invocation command string for # "cylc --version". # (save verbose flag as gets reset in remrun) verbose = cylc.flow.flags.verbose warn = {} contacted = 0 for user, host in sorted(accounts): argv = ["cylc", "version"] if user and host: argv += ["--user=%s" % user, "--host=%s" % host] user_at_host = "%s@%s" % (user, host) elif user: argv += ["--user=%s" % user] user_at_host = "%s@localhost" % user elif host: argv += ["--host=%s" % host] user_at_host = host if verbose: print("%s: %s" % (user_at_host, ' '.join(argv))) proc = procopen(argv, stdin=open(os.devnull), stdoutpipe=True, stderrpipe=True) out, err = proc.communicate() out = out.decode() err = err.decode() if proc.wait() == 0: if verbose: print(" %s" % out) contacted += 1 out = out.strip() if out != CYLC_VERSION: warn[user_at_host] = out else: print('ERROR ' + user_at_host + ':', file=sys.stderr) print(err, file=sys.stderr) # report results if not warn: if contacted: print("All", contacted, "accounts have cylc-" + CYLC_VERSION) else: print("WARNING: failed to invoke cylc-%s on %d accounts:" % (CYLC_VERSION, len(warn))) m = max(len(ac) for ac in warn) for ac, warning in warn.items(): print(' ', ac.ljust(m), warning) if options.error: sys.exit(1)
def test_get_temporary_file(self): """Test SubProcPool.get_temporary_file.""" self.assertIsInstance( SubProcPool.get_temporary_file(), SpooledTemporaryFile)
def main(parser, options, suite, *task_ids): """cylc submit CLI. No TASK EVENT HOOKS are set for the submit command because there is no scheduler instance watching for task failure etc. Note: a suite contact env file is not written by this command (it would overwrite the real one if the suite is running). """ if not options.verbose and not options.debug: LOG.setLevel(WARNING) for task_id in task_ids: if not TaskID.is_valid_id(task_id): raise UserInputError("Invalid task ID %s" % task_id) suiterc = get_suite_rc(suite) suite_dir = os.path.dirname(suiterc) # For user-defined batch system handlers sys.path.append(os.path.join(suite_dir, 'python')) # Load suite config and tasks config = SuiteConfig( suite, suiterc, options, load_template_vars(options.templatevars, options.templatevars_file)) itasks = [] for task_id in task_ids: name_str, point_str = TaskID.split(task_id) taskdefs = config.find_taskdefs(name_str) if not taskdefs: raise UserInputError("No task found for %s" % task_id) for taskdef in taskdefs: itasks.append( TaskProxy(taskdef, get_point(point_str).standardise(), is_startup=True)) # Initialise job submit environment make_suite_run_tree(suite) # Extract job.sh from library, for use in job scripts. extract_resources(get_suite_srv_dir(suite), ['etc/job.sh']) pool = SubProcPool() owner = get_user() job_pool = JobPool(suite, owner) db_mgr = SuiteDatabaseManager() task_job_mgr = TaskJobManager( suite, pool, db_mgr, TaskEventsManager(suite, pool, db_mgr, BroadcastMgr(db_mgr), job_pool), job_pool) task_job_mgr.task_remote_mgr.single_task_mode = True task_job_mgr.job_file_writer.set_suite_env({ 'CYLC_UTC': str(config.cfg['cylc']['UTC mode']), 'CYLC_DEBUG': str(cylc.flow.flags.debug).lower(), 'CYLC_VERBOSE': str(cylc.flow.flags.verbose).lower(), 'CYLC_SUITE_NAME': suite, 'CYLC_CYCLING_MODE': str(config.cfg['scheduling']['cycling mode']), 'CYLC_SUITE_INITIAL_CYCLE_POINT': str(config.cfg['scheduling']['initial cycle point']), 'CYLC_SUITE_FINAL_CYCLE_POINT': str(config.cfg['scheduling']['final cycle point']), }) ret_code = 0 waiting_tasks = list(itasks) if options.dry_run: while waiting_tasks: prep_tasks, bad_tasks = task_job_mgr.prep_submit_task_jobs( suite, waiting_tasks, dry_run=True) for itask in prep_tasks + bad_tasks: waiting_tasks.remove(itask) if waiting_tasks: task_job_mgr.proc_pool.process() sleep(1.0) for itask in itasks: if itask.local_job_file_path: print(('JOB SCRIPT=%s' % itask.local_job_file_path)) else: print(('Unable to prepare job file for %s' % itask.identity), file=sys.stderr) ret_code = 1 else: while waiting_tasks: for itask in task_job_mgr.submit_task_jobs(suite, waiting_tasks): waiting_tasks.remove(itask) if waiting_tasks: task_job_mgr.proc_pool.process() sleep(1.0) while task_job_mgr.proc_pool.is_not_done(): task_job_mgr.proc_pool.process() for itask in itasks: if itask.summary.get('submit_method_id') is not None: print(('[%s] Job ID: %s' % (itask.identity, itask.summary['submit_method_id']))) if itask.state(TASK_STATUS_SUBMIT_FAILED): ret_code = 1 sys.exit(ret_code)
def test_get_temporary_file(self): """Test SubProcPool.get_temporary_file.""" self.assertIsInstance(SubProcPool.get_temporary_file(), SpooledTemporaryFile)
def test_ssh_255_fail(mock_ctx, expect, ctx_kwargs): """It knows when a ctx has failed """ output = SubProcPool.ssh_255_fail(mock_ctx(**ctx_kwargs)) assert output == expect
def test__run_command_exit_no_255_callback(caplog, mock_ctx): """It runs the vanilla callback if no 255 callback provided""" SubProcPool._run_command_exit(mock_ctx(), callback=_test_callback) assert 'callback called' in caplog.records[0].msg