def setUp(self):
env.reset()
subprocess.call('sos remove -s', shell=True)
#self.resetDir('~/.sos')
self.temp_files = []
self.resetDir('temp')
Host.reset()
def setUp(self):
env.reset()
#self.resetDir('~/.sos')
self.temp_files = []
Host.reset()
# remove .status file left by failed workflows.
subprocess.call('sos purge', shell=True)
def submit_tasks(self, tasks):
if not tasks:
return
if self.host is None:
if 'queue' in env.sos_dict['_runtime']:
queue = env.sos_dict['_runtime']['queue']
elif env.config['default_queue']:
queue = env.config['default_queue']
else:
queue = 'localhost'
self.host = Host(queue)
for task in tasks:
self.host.submit_task(task)
def testRemoteExecution(self):
subprocess.check_output('sos purge', shell=True).decode()
script = SoS_Script('''
[10]
input: for_each={'i': range(5)}
task:
run: expand=True
echo I am {i}
sleep {5+i}
''')
wf = script.workflow()
res = Base_Executor(wf, config={
'config_file': '~/docker.yml',
# do not wait for jobs
'wait_for_task': False,
'default_queue': 'docker',
'max_running_jobs': 5,
'sig_mode': 'force',
}).run()
import time
# we should be able to get status
tasks = ' '.join(res['pending_tasks'])
# wait another 15 seconds?
time.sleep(15)
out = subprocess.check_output('sos status {} -c ~/docker.yml -q docker'.format(tasks), shell=True).decode()
self.assertEqual(out.count('completed'), len(res['pending_tasks']), 'Expect all completed jobs: ' + out)
Host.reset()
# until we run the workflow again
#st = time.time()
Base_Executor(wf, config={
'config_file': '~/docker.yml',
# do not wait for jobs
'wait_for_task': True,
'default_queue': 'docker',
'resume_mode': True,
}).run()
# should finish relatively fast?
#self.assertLess(time.time() - st, 5)
out = subprocess.check_output('sos status {} -c ~/docker.yml'.format(tasks), shell=True).decode()
self.assertEqual(out.count('completed'), len(res['pending_tasks']), 'Expect all completed jobs: ' + out)
class Interactive_Step_Executor(Base_Step_Executor):
def __init__(self, step, mode='interactive'):
super(Interactive_Step_Executor, self).__init__(step)
self.run_mode = mode
self.host = None
def init_input_output_vars(self):
# we keep these variables (which can be result of stepping through previous statements)
# if no input and/or output statement is defined
for key in ('step_input', '_depends', 'step_output', 'step_depends',
'_depends'):
if key not in env.sos_dict:
env.sos_dict.set(key, sos_targets([]))
if '_output' not in env.sos_dict:
env.sos_dict.set('_output', sos_targets(_undetermined=True))
if any(x[0] == ':' and x[1] == 'input' for x in self.step.statements):
env.sos_dict.set('step_input', sos_targets([]))
env.sos_dict.set('_input', sos_targets([]))
if any(x[0] == ':' and x[1] == 'output' for x in self.step.statements):
env.sos_dict.set('step_output', sos_targets([]))
env.sos_dict.set('_output', sos_targets([]))
env.sos_dict.pop('__default_output__', None)
def submit_tasks(self, tasks):
if not tasks:
return
if self.host is None:
if 'queue' in env.sos_dict['_runtime']:
queue = env.sos_dict['_runtime']['queue']
elif env.config['default_queue']:
queue = env.config['default_queue']
else:
queue = 'localhost'
self.host = Host(queue)
for task in tasks:
self.host.submit_task(task)
def wait_for_tasks(self, tasks, all_submitted):
if not tasks:
return {}
# when we wait, the "outsiders" also need to see the tags etc
# of the tasks so we have to write to the database. #156
env.master_push_socket.send_pyobj(['commit_sig'])
# turn this function to a generator to satisfy the interface, but do not
# actually wait for any socket.
yield None
# wait till the executor responde
if all(x == 'completed' for x in self.host.check_status(tasks)):
if len(tasks) > 4:
print('HINT: {} task{} completed: {}, {}, ..., {}'.format(
len(tasks), 's' if len(tasks) > 1 else '',
f"""<a onclick="task_info('{tasks[0]}', '{self.host.alias}')">{tasks[0][:4]}</a>""",
f"""<a onclick="task_info('{tasks[1]}', '{self.host.alias}')">{tasks[1][:4]}</a>""",
f"""<a onclick="task_info('{tasks[-1]}', '{self.host.alias}')">{tasks[-1][:4]}</a>"""
))
else:
print('HINT: {} task{} completed: {}'.format(
len(tasks), 's' if len(tasks) > 1 else '', ','.join([
f"""<a onclick="task_info('{x}', '{self.host.alias}')">{x[:4]}</a>"""
for x in tasks
])))
return self.host.retrieve_results(tasks)
res = self.host.check_status(tasks)
if all(x not in ('submitted', 'pending', 'running') for x in res):
# completed = [task for task, status in zip(tasks, res) if status == 'completed']
return self.host.retrieve_results(tasks)
raise TerminateExecution('Terminate with Running Tasks')
def run(self):
try:
runner = Base_Step_Executor.run(self)
yreq = next(runner)
while True:
yreq = runner.send(yreq)
except StopIteration as e:
return e.value
def log(self, stage=None, msg=None):
if stage == 'start':
env.logger.debug('{} ``{}``: {}'.format(
'Checking' if self.run_mode == 'dryrun' else 'Executing',
self.step.step_name(), self.step.comment.strip()))
elif stage == 'input':
if env.sos_dict['step_input'] is not None:
env.logger.debug('input: ``{}``'.format(
short_repr(env.sos_dict['step_input'])))
elif stage == 'output':
if env.sos_dict['step_output'] is not None:
env.logger.debug('output: ``{}``'.format(
short_repr(env.sos_dict['step_output'])))
def wait_for_subworkflows(self, workflow_results):
'''Wait for results from subworkflows'''
raise RuntimeError(
'Nested workflow is not supported in interactive mode')
def handle_unknown_target(self, e):
# wait for the clearnce of unknown target
yield None
raise e
def verify_dynamic_targets(self, targets):
raise RuntimeError(
'Dynamic targets are not supported in interative mode')
def testActiveActionOption(self):
'''Test the active option of actions'''
# disallow
self.assertRaises(
ParsingError, SoS_Script, '''
[1]
rep = range(5)
input: for_each = 'rep'
# ff should change and be usable inside run
ff = f"{_rep}.txt"
run: expand=True, active=1,2
echo {ff}
touch temp/{ff}
''')
#
for active, result in [
('0', ['temp/0.txt']),
('-1', ['temp/4.txt']),
('(1,2)', ['temp/1.txt', 'temp/2.txt']),
('[2,3]', ['temp/2.txt', 'temp/3.txt']),
('(0,2,4)', ['temp/0.txt', 'temp/2.txt', 'temp/4.txt']),
('slice(1,None)',
['temp/1.txt', 'temp/2.txt', 'temp/3.txt', 'temp/4.txt']),
('slice(1,-2)', ['temp/1.txt', 'temp/2.txt']),
('slice(None,None,2)', ['temp/0.txt', 'temp/2.txt', 'temp/4.txt']),
]:
if os.path.isdir('temp'):
shutil.rmtree('temp')
os.mkdir('temp')
# test first iteration
script = SoS_Script(('''
[1]
rep = range(5)
input: for_each = 'rep'
# ff should change and be usable inside run
ff = f"{_rep}.txt"
run: expand=True, active=%s
echo {ff}
touch temp/{ff}
''' % active).replace('/', os.sep))
wf = script.workflow()
env.config['sig_mode'] = 'force'
env.config['wait_for_task'] = True
Host.reset()
Base_Executor(wf).run()
files = list(glob.glob(os.path.join('temp', '*.txt')))
self.assertEqual(sorted(files),
sorted([x.replace('/', os.sep) for x in result]))
#
# test last iteration
shutil.rmtree('temp')
#
# test active option for task
os.mkdir('temp')
script = SoS_Script(('''
[1]
rep = range(5)
input: for_each = 'rep'
# ff should change and be usable inside run
ff = f"{_rep}.txt"
task: active=%s
run: expand=True
echo {ff}
touch temp/{ff}
''' % active).replace('/', os.sep))
wf = script.workflow()
env.config['sig_mode'] = 'force'
env.config['wait_for_task'] = True
Host.reset()
Base_Executor(wf).run()
files = list(glob.glob(os.path.join('temp', '*.txt')))
self.assertEqual(sorted(files),
sorted([x.replace('/', os.sep) for x in result]),
'With option {}'.format(active))
#
# test last iteration
shutil.rmtree('temp')
def runfile(script=None,
raw_args='',
wdir='.',
code=None,
kernel=None,
**kwargs):
# this has something to do with Prefix matching rule of parse_known_args
#
# That is to say
#
# --rep 3
#
# would be parsed as
#
# args.workflow=3, unknown --rep
#
# instead of
#
# args.workflow=None, unknown --rep 3
#
# we then have to change the parse to disable args.workflow when
# there is no workflow option.
raw_args = shlex.split(raw_args) if isinstance(raw_args, str) else raw_args
if (script is None and code is None) or '-h' in raw_args:
parser = get_run_parser(interactive=True, with_workflow=True)
parser.print_help()
return
if raw_args and raw_args[0].lstrip().startswith('-'):
parser = get_run_parser(interactive=True, with_workflow=False)
parser.error = _parse_error
args, workflow_args = parser.parse_known_args(raw_args)
args.workflow = None
else:
parser = get_run_parser(interactive=True, with_workflow=True)
parser.error = _parse_error
args, workflow_args = parser.parse_known_args(raw_args)
# for reporting purpose
sys.argv = ['%run'] + raw_args
env.verbosity = args.verbosity
if kernel and not isinstance(env.logger.handlers[0],
NotebookLoggingHandler):
env.logger.handlers = []
levels = {
0: logging.ERROR,
1: logging.WARNING,
2: logging.INFO,
3: logging.DEBUG,
4: logging.TRACE,
None: logging.INFO
}
env.logger.addHandler(
NotebookLoggingHandler(levels[env.verbosity],
kernel,
title=' '.join(sys.argv)))
else:
env.logger.handers[0].setTitle(' '.join(sys.argv))
dt = datetime.datetime.now().strftime('%m%d%y_%H%M')
if args.__dag__ is None:
args.__dag__ = f'workflow_{dt}.dot'
elif args.__dag__ == '':
args.__dag__ = None
if args.__report__ is None:
args.__report__ = f'workflow_{dt}.html'
elif args.__report__ == '':
args.__report__ = None
if args.__remote__:
from sos.utils import load_config_files
cfg = load_config_files(args.__config__)
env.sos_dict.set('CONFIG', cfg)
# if executing on a remote host...
from sos.hosts import Host
host = Host(args.__remote__)
#
if script is None:
if not code.strip():
return
script = os.path.join('.sos', '__interactive__.sos')
with open(script, 'w') as s:
s.write(code)
# copy script to remote host...
host.send_to_host(script)
from sos.utils import remove_arg
argv = shlex.split(raw_args) if isinstance(raw_args, str) else raw_args
argv = remove_arg(argv, '-r')
argv = remove_arg(argv, '-c')
# execute the command on remote host
try:
with kernel.redirect_sos_io():
ret = host._host_agent.run_command(['sos', 'run', script] +
argv,
wait_for_task=True,
realtime=True)
if ret:
kernel.send_response(
kernel.iopub_socket, 'stream',
dict(name='stderr',
text=
f'remote execution of workflow exited with code {ret}'
))
except Exception as e:
if kernel:
kernel.send_response(kernel.iopub_socket, 'stream', {
'name': 'stdout',
'text': str(e)
})
return
if args.__bin_dirs__:
for d in args.__bin_dirs__:
if d == '~/.sos/bin' and not os.path.isdir(os.path.expanduser(d)):
os.makedirs(os.path.expanduser(d), exist_ok=True)
os.environ['PATH'] = os.pathsep.join(
[os.path.expanduser(x)
for x in args.__bin_dirs__]) + os.pathsep + os.environ['PATH']
# clear __step_input__, __step_output__ etc because there is
# no concept of passing input/outputs across cells.
env.sos_dict.set('__step_output__', sos_targets([]))
for k in [
'__step_input__', '__default_output__', 'step_input',
'step_output', 'step_depends', '_input', '_output', '_depends'
]:
env.sos_dict.pop(k, None)
try:
if script is None:
if not code.strip():
return
if kernel is None:
script = SoS_Script(content=code)
else:
if kernel._workflow_mode:
# in workflow mode, the content is sent by magics %run and %sosrun
script = SoS_Script(content=code)
else:
# this is a scratch step...
# if there is no section header, add a header so that the block
# appears to be a SoS script with one section
if not any([
SOS_SECTION_HEADER.match(line)
or line.startswith('%from')
or line.startswith('%include')
for line in code.splitlines()
]):
code = '[scratch_0]\n' + code
script = SoS_Script(content=code)
else:
#kernel.send_frontend_msg('stream',
# {'name': 'stdout', 'text': 'Workflow cell can only be executed with magic %run or %sosrun.'},
# title='# SoS warning')
return
else:
script = SoS_Script(filename=script)
workflow = script.workflow(args.workflow,
use_default=not args.__targets__)
env.config: DefaultDict[str, Union[None, bool, str]] = defaultdict(str)
executor = Interactive_Executor(
workflow,
args=workflow_args,
config={
'config_file':
args.__config__,
'output_dag':
args.__dag__,
'output_report':
args.__report__,
'sig_mode':
'ignore' if args.dryrun else args.__sig_mode__,
'default_queue':
'' if args.__queue__ is None else args.__queue__,
'wait_for_task':
True if args.__wait__ is True or args.dryrun else
(False if args.__no_wait__ else None),
'resume_mode':
kernel is not None and kernel._resume_execution,
'run_mode':
'dryrun' if args.dryrun else 'interactive',
'verbosity':
args.verbosity,
# wait if -w or in dryrun mode, not wait if -W, otherwise use queue default
'max_procs':
args.__max_procs__,
'max_running_jobs':
args.__max_running_jobs__,
# for infomration and resume only
'workdir':
os.getcwd(),
'script':
"interactive",
'workflow':
args.workflow,
'targets':
args.__targets__,
'bin_dirs':
args.__bin_dirs__,
'workflow_args':
workflow_args
})
return executor.run(args.__targets__)['__last_res__']
except PendingTasks:
raise
except SystemExit:
# this happens because the executor is in resume mode but nothing
# needs to be resumed, we simply pass
return
except Exception:
if args.verbosity and args.verbosity > 2:
sys.stderr.write(get_traceback())
raise
finally:
env.config['sig_mode'] = 'ignore'
env.verbosity = 2
class Interactive_Step_Executor(Step_Executor):
def __init__(self, step):
# This is the only interesting part of this executor. Basically
# it derives everything from SP_Step_Executor but does not
# use the Queue mechanism, so the __init__ and the run
# functions are copied from Base_Step_Executor
Base_Step_Executor.__init__(self, step)
self.run_mode='interactive'
self.host = None
def submit_tasks(self, tasks):
if not tasks:
return
if self.host is None:
if 'queue' in env.sos_dict['_runtime']:
self.queue = env.sos_dict['_runtime']['queue']
elif env.config['default_queue']:
self.queue = env.config['default_queue']
else:
self.queue = 'localhost'
self.host = Host(self.queue)
for task in tasks:
self.host.submit_task(task)
def wait_for_tasks(self, tasks):
if not tasks:
return {}
# wait till the executor responde
if all(x == 'completed' for x in self.host.check_status(tasks)):
if len(tasks) > 4:
print('!sos_hint: {} task{} completed: {}, {}, ..., {}'.format(
len(tasks), 's' if len(tasks) > 1 else '',
f"""<a onclick="task_info('{tasks[0]}', '{self.queue}')">{tasks[0][:4]}</a>""",
f"""<a onclick="task_info('{tasks[1]}', '{self.queue}')">{tasks[1][:4]}</a>""",
f"""<a onclick="task_info('{tasks[-1]}', '{self.queue}')">{tasks[-1][:4]}</a>"""))
else:
print('!sos_hint: {} task{} completed: {}'.format(len(tasks), 's' if len(tasks) > 1 else '',
','.join([f"""<a onclick="task_info('{x}', '{self.queue}')">{x[:4]}</a>""" for x in tasks])))
self.host._task_engine.remove_tasks(tasks)
return self.host.retrieve_results(tasks)
while True:
res = self.host.check_status(tasks)
if all(x not in ('submitted', 'pending', 'running') for x in res):
#completed = [task for task, status in zip(tasks, res) if status == 'completed']
self.host._task_engine.remove_tasks(tasks)
return self.host.retrieve_results(tasks)
# no pending
elif not env.config['wait_for_task']:
raise PendingTasks([(self.queue, x) for x,y in zip(tasks, res) if y in ('pending', 'submitted', 'running')])
time.sleep(1)
def run(self):
return Base_Step_Executor.run(self)
def log(self, stage=None, msg=None):
if stage == 'start':
env.logger.debug('{} ``{}``: {}'.format('Checking' if self.run_mode == 'dryrun' else 'Executing',
self.step.step_name(), self.step.comment.strip()))
elif stage == 'input':
if env.sos_dict['input'] is not None:
env.logger.debug('input: ``{}``'.format(short_repr(env.sos_dict['input'])))
elif stage == 'output':
if env.sos_dict['output'] is not None:
env.logger.debug('output: ``{}``'.format(short_repr(env.sos_dict['output'])))
def runfile(script=None,
raw_args='',
wdir='.',
code=None,
kernel=None,
**kwargs):
# this has something to do with Prefix matching rule of parse_known_args
#
# That is to say
#
# --rep 3
#
# would be parsed as
#
# args.workflow=3, unknown --rep
#
# instead of
#
# args.workflow=None, unknown --rep 3
#
# we then have to change the parse to disable args.workflow when
# there is no workflow option.
args = shlex.split(raw_args) if isinstance(raw_args, str) else raw_args
if (script is None and code is None) or '-h' in args:
parser = get_run_parser(interactive=True, with_workflow=True)
parser.print_help()
return
if args and args[0].lstrip().startswith('-'):
parser = get_run_parser(interactive=True, with_workflow=False)
parser.error = _parse_error
args, workflow_args = parser.parse_known_args(args)
args.workflow = None
else:
parser = get_run_parser(interactive=True, with_workflow=True)
parser.error = _parse_error
args, workflow_args = parser.parse_known_args(args)
# no multi-processing in interactive mode
env.max_jobs = 1
env.verbosity = args.verbosity
if args.__queue__ == '':
from sos.hosts import list_queues
list_queues(args.__config__, args.verbosity)
return
if args.__remote__:
from sos.utils import load_config_files
cfg = load_config_files(args.__config__)
env.sos_dict.set('CONFIG', cfg)
if args.__remote__ == '':
from .hosts import list_queues
list_queues(cfg, args.verbosity)
return
# if executing on a remote host...
from sos.hosts import Host
host = Host(args.__remote__)
#
if script is None:
if not code.strip():
return
script = os.path.join('.sos', '__interactive__.sos')
with open(script, 'w') as s:
s.write(code)
# copy script to remote host...
host.send_to_host(script)
from sos.utils import remove_arg
argv = shlex.split(raw_args) if isinstance(raw_args, str) else raw_args
argv = remove_arg(argv, '-r')
argv = remove_arg(argv, '-c')
# execute the command on remote host
try:
with kernel.redirect_sos_io():
ret = host._host_agent.run_command(['sos', 'run', script] +
argv,
wait_for_task=True,
realtime=True)
if ret:
kernel.send_response(
kernel.iopub_socket, 'stream',
dict(name='stderr',
text=
f'remote execution of workflow exited with code {ret}'
))
except Exception as e:
if kernel:
kernel.send_response(kernel.iopub_socket, 'stream', {
'name': 'stdout',
'text': str(e)
})
return
if args.__bin_dirs__:
import fasteners
for d in args.__bin_dirs__:
if d == '~/.sos/bin' and not os.path.isdir(os.path.expanduser(d)):
with fasteners.InterProcessLock(
os.path.join(tempfile.gettempdir(), 'sos_lock_bin')):
os.makedirs(os.path.expanduser(d))
elif not os.path.isdir(os.path.expanduser(d)):
raise ValueError(f'directory does not exist: {d}')
os.environ['PATH'] = os.pathsep.join(
[os.path.expanduser(x)
for x in args.__bin_dirs__]) + os.pathsep + os.environ['PATH']
# clear __step_input__, __step_output__ etc because there is
# no concept of passing input/outputs across cells.
env.sos_dict.set('__step_output__', [])
for k in ['__step_input__', '__default_output__', 'input', 'output', \
'depends', '_input', '_output', '_depends']:
env.sos_dict.pop(k, None)
try:
if script is None:
if not code.strip():
return
if kernel is None:
script = SoS_Script(content=code)
else:
if kernel._workflow_mode:
# in workflow mode, the content is sent by magics %run and %sosrun
script = SoS_Script(content=code)
else:
# this is a scratch step...
# if there is no section header, add a header so that the block
# appears to be a SoS script with one section
if not any([
SOS_SECTION_HEADER.match(line)
or line.startswith('%from')
or line.startswith('%include')
for line in code.splitlines()
]):
code = '[scratch_0]\n' + code
script = SoS_Script(content=code)
else:
if kernel.cell_idx == -1:
kernel.send_frontend_msg(
'stream', {
'name':
'stdout',
'text':
'Workflow can only be executed with magic %run or %sosrun.'
})
return
else:
script = SoS_Script(filename=script)
workflow = script.workflow(args.workflow)
executor = Interactive_Executor(
workflow,
args=workflow_args,
config={
'config_file':
args.__config__,
'output_dag':
args.__dag__,
'sig_mode':
args.__sig_mode__,
'default_queue':
'' if args.__queue__ is None else args.__queue__,
'wait_for_task':
True if args.__wait__ is True or args.dryrun else
(False if args.__no_wait__ else None),
'resume_mode':
kernel is not None and kernel._resume_execution,
'run_mode':
'dryrun' if args.dryrun else 'interactive',
'verbosity':
args.verbosity,
# wait if -w or in dryrun mode, not wait if -W, otherwise use queue default
'max_procs':
1,
'max_running_jobs':
args.__max_running_jobs__,
# for infomration and resume only
'workdir':
os.getcwd(),
'script':
"interactive",
'workflow':
args.workflow,
'targets':
args.__targets__,
'bin_dirs':
args.__bin_dirs__,
'workflow_args':
workflow_args
})
return executor.run(args.__targets__)
except PendingTasks:
raise
except SystemExit:
# this happens because the executor is in resume mode but nothing
# needs to be resumed, we simply pass
return
except Exception:
if args.verbosity and args.verbosity > 2:
sys.stderr.write(get_traceback())
raise
finally:
env.config['sig_mode'] = 'ignore'
env.verbosity = 2
class Interactive_Step_Executor(Step_Executor):
def __init__(self, step, mode='interactive'):
# This is the only interesting part of this executor. Basically
# it derives everything from SP_Step_Executor but does not
# use the Queue mechanism, so the __init__ and the run
# functions are copied from Base_Step_Executor
Base_Step_Executor.__init__(self, step)
self.run_mode = mode
self.host = None
def submit_tasks(self, tasks):
if not tasks:
return
if self.host is None:
if 'queue' in env.sos_dict['_runtime']:
queue = env.sos_dict['_runtime']['queue']
elif env.config['default_queue']:
queue = env.config['default_queue']
else:
queue = 'localhost'
self.host = Host(queue)
for task in tasks:
self.host.submit_task(task)
def wait_for_tasks(self, tasks, all_submitted):
if not tasks:
return {}
# when we wait, the "outsiders" also need to see the tags etc
# of the tasks so we have to write to the database. #156
env.master_push_socket.send_pyobj(['commit_sig'])
if all_submitted and 'shared' not in env.sos_dict['_runtime']:
# if no shared and all taks have been submited return
sys.exit(0)
# turn this function to a generator to satisfy the interface, but do not
# actually wait for any socket.
yield None
# wait till the executor responde
if all(x == 'completed' for x in self.host.check_status(tasks)):
if len(tasks) > 4:
print('HINT: {} task{} completed: {}, {}, ..., {}'.format(
len(tasks), 's' if len(tasks) > 1 else '',
f"""<a onclick="task_info('{tasks[0]}', '{self.host.alias}')">{tasks[0][:4]}</a>""",
f"""<a onclick="task_info('{tasks[1]}', '{self.host.alias}')">{tasks[1][:4]}</a>""",
f"""<a onclick="task_info('{tasks[-1]}', '{self.host.alias}')">{tasks[-1][:4]}</a>"""
))
else:
print('HINT: {} task{} completed: {}'.format(
len(tasks), 's' if len(tasks) > 1 else '', ','.join([
f"""<a onclick="task_info('{x}', '{self.host.alias}')">{x[:4]}</a>"""
for x in tasks
])))
return self.host.retrieve_results(tasks)
while True:
res = self.host.check_status(tasks)
if all(x not in ('submitted', 'pending', 'running') for x in res):
#completed = [task for task, status in zip(tasks, res) if status == 'completed']
return self.host.retrieve_results(tasks)
time.sleep(0.1)
def run(self):
try:
runner = Base_Step_Executor.run(self)
yreq = next(runner)
while True:
yreq = runner.send(yreq)
except StopIteration as e:
return e.value
def log(self, stage=None, msg=None):
if stage == 'start':
env.logger.debug('{} ``{}``: {}'.format(
'Checking' if self.run_mode == 'dryrun' else 'Executing',
self.step.step_name(), self.step.comment.strip()))
elif stage == 'input':
if env.sos_dict['step_input'] is not None:
env.logger.debug('input: ``{}``'.format(
short_repr(env.sos_dict['step_input'])))
elif stage == 'output':
if env.sos_dict['step_output'] is not None:
env.logger.debug('output: ``{}``'.format(
short_repr(env.sos_dict['step_output'])))
