def manager(args):
""" Manage which job should run next """
if args.run:
if not os.path.isdir(gs.WORK_DIR):
prompt = '%s does not exist, should I continue? ' \
'The directory will be created if needed inplace (y/N)' % gs.WORK_DIR
if args.ui:
ret = args.ui.ask_user(prompt)
logging.info('%s %s' % (prompt, ret))
else:
ret = input(prompt)
if ret.lower() != 'y':
logging.warning(
'Abort, create directory or link it to the wished work destination'
)
return
# try to load fuse filesystem
filesystem = None
if args.filesystem:
import sisyphus.filesystem as filesystem
# Ensure this thread has a event loop
loop = asyncio.new_event_loop()
asyncio.set_event_loop(loop)
start = time.time()
config_manager.load_configs(args.config_files)
load_time = time.time() - start
if load_time < 5:
logging.debug("Config loaded (time needed: %.2f)" % load_time)
else:
logging.info("Config loaded (time needed: %.2f)" % load_time)
sis_graph = toolkit.sis_graph
Block.sis_graph = sis_graph
job_engine = toolkit.cached_engine()
job_engine.start_engine()
manager = None
try:
if args.run:
create_aliases(sis_graph.jobs())
# The actual work loop
if args.http_port is not None:
logging.debug("Start http server")
start_http_server(sis_graph=sis_graph,
sis_engine=job_engine,
port=args.http_port,
thread=True)
manager = Manager(sis_graph=sis_graph,
job_engine=job_engine,
link_outputs=args.run,
clear_errors_once=args.clear_errors_once,
clear_interrupts_once=args.clear_interrupts_once,
ignore_once=args.ignore_once,
start_computations=args.run,
interative=args.interactive,
ui=args.ui)
if args.ui:
args.ui.manager = manager
args.ui.update_menu()
else:
manager.unpause()
kernel_connect_file = None
if gs.START_KERNEL:
kernel_connect_file = init_IPython_kernel(
user_ns={
'manager': manager,
'job_engine': job_engine,
'tk': toolkit,
'sis_graph': sis_graph
})
try:
if args.filesystem:
# Start main loop
logging.debug("Start main loop")
manager.start()
# graph updates
graph_update_thread = threading.Thread(
target=tools.
default_handle_exception_interrupt_main_thread(
sis_graph.update_nodes))
graph_update_thread.start()
# Start filesystem
# run in main thread to allow signal handling of FUSE
if not os.path.isdir(args.filesystem):
os.mkdir(args.filesystem)
filesystem.start(work_dir=gs.WORK_DIR,
sis_graph=sis_graph,
mountpoint=args.filesystem)
else:
manager.run()
except KeyboardInterrupt:
logging.info('Got user interrupt signal stop engine and exit')
if kernel_connect_file:
try:
os.remove(kernel_connect_file)
except (IOError, OSError):
pass
# Print traceback in debug mode
if logging.root.isEnabledFor(logging.DEBUG):
raise
sys.exit(1)
finally:
if manager and manager.job_cleaner:
manager.job_cleaner.close()
job_engine.stop_engine()
def show_job(self, job):
items = []
def add_button(label, on_press, user_data=None):
button = RightButton(label, on_press=on_press, user_data=user_data)
return None, urwid.AttrWrap(button, 'button normal',
'button select')
items.append(
add_button('Action: remove job and descendants', self.run_commands,
[(tk.remove_job_and_descendants, [job], {})]))
items.append(
add_button('Setup job directory', self.run_commands,
[(job._sis_setup_directory, [], {})]))
items.append((None, urwid.Text("------------------------")))
if job._sis_runnable() and not job._sis_finished():
next_task = None
for task in job._sis_tasks():
if task.finished():
items.append(
(None, urwid.Text("Finished Task: %s" % task.name())))
else:
task_ids = task.task_ids()
for task_id in task_ids:
state = task.state(tk.cached_engine(), task_id)
if len(task_ids) > 1:
task_label = "%s <%i>" % (task.name(), task_id)
else:
task_label = task.name()
items.append((None,
urwid.Text('Task %s is in state: "%s"' %
(task_label, state))))
if state == gs.STATE_RUNNABLE:
next_task = task
break
if next_task:
items.append(
add_button(
'Submit task: "%s" to engine' % next_task.name(),
self.run_commands, [(tk.submit_next_task, [job], {})]))
items.append((None, urwid.Text("------------------------")))
attributes = job.__dict__
for name in ('_sis_kwargs', '_sis_aliases', '_sis_keep_value',
'_sis_stacktrace', '_sis_blocks', '_sis_tags',
'_sis_task_rqmt_overwrite', '_sis_vis_name',
'_sis_outputs', '_sis_environment'):
attr = attributes[name]
if attr:
items.append((name[5:], attr))
items.append((None, urwid.Text("------------------------")))
for k, v in attributes.items():
if not k.startswith('_sis_'):
items.append((k, v))
self.show_items(items)
self.loop.widget = self.obj_view
self.redraw()