def console(args):
""" Start an interactive ipython console """
user_ns = {
'tk': sisyphus.toolkit,
'config_files': args.config_files,
}
if args.load:
jobs = [sisyphus.toolkit.load_job(i) for i in args.load]
user_ns['jobs'] = jobs
for i, job in enumerate(jobs):
print("jobs[%i]: %s" % (i, job))
elif not args.not_load_config:
load_configs(args.config_files)
# TODO Update welcome message
welcome_msg = """
Info: IPCompleter.greedy = True is set to True.
This allows to auto complete lists and dictionaries entries, but may evaluates functions on tab.
Enter tk? for help"""
import IPython
from traitlets.config.loader import Config
c = Config()
c.InteractiveShell.banner2 = welcome_msg
c.IPCompleter.greedy = True
c.InteractiveShellApp.exec_lines = ['%rehashx'] + args.commands
IPython.start_ipython(config=c, argv=[], user_ns=user_ns)
def reload_config(config_files: List[str] = []):
""" Reset state, reload old config files, and load given config_files
:param config_files([str, ...]):
"""
# Reset current state
import sisyphus.job
sisyphus.job.created_jobs = {}
global sis_graph
sis_graph = graph.SISGraph()
_reload_prefix(gs.CONFIG_PREFIX)
# Load new config
load_configs(config_files)
def console(args):
""" Start an interactive ipython console """
user_ns = {
'tk': sisyphus.toolkit,
'config_files': args.config_files,
}
if args.load:
jobs = [sisyphus.toolkit.load_job(i) for i in args.load]
user_ns['jobs'] = jobs
for i, job in enumerate(jobs):
print("jobs[%i]: %s" % (i, job))
elif not args.not_load_config:
load_configs(args.config_files)
# TODO Update welcome message
welcome_msg = """
Info: IPCompleter.greedy = True is set to True.
This allows to auto complete lists and dictionaries entries, but may evaluates functions on tab.
Enter tk? for help"""
import IPython
from traitlets.config.loader import Config
c = Config()
c.InteractiveShellApp.exec_lines = [
# register load_job and load_configs as magic
# 'import IPython.core.magic\n'
# 'shut_up=IPython.core.magic.register_line_magic(tk.load_job)\n'
# 'shut_up=IPython.core.magic.register_line_magic(tk.load_configs)\n',
# settings that make the ipython console behave more like a system console
# 'del shut_up', # shut_up is used to silence the return value
# 'initialize()',
'%rehashx',
# '%autocall',
'%config IPCompleter.greedy = True',
'print(%s)' % repr(welcome_msg)
] + args.commands
IPython.start_ipython(config=c, argv=[], user_ns=user_ns)
def manager(args):
""" Manage which job should run next """
if args.run:
if not os.path.isdir(gs.WORK_DIR):
prompt = '%s does not exist, should I continue? ' \
'The directory will be created if needed inplace (y/N)' % gs.WORK_DIR
if args.ui:
ret = args.ui.ask_user(prompt)
logging.info('%s %s' % (prompt, ret))
else:
ret = input(prompt)
if ret.lower() != 'y':
logging.warning('Abort, create directory or link it to the wished work destination')
return
# try to load fuse filesystem
filesystem = None
if args.filesystem:
import sisyphus.filesystem as filesystem
# Ensure this thread has a event loop
loop = asyncio.new_event_loop()
asyncio.set_event_loop(loop)
start = time.time()
load_configs(args.config_files)
load_time = time.time() - start
if load_time < 5:
logging.debug("Config loaded (time needed: %.2f)" % load_time)
else:
logging.info("Config loaded (time needed: %.2f)" % load_time)
sis_graph = toolkit.sis_graph
Block.sis_graph = sis_graph
job_engine = toolkit.cached_engine()
job_engine.start_engine()
job_cleaner = None
try:
if args.run:
create_aliases(sis_graph.jobs())
else:
gs.JOB_AUTO_CLEANUP = False
if gs.JOB_AUTO_CLEANUP:
job_cleaner = JobCleaner(sis_graph=sis_graph)
job_cleaner.start()
# The actual work loop
if args.http_port is not None:
logging.debug("Start http server")
start_http_server(sis_graph=sis_graph,
sis_engine=job_engine,
port=args.http_port,
thread=True)
manager = Manager(sis_graph=sis_graph,
job_engine=job_engine,
link_outputs=args.run,
clear_errors_once=args.clear_errors_once,
clear_interrupts_once=args.clear_interrupts_once,
ignore_once=args.ignore_once,
start_computations=args.run,
job_cleaner=job_cleaner,
interative=args.interactive,
ui=args.ui)
if args.ui:
args.ui.manager = manager
args.ui.update_menu()
else:
manager.unpause()
kernel_connect_file = None
if gs.START_KERNEL:
kernel_connect_file = init_IPython_kernel(user_ns={'manager': manager,
'job_engine': job_engine,
'tk': toolkit,
'sis_graph': sis_graph})
try:
if args.filesystem:
# Start main loop
logging.debug("Start main loop")
manager.start()
# graph updates
graph_update_thread = threading.Thread(
target=tools.default_handle_exception_interrupt_main_thread(sis_graph.update_nodes))
graph_update_thread.start()
# Start filesystem
# run in main thread to allow signal handling of FUSE
if not os.path.isdir(args.filesystem):
os.mkdir(args.filesystem)
filesystem.start(work_dir=gs.WORK_DIR, sis_graph=sis_graph, mountpoint=args.filesystem)
else:
manager.run()
except KeyboardInterrupt:
logging.info('Got user interrupt signal stop engine and exit')
if kernel_connect_file:
try:
os.remove(kernel_connect_file)
except (IOError, OSError):
pass
# Print traceback in debug mode
if logging.root.isEnabledFor(logging.DEBUG):
raise
sys.exit(1)
finally:
if job_cleaner:
job_cleaner.close()
job_engine.stop_engine()