def main(args):
signal.signal(signal.SIGINT, sig_handler)
signal.signal(signal.SIGTERM, sig_handler)
wf_filter = args.wf_filter
job_mode = args.job_mode
timelimit_min = args.time_limit_minutes
nthread = (args.num_transition_threads if args.num_transition_threads else
settings.NUM_TRANSITION_THREADS)
gpus_per_node = args.gpus_per_node
Launcher = MPILauncher if job_mode == 'mpi' else SerialLauncher
try:
if nthread > 0:
transition_pool = transitions.TransitionProcessPool(
nthread, wf_filter)
else:
transition_pool = None
launcher = Launcher(wf_filter, timelimit_min, gpus_per_node)
launcher.run()
except:
raise
finally:
if transition_pool is not None:
transition_pool.terminate()
logger.info("Exit: Launcher exit graceful\n\n")
QueuedLaunch.refresh_from_scheduler()
open_queues = get_open_queues()
if open_queues:
logger.info(f"Open queues: {list(open_queues.keys())}")
qlaunch = jobpacker.create_qlaunch(open_queues)
if qlaunch:
submit_qlaunch(qlaunch)
if not QueuedLaunch.acquire_advisory():
logger.error('Failed to refresh advisory lock; aborting')
break
elif not EXIT_FLAG:
source.clear_stale_locks()
time.sleep(10)
if __name__ == "__main__":
setup()
config_logging('service')
logger.info(f"Balsam Service starting on {gethostname()}")
parser = service_subparser()
transition_pool = transitions.TransitionProcessPool(5, '')
source.start_tick()
try:
main(parser.parse_args())
except:
raise
finally:
transition_pool.terminate()
source.release_all_owned()
logger.info(f"Balsam Service shutdown: released all locks OK")