def run(self):
"""Entry point for the VELES execution engine.
"""
veles.validate_environment()
ret = self._process_special_args()
if ret is not None:
return ret
parser = Main.init_parser()
args = parser.parse_args(self.argv)
self._apply_args(args)
self.setup_logging(args.verbosity)
self._print_logo(args)
for name in filter(str.strip, args.debug.split(',')):
logging.getLogger(name).setLevel(logging.DEBUG)
self._seed_random(args.random_seed)
if args.debug_pickle:
setup_pickle_debug()
ThreadPool.reset()
self._register_print_max_rss()
if self.logger.isEnabledFor(logging.DEBUG):
self._print_config(root)
wm = self._load_model(self.workflow_file)
self._apply_config(self.config_file)
self._override_config(args.config_list)
if self.logger.isEnabledFor(logging.DEBUG):
self._print_config(root)
self._run_core(wm)
if not self.interactive:
self.info("End of job")
else:
self.info("\033[1;35mReturned the control\033[0m")
return Main.EXIT_SUCCESS
def run(self):
"""Entry point for the VELES execution engine.
"""
veles.validate_environment()
ret = self._process_special_args()
if ret is not None:
return ret
parser = Main.init_parser()
args = parser.parse_args(self.argv)
self._apply_args(args)
self.setup_logging(args.verbosity)
self._print_logo(args)
for name in filter(str.strip, args.debug.split(',')):
logging.getLogger(name).setLevel(logging.DEBUG)
self._seed_random(args.random_seed)
if args.debug_pickle:
setup_pickle_debug()
ThreadPool.reset()
self._register_print_max_rss()
if self.logger.isEnabledFor(logging.DEBUG):
self._print_config(root)
wm = self._load_model(self.workflow_file)
self._apply_config(self.config_file)
self._override_config(args.config_list)
if self.logger.isEnabledFor(logging.DEBUG):
self._print_config(root)
self._run_core(wm)
if not self.interactive:
self.info("End of job")
else:
self.info("\033[1;35mReturned the control\033[0m")
return Main.EXIT_SUCCESS
def initialize(self, snapshot=False, **kwargs):
# Ensure reactor stops in some rare cases when it does not normally
if not self.interactive:
self.workflow.thread_pool.register_on_shutdown(
Launcher._reactor_shutdown)
else:
self._interactive_shutdown_ref = self._interactive_shutdown
ThreadPool.register_atexit(self._interactive_shutdown_ref)
if self.is_slave:
self._agent = SlaveManager(self.args.master_address, self.workflow)
def on_id_received(node_id, log_id):
self.id = node_id
self.log_id = log_id
if self.logs_to_mongo:
logger.Logger.duplicate_all_logging_to_mongo(
self.args.log_mongo, self.log_id, node_id)
self.agent.on_id_received = on_id_received
else:
if self.reports_web_status:
timeout = self._notify_update_interval / 2
self._web_status_agent = Agent(
reactor, pool=HTTPConnectionPool(reactor),
connectTimeout=timeout)
# Launch the status server if it's not been running yet
self._launch_status()
if self.workflow.plotters_are_enabled and \
(not self.interactive or Launcher.graphics_client is None):
try:
Launcher.graphics_server, Launcher.graphics_client = \
graphics_server.GraphicsServer.launch(
self.workflow.thread_pool,
self.matplotlib_backend,
self._set_webagg_port,
self.args.no_graphics_client)
except Exception as e:
self.error("Failed to create the graphics server and/or "
"client. Try to completely disable plotting "
"with -p ''.")
raise from_none(e)
elif self.args.no_graphics_client:
self.warning("Plotters are disabled. --no-graphics-client has "
"no effect.")
if self.is_master:
try:
self._agent = MasterManager(self.args.listen_address,
self.workflow)
# Launch the nodes described in the command line or config
self._launch_nodes()
except Exception as e:
self._stop_graphics()
raise from_none(e)
# The last moment when we can do this, because OpenCL device curses
# new process creation
try:
self._generate_workflow_graphs()
except Exception as e:
self.error("Failed to generate the workflow graph(s)")
self._stop_graphics()
raise from_none(e)
try:
if not self.is_master and not kwargs.get("no_device", False):
self._device = Device()
except Exception as e:
self.error("Failed to create the OpenCL device")
self._stop_graphics()
raise from_none(e)
if "no_device" in kwargs:
del kwargs["no_device"]
self.workflow.reset_thread_pool()
def greet_reactor():
def set_thread_ident():
self._reactor_thread_ident = threading.current_thread().ident
reactor.callWhenRunning(self.info, "Reactor is running")
reactor.callWhenRunning(set_thread_ident)
def initialize_workflow():
try:
self.workflow.initialize(device=self.device, snapshot=snapshot,
**kwargs)
except Exception as ie:
self.error("Failed to initialize the workflow")
self._stop_graphics()
self.device_thread_pool_detach()
raise from_none(ie)
if not self.interactive:
# delay greet_reactor() until everything else is initialized
initialize_workflow()
else:
greet_reactor()
reactor.callWhenRunning(initialize_workflow)
if not self.is_standalone:
self._agent.initialize()
if not self.interactive:
trigger = reactor.addSystemEventTrigger
trigger('before', 'shutdown', self._on_stop)
trigger('after', 'shutdown', self._print_stats)
trigger('after', 'shutdown', self.event, "work", "end", height=0.1)
else:
register = self.workflow.thread_pool.register_on_shutdown
self._on_stop_ref = self._on_stop
register(self._on_stop_ref)
self._print_stats_ref = self._print_stats
register(self._print_stats_ref)
def work_end():
self.event("work", "end", height=0.1)
self._work_end = work_end
register(self._work_end)
for unit in self.workflow:
if isinstance(unit, Plotter):
unit.graphics_server = Launcher.graphics_server
greet_reactor()
self._initialized = True
def initialize(self, snapshot=False, **kwargs):
# Ensure reactor stops in some rare cases when it does not normally
if not self.interactive:
self.workflow.thread_pool.register_on_shutdown(
Launcher._reactor_shutdown)
else:
self._interactive_shutdown_ref = self._interactive_shutdown
ThreadPool.register_atexit(self._interactive_shutdown_ref)
if self.is_slave:
self._agent = SlaveManager(self.args.master_address, self.workflow)
def on_id_received(node_id, log_id):
self.id = node_id
self.log_id = log_id
if self.logs_to_mongo:
logger.Logger.duplicate_all_logging_to_mongo(
self.args.log_mongo, self.log_id, node_id)
self.agent.on_id_received = on_id_received
else:
if self.reports_web_status:
timeout = self._notify_update_interval / 2
self._web_status_agent = Agent(
reactor,
pool=HTTPConnectionPool(reactor),
connectTimeout=timeout)
# Launch the status server if it's not been running yet
self._launch_status()
if self.workflow.plotters_are_enabled and \
(not self.interactive or Launcher.graphics_client is None):
try:
Launcher.graphics_server, Launcher.graphics_client = \
graphics_server.GraphicsServer.launch(
self.workflow.thread_pool,
self.matplotlib_backend,
self._set_webagg_port,
self.args.no_graphics_client)
except Exception as e:
self.error("Failed to create the graphics server and/or "
"client. Try to completely disable plotting "
"with -p ''.")
raise from_none(e)
elif self.args.no_graphics_client:
self.warning("Plotters are disabled. --no-graphics-client has "
"no effect.")
if self.is_master:
try:
self._agent = MasterManager(self.args.listen_address,
self.workflow)
# Launch the nodes described in the command line or config
self._launch_nodes()
except Exception as e:
self._stop_graphics()
raise from_none(e)
# The last moment when we can do this, because OpenCL device curses
# new process creation
try:
self._generate_workflow_graphs()
except Exception as e:
self.error("Failed to generate the workflow graph(s)")
self._stop_graphics()
raise from_none(e)
try:
if not self.is_master and not kwargs.get("no_device", False):
self._device = Device()
except Exception as e:
self.error("Failed to create the OpenCL device")
self._stop_graphics()
raise from_none(e)
if "no_device" in kwargs:
del kwargs["no_device"]
self.workflow.reset_thread_pool()
def greet_reactor():
def set_thread_ident():
self._reactor_thread_ident = threading.current_thread().ident
reactor.callWhenRunning(self.info, "Reactor is running")
reactor.callWhenRunning(set_thread_ident)
def initialize_workflow():
try:
self.workflow.initialize(device=self.device,
snapshot=snapshot,
**kwargs)
except Exception as ie:
self.error("Failed to initialize the workflow")
self._stop_graphics()
self.device_thread_pool_detach()
raise from_none(ie)
if not self.interactive:
# delay greet_reactor() until everything else is initialized
initialize_workflow()
else:
greet_reactor()
reactor.callWhenRunning(initialize_workflow)
if not self.is_standalone:
self._agent.initialize()
if not self.interactive:
trigger = reactor.addSystemEventTrigger
trigger('before', 'shutdown', self._on_stop)
trigger('after', 'shutdown', self._print_stats)
trigger('after', 'shutdown', self.event, "work", "end", height=0.1)
else:
register = self.workflow.thread_pool.register_on_shutdown
self._on_stop_ref = self._on_stop
register(self._on_stop_ref)
self._print_stats_ref = self._print_stats
register(self._print_stats_ref)
def work_end():
self.event("work", "end", height=0.1)
self._work_end = work_end
register(self._work_end)
for unit in self.workflow:
if isinstance(unit, Plotter):
unit.graphics_server = Launcher.graphics_server
greet_reactor()
self._initialized = True