def construct(self): # This is the working dir by now. sys.path.insert(0, '') self.start_mpi() self.start_logging() # Create the underlying shell class and EngineService shell_class = import_item(self.master_config.Global.shell_class) self.engine_service = EngineService(shell_class, mpi=mpi) self.exec_lines() # Create the service hierarchy self.main_service = service.MultiService() self.engine_service.setServiceParent(self.main_service) self.tub_service = Tub() self.tub_service.setServiceParent(self.main_service) # This needs to be called before the connection is initiated self.main_service.startService() # This initiates the connection to the controller and calls # register_engine to tell the controller we are ready to do work self.engine_connector = EngineConnector(self.tub_service) log.msg("Using furl file: %s" % self.master_config.Global.furl_file) reactor.callWhenRunning(self.call_connect)
def setUp(self): """Setup the EngineService and FrontEndBase""" self.fb = FrontEndCallbackChecker(engine=EngineService())
class IPEngineApp(ApplicationWithClusterDir): name = u'ipengine' description = _description command_line_loader = IPEngineAppConfigLoader default_config_file_name = default_config_file_name auto_create_cluster_dir = True def create_default_config(self): super(IPEngineApp, self).create_default_config() # The engine should not clean logs as we don't want to remove the # active log files of other running engines. self.default_config.Global.clean_logs = False # Global config attributes self.default_config.Global.exec_lines = [] self.default_config.Global.shell_class = 'IPython.kernel.core.interpreter.Interpreter' # Configuration related to the controller # This must match the filename (path not included) that the controller # used for the FURL file. self.default_config.Global.furl_file_name = u'ipcontroller-engine.furl' # If given, this is the actual location of the controller's FURL file. # If not, this is computed using the profile, app_dir and furl_file_name self.default_config.Global.furl_file = u'' # The max number of connection attemps and the initial delay between # those attemps. self.default_config.Global.connect_delay = 0.1 self.default_config.Global.connect_max_tries = 15 # MPI related config attributes self.default_config.MPI.use = '' self.default_config.MPI.mpi4py = mpi4py_init self.default_config.MPI.pytrilinos = pytrilinos_init def post_load_command_line_config(self): pass def pre_construct(self): super(IPEngineApp, self).pre_construct() self.find_cont_furl_file() def find_cont_furl_file(self): """Set the furl file. Here we don't try to actually see if it exists for is valid as that is hadled by the connection logic. """ config = self.master_config # Find the actual controller FURL file if not config.Global.furl_file: try_this = os.path.join(config.Global.cluster_dir, config.Global.security_dir, config.Global.furl_file_name) config.Global.furl_file = try_this def construct(self): # This is the working dir by now. sys.path.insert(0, '') self.start_mpi() self.start_logging() # Create the underlying shell class and EngineService shell_class = import_item(self.master_config.Global.shell_class) self.engine_service = EngineService(shell_class, mpi=mpi) self.exec_lines() # Create the service hierarchy self.main_service = service.MultiService() self.engine_service.setServiceParent(self.main_service) self.tub_service = Tub() self.tub_service.setServiceParent(self.main_service) # This needs to be called before the connection is initiated self.main_service.startService() # This initiates the connection to the controller and calls # register_engine to tell the controller we are ready to do work self.engine_connector = EngineConnector(self.tub_service) log.msg("Using furl file: %s" % self.master_config.Global.furl_file) reactor.callWhenRunning(self.call_connect) def call_connect(self): d = self.engine_connector.connect_to_controller( self.engine_service, self.master_config.Global.furl_file, self.master_config.Global.connect_delay, self.master_config.Global.connect_max_tries) def handle_error(f): log.msg( 'Error connecting to controller. This usually means that ' 'i) the controller was not started, ii) a firewall was blocking ' 'the engine from connecting to the controller or iii) the engine ' ' was not pointed at the right FURL file:') log.msg(f.getErrorMessage()) reactor.callLater(0.1, reactor.stop) d.addErrback(handle_error) def start_mpi(self): global mpi mpikey = self.master_config.MPI.use mpi_import_statement = self.master_config.MPI.get(mpikey, None) if mpi_import_statement is not None: try: self.log.info("Initializing MPI:") self.log.info(mpi_import_statement) exec mpi_import_statement in globals() except: mpi = None else: mpi = None def exec_lines(self): for line in self.master_config.Global.exec_lines: try: log.msg("Executing statement: '%s'" % line) self.engine_service.execute(line) except: log.msg("Error executing statement: %s" % line) def start_app(self): reactor.run()
def start_engine(): """ Start the engine, by creating it and starting the Twisted reactor. This method does: * If it exists, runs the `mpi_import_statement` to call `MPI_Init` * Starts the engine logging * Creates an IPython shell and wraps it in an `EngineService` * Creates a `foolscap.Tub` to use in connecting to a controller. * Uses the tub and the `EngineService` along with a Foolscap URL (or FURL) to connect to the controller and register the engine with the controller """ kernel_config = kernel_config_manager.get_config_obj() core_config = core_config_manager.get_config_obj() # Execute the mpi import statement that needs to call MPI_Init global mpi mpikey = kernel_config['mpi']['default'] mpi_import_statement = kernel_config['mpi'].get(mpikey, None) if mpi_import_statement is not None: try: exec mpi_import_statement in globals() except: mpi = None else: mpi = None # Start logging logfile = kernel_config['engine']['logfile'] if logfile: logfile = logfile + str(os.getpid()) + '.log' try: openLogFile = open(logfile, 'w') except: openLogFile = sys.stdout else: openLogFile = sys.stdout log.startLogging(openLogFile) # Create the underlying shell class and EngineService shell_class = import_item(core_config['shell']['shell_class']) engine_service = EngineService(shell_class, mpi=mpi) shell_import_statement = core_config['shell']['import_statement'] if shell_import_statement: try: engine_service.execute(shell_import_statement) except: log.msg("Error running import_statement: %s" % sis) # Create the service hierarchy main_service = service.MultiService() engine_service.setServiceParent(main_service) tub_service = Tub() tub_service.setServiceParent(main_service) # This needs to be called before the connection is initiated main_service.startService() # This initiates the connection to the controller and calls # register_engine to tell the controller we are ready to do work engine_connector = EngineConnector(tub_service) furl_file = kernel_config['engine']['furl_file'] log.msg("Using furl file: %s" % furl_file) d = engine_connector.connect_to_controller(engine_service, furl_file) d.addErrback(lambda _: reactor.stop()) reactor.run()
class IPEngineApp(ApplicationWithClusterDir): name = u'ipengine' description = _description command_line_loader = IPEngineAppConfigLoader default_config_file_name = default_config_file_name auto_create_cluster_dir = True def create_default_config(self): super(IPEngineApp, self).create_default_config() # The engine should not clean logs as we don't want to remove the # active log files of other running engines. self.default_config.Global.clean_logs = False # Global config attributes self.default_config.Global.exec_lines = [] self.default_config.Global.shell_class = 'IPython.kernel.core.interpreter.Interpreter' # Configuration related to the controller # This must match the filename (path not included) that the controller # used for the FURL file. self.default_config.Global.furl_file_name = u'ipcontroller-engine.furl' # If given, this is the actual location of the controller's FURL file. # If not, this is computed using the profile, app_dir and furl_file_name self.default_config.Global.furl_file = u'' # The max number of connection attemps and the initial delay between # those attemps. self.default_config.Global.connect_delay = 0.1 self.default_config.Global.connect_max_tries = 15 # MPI related config attributes self.default_config.MPI.use = '' self.default_config.MPI.mpi4py = mpi4py_init self.default_config.MPI.pytrilinos = pytrilinos_init def post_load_command_line_config(self): pass def pre_construct(self): super(IPEngineApp, self).pre_construct() self.find_cont_furl_file() def find_cont_furl_file(self): """Set the furl file. Here we don't try to actually see if it exists for is valid as that is hadled by the connection logic. """ config = self.master_config # Find the actual controller FURL file if not config.Global.furl_file: try_this = os.path.join( config.Global.cluster_dir, config.Global.security_dir, config.Global.furl_file_name ) config.Global.furl_file = try_this def construct(self): # This is the working dir by now. sys.path.insert(0, '') self.start_mpi() self.start_logging() # Create the underlying shell class and EngineService shell_class = import_item(self.master_config.Global.shell_class) self.engine_service = EngineService(shell_class, mpi=mpi) self.exec_lines() # Create the service hierarchy self.main_service = service.MultiService() self.engine_service.setServiceParent(self.main_service) self.tub_service = Tub() self.tub_service.setServiceParent(self.main_service) # This needs to be called before the connection is initiated self.main_service.startService() # This initiates the connection to the controller and calls # register_engine to tell the controller we are ready to do work self.engine_connector = EngineConnector(self.tub_service) log.msg("Using furl file: %s" % self.master_config.Global.furl_file) reactor.callWhenRunning(self.call_connect) def call_connect(self): d = self.engine_connector.connect_to_controller( self.engine_service, self.master_config.Global.furl_file, self.master_config.Global.connect_delay, self.master_config.Global.connect_max_tries ) def handle_error(f): log.msg('Error connecting to controller. This usually means that ' 'i) the controller was not started, ii) a firewall was blocking ' 'the engine from connecting to the controller or iii) the engine ' ' was not pointed at the right FURL file:') log.msg(f.getErrorMessage()) reactor.callLater(0.1, reactor.stop) d.addErrback(handle_error) def start_mpi(self): global mpi mpikey = self.master_config.MPI.use mpi_import_statement = self.master_config.MPI.get(mpikey, None) if mpi_import_statement is not None: try: self.log.info("Initializing MPI:") self.log.info(mpi_import_statement) exec mpi_import_statement in globals() except: mpi = None else: mpi = None def exec_lines(self): for line in self.master_config.Global.exec_lines: try: log.msg("Executing statement: '%s'" % line) self.engine_service.execute(line) except: log.msg("Error executing statement: %s" % line) def start_app(self): reactor.run()
def start_engine(): """ Start the engine, by creating it and starting the Twisted reactor. This method does: * If it exists, runs the `mpi_import_statement` to call `MPI_Init` * Starts the engine logging * Creates an IPython shell and wraps it in an `EngineService` * Creates a `foolscap.Tub` to use in connecting to a controller. * Uses the tub and the `EngineService` along with a Foolscap URL (or FURL) to connect to the controller and register the engine with the controller """ kernel_config = kernel_config_manager.get_config_obj() core_config = core_config_manager.get_config_obj() # Execute the mpi import statement that needs to call MPI_Init global mpi mpikey = kernel_config['mpi']['default'] mpi_import_statement = kernel_config['mpi'].get(mpikey, None) if mpi_import_statement is not None: try: exec mpi_import_statement in globals() except: mpi = None else: mpi = None # Start logging logfile = kernel_config['engine']['logfile'] if logfile: logfile = logfile + str(os.getpid()) + '.log' try: openLogFile = open(logfile, 'w') except: openLogFile = sys.stdout else: openLogFile = sys.stdout log.startLogging(openLogFile) # Create the underlying shell class and EngineService shell_class = import_item(core_config['shell']['shell_class']) engine_service = EngineService(shell_class, mpi=mpi) shell_import_statement = core_config['shell']['import_statement'] if shell_import_statement: try: engine_service.execute(shell_import_statement) except: log.msg("Error running import_statement: %s" % shell_import_statement) # Create the service hierarchy main_service = service.MultiService() engine_service.setServiceParent(main_service) tub_service = Tub() tub_service.setServiceParent(main_service) # This needs to be called before the connection is initiated main_service.startService() # This initiates the connection to the controller and calls # register_engine to tell the controller we are ready to do work engine_connector = EngineConnector(tub_service) furl_file = kernel_config['engine']['furl_file'] log.msg("Using furl file: %s" % furl_file) def call_connect(engine_service, furl_file): d = engine_connector.connect_to_controller(engine_service, furl_file) def handle_error(f): # If this print statement is replaced by a log.err(f) I get # an unhandled error, which makes no sense. I shouldn't have # to use a print statement here. My only thought is that # at the beginning of the process the logging is still starting up print "error connecting to controller:", f.getErrorMessage() reactor.callLater(0.1, reactor.stop) d.addErrback(handle_error) reactor.callWhenRunning(call_connect, engine_service, furl_file) reactor.run()