def construct(self):
# This is the working dir by now.
sys.path.insert(0, '')
self.start_mpi()
self.start_logging()
# Create the underlying shell class and EngineService
shell_class = import_item(self.master_config.Global.shell_class)
self.engine_service = EngineService(shell_class, mpi=mpi)
self.exec_lines()
# Create the service hierarchy
self.main_service = service.MultiService()
self.engine_service.setServiceParent(self.main_service)
self.tub_service = Tub()
self.tub_service.setServiceParent(self.main_service)
# This needs to be called before the connection is initiated
self.main_service.startService()
# This initiates the connection to the controller and calls
# register_engine to tell the controller we are ready to do work
self.engine_connector = EngineConnector(self.tub_service)
log.msg("Using furl file: %s" % self.master_config.Global.furl_file)
reactor.callWhenRunning(self.call_connect)
def construct(self):
# This is the working dir by now.
sys.path.insert(0, '')
self.start_mpi()
self.start_logging()
# Create the underlying shell class and EngineService
shell_class = import_item(self.master_config.Global.shell_class)
self.engine_service = EngineService(shell_class, mpi=mpi)
self.exec_lines()
# Create the service hierarchy
self.main_service = service.MultiService()
self.engine_service.setServiceParent(self.main_service)
self.tub_service = Tub()
self.tub_service.setServiceParent(self.main_service)
# This needs to be called before the connection is initiated
self.main_service.startService()
# This initiates the connection to the controller and calls
# register_engine to tell the controller we are ready to do work
self.engine_connector = EngineConnector(self.tub_service)
log.msg("Using furl file: %s" % self.master_config.Global.furl_file)
reactor.callWhenRunning(self.call_connect)
def setUp(self):
"""Setup the EngineService and FrontEndBase"""
self.fb = FrontEndCallbackChecker(engine=EngineService())
class IPEngineApp(ApplicationWithClusterDir):
name = u'ipengine'
description = _description
command_line_loader = IPEngineAppConfigLoader
default_config_file_name = default_config_file_name
auto_create_cluster_dir = True
def create_default_config(self):
super(IPEngineApp, self).create_default_config()
# The engine should not clean logs as we don't want to remove the
# active log files of other running engines.
self.default_config.Global.clean_logs = False
# Global config attributes
self.default_config.Global.exec_lines = []
self.default_config.Global.shell_class = 'IPython.kernel.core.interpreter.Interpreter'
# Configuration related to the controller
# This must match the filename (path not included) that the controller
# used for the FURL file.
self.default_config.Global.furl_file_name = u'ipcontroller-engine.furl'
# If given, this is the actual location of the controller's FURL file.
# If not, this is computed using the profile, app_dir and furl_file_name
self.default_config.Global.furl_file = u''
# The max number of connection attemps and the initial delay between
# those attemps.
self.default_config.Global.connect_delay = 0.1
self.default_config.Global.connect_max_tries = 15
# MPI related config attributes
self.default_config.MPI.use = ''
self.default_config.MPI.mpi4py = mpi4py_init
self.default_config.MPI.pytrilinos = pytrilinos_init
def post_load_command_line_config(self):
pass
def pre_construct(self):
super(IPEngineApp, self).pre_construct()
self.find_cont_furl_file()
def find_cont_furl_file(self):
"""Set the furl file.
Here we don't try to actually see if it exists for is valid as that
is hadled by the connection logic.
"""
config = self.master_config
# Find the actual controller FURL file
if not config.Global.furl_file:
try_this = os.path.join(config.Global.cluster_dir,
config.Global.security_dir,
config.Global.furl_file_name)
config.Global.furl_file = try_this
def construct(self):
# This is the working dir by now.
sys.path.insert(0, '')
self.start_mpi()
self.start_logging()
# Create the underlying shell class and EngineService
shell_class = import_item(self.master_config.Global.shell_class)
self.engine_service = EngineService(shell_class, mpi=mpi)
self.exec_lines()
# Create the service hierarchy
self.main_service = service.MultiService()
self.engine_service.setServiceParent(self.main_service)
self.tub_service = Tub()
self.tub_service.setServiceParent(self.main_service)
# This needs to be called before the connection is initiated
self.main_service.startService()
# This initiates the connection to the controller and calls
# register_engine to tell the controller we are ready to do work
self.engine_connector = EngineConnector(self.tub_service)
log.msg("Using furl file: %s" % self.master_config.Global.furl_file)
reactor.callWhenRunning(self.call_connect)
def call_connect(self):
d = self.engine_connector.connect_to_controller(
self.engine_service, self.master_config.Global.furl_file,
self.master_config.Global.connect_delay,
self.master_config.Global.connect_max_tries)
def handle_error(f):
log.msg(
'Error connecting to controller. This usually means that '
'i) the controller was not started, ii) a firewall was blocking '
'the engine from connecting to the controller or iii) the engine '
' was not pointed at the right FURL file:')
log.msg(f.getErrorMessage())
reactor.callLater(0.1, reactor.stop)
d.addErrback(handle_error)
def start_mpi(self):
global mpi
mpikey = self.master_config.MPI.use
mpi_import_statement = self.master_config.MPI.get(mpikey, None)
if mpi_import_statement is not None:
try:
self.log.info("Initializing MPI:")
self.log.info(mpi_import_statement)
exec mpi_import_statement in globals()
except:
mpi = None
else:
mpi = None
def exec_lines(self):
for line in self.master_config.Global.exec_lines:
try:
log.msg("Executing statement: '%s'" % line)
self.engine_service.execute(line)
except:
log.msg("Error executing statement: %s" % line)
def start_app(self):
reactor.run()
def start_engine():
"""
Start the engine, by creating it and starting the Twisted reactor.
This method does:
* If it exists, runs the `mpi_import_statement` to call `MPI_Init`
* Starts the engine logging
* Creates an IPython shell and wraps it in an `EngineService`
* Creates a `foolscap.Tub` to use in connecting to a controller.
* Uses the tub and the `EngineService` along with a Foolscap URL
(or FURL) to connect to the controller and register the engine
with the controller
"""
kernel_config = kernel_config_manager.get_config_obj()
core_config = core_config_manager.get_config_obj()
# Execute the mpi import statement that needs to call MPI_Init
global mpi
mpikey = kernel_config['mpi']['default']
mpi_import_statement = kernel_config['mpi'].get(mpikey, None)
if mpi_import_statement is not None:
try:
exec mpi_import_statement in globals()
except:
mpi = None
else:
mpi = None
# Start logging
logfile = kernel_config['engine']['logfile']
if logfile:
logfile = logfile + str(os.getpid()) + '.log'
try:
openLogFile = open(logfile, 'w')
except:
openLogFile = sys.stdout
else:
openLogFile = sys.stdout
log.startLogging(openLogFile)
# Create the underlying shell class and EngineService
shell_class = import_item(core_config['shell']['shell_class'])
engine_service = EngineService(shell_class, mpi=mpi)
shell_import_statement = core_config['shell']['import_statement']
if shell_import_statement:
try:
engine_service.execute(shell_import_statement)
except:
log.msg("Error running import_statement: %s" % sis)
# Create the service hierarchy
main_service = service.MultiService()
engine_service.setServiceParent(main_service)
tub_service = Tub()
tub_service.setServiceParent(main_service)
# This needs to be called before the connection is initiated
main_service.startService()
# This initiates the connection to the controller and calls
# register_engine to tell the controller we are ready to do work
engine_connector = EngineConnector(tub_service)
furl_file = kernel_config['engine']['furl_file']
log.msg("Using furl file: %s" % furl_file)
d = engine_connector.connect_to_controller(engine_service, furl_file)
d.addErrback(lambda _: reactor.stop())
reactor.run()
class IPEngineApp(ApplicationWithClusterDir):
name = u'ipengine'
description = _description
command_line_loader = IPEngineAppConfigLoader
default_config_file_name = default_config_file_name
auto_create_cluster_dir = True
def create_default_config(self):
super(IPEngineApp, self).create_default_config()
# The engine should not clean logs as we don't want to remove the
# active log files of other running engines.
self.default_config.Global.clean_logs = False
# Global config attributes
self.default_config.Global.exec_lines = []
self.default_config.Global.shell_class = 'IPython.kernel.core.interpreter.Interpreter'
# Configuration related to the controller
# This must match the filename (path not included) that the controller
# used for the FURL file.
self.default_config.Global.furl_file_name = u'ipcontroller-engine.furl'
# If given, this is the actual location of the controller's FURL file.
# If not, this is computed using the profile, app_dir and furl_file_name
self.default_config.Global.furl_file = u''
# The max number of connection attemps and the initial delay between
# those attemps.
self.default_config.Global.connect_delay = 0.1
self.default_config.Global.connect_max_tries = 15
# MPI related config attributes
self.default_config.MPI.use = ''
self.default_config.MPI.mpi4py = mpi4py_init
self.default_config.MPI.pytrilinos = pytrilinos_init
def post_load_command_line_config(self):
pass
def pre_construct(self):
super(IPEngineApp, self).pre_construct()
self.find_cont_furl_file()
def find_cont_furl_file(self):
"""Set the furl file.
Here we don't try to actually see if it exists for is valid as that
is hadled by the connection logic.
"""
config = self.master_config
# Find the actual controller FURL file
if not config.Global.furl_file:
try_this = os.path.join(
config.Global.cluster_dir,
config.Global.security_dir,
config.Global.furl_file_name
)
config.Global.furl_file = try_this
def construct(self):
# This is the working dir by now.
sys.path.insert(0, '')
self.start_mpi()
self.start_logging()
# Create the underlying shell class and EngineService
shell_class = import_item(self.master_config.Global.shell_class)
self.engine_service = EngineService(shell_class, mpi=mpi)
self.exec_lines()
# Create the service hierarchy
self.main_service = service.MultiService()
self.engine_service.setServiceParent(self.main_service)
self.tub_service = Tub()
self.tub_service.setServiceParent(self.main_service)
# This needs to be called before the connection is initiated
self.main_service.startService()
# This initiates the connection to the controller and calls
# register_engine to tell the controller we are ready to do work
self.engine_connector = EngineConnector(self.tub_service)
log.msg("Using furl file: %s" % self.master_config.Global.furl_file)
reactor.callWhenRunning(self.call_connect)
def call_connect(self):
d = self.engine_connector.connect_to_controller(
self.engine_service,
self.master_config.Global.furl_file,
self.master_config.Global.connect_delay,
self.master_config.Global.connect_max_tries
)
def handle_error(f):
log.msg('Error connecting to controller. This usually means that '
'i) the controller was not started, ii) a firewall was blocking '
'the engine from connecting to the controller or iii) the engine '
' was not pointed at the right FURL file:')
log.msg(f.getErrorMessage())
reactor.callLater(0.1, reactor.stop)
d.addErrback(handle_error)
def start_mpi(self):
global mpi
mpikey = self.master_config.MPI.use
mpi_import_statement = self.master_config.MPI.get(mpikey, None)
if mpi_import_statement is not None:
try:
self.log.info("Initializing MPI:")
self.log.info(mpi_import_statement)
exec mpi_import_statement in globals()
except:
mpi = None
else:
mpi = None
def exec_lines(self):
for line in self.master_config.Global.exec_lines:
try:
log.msg("Executing statement: '%s'" % line)
self.engine_service.execute(line)
except:
log.msg("Error executing statement: %s" % line)
def start_app(self):
reactor.run()
def start_engine():
"""
Start the engine, by creating it and starting the Twisted reactor.
This method does:
* If it exists, runs the `mpi_import_statement` to call `MPI_Init`
* Starts the engine logging
* Creates an IPython shell and wraps it in an `EngineService`
* Creates a `foolscap.Tub` to use in connecting to a controller.
* Uses the tub and the `EngineService` along with a Foolscap URL
(or FURL) to connect to the controller and register the engine
with the controller
"""
kernel_config = kernel_config_manager.get_config_obj()
core_config = core_config_manager.get_config_obj()
# Execute the mpi import statement that needs to call MPI_Init
global mpi
mpikey = kernel_config['mpi']['default']
mpi_import_statement = kernel_config['mpi'].get(mpikey, None)
if mpi_import_statement is not None:
try:
exec mpi_import_statement in globals()
except:
mpi = None
else:
mpi = None
# Start logging
logfile = kernel_config['engine']['logfile']
if logfile:
logfile = logfile + str(os.getpid()) + '.log'
try:
openLogFile = open(logfile, 'w')
except:
openLogFile = sys.stdout
else:
openLogFile = sys.stdout
log.startLogging(openLogFile)
# Create the underlying shell class and EngineService
shell_class = import_item(core_config['shell']['shell_class'])
engine_service = EngineService(shell_class, mpi=mpi)
shell_import_statement = core_config['shell']['import_statement']
if shell_import_statement:
try:
engine_service.execute(shell_import_statement)
except:
log.msg("Error running import_statement: %s" % shell_import_statement)
# Create the service hierarchy
main_service = service.MultiService()
engine_service.setServiceParent(main_service)
tub_service = Tub()
tub_service.setServiceParent(main_service)
# This needs to be called before the connection is initiated
main_service.startService()
# This initiates the connection to the controller and calls
# register_engine to tell the controller we are ready to do work
engine_connector = EngineConnector(tub_service)
furl_file = kernel_config['engine']['furl_file']
log.msg("Using furl file: %s" % furl_file)
def call_connect(engine_service, furl_file):
d = engine_connector.connect_to_controller(engine_service, furl_file)
def handle_error(f):
# If this print statement is replaced by a log.err(f) I get
# an unhandled error, which makes no sense. I shouldn't have
# to use a print statement here. My only thought is that
# at the beginning of the process the logging is still starting up
print "error connecting to controller:", f.getErrorMessage()
reactor.callLater(0.1, reactor.stop)
d.addErrback(handle_error)
reactor.callWhenRunning(call_connect, engine_service, furl_file)
reactor.run()
def start_engine():
"""
Start the engine, by creating it and starting the Twisted reactor.
This method does:
* If it exists, runs the `mpi_import_statement` to call `MPI_Init`
* Starts the engine logging
* Creates an IPython shell and wraps it in an `EngineService`
* Creates a `foolscap.Tub` to use in connecting to a controller.
* Uses the tub and the `EngineService` along with a Foolscap URL
(or FURL) to connect to the controller and register the engine
with the controller
"""
kernel_config = kernel_config_manager.get_config_obj()
core_config = core_config_manager.get_config_obj()
# Execute the mpi import statement that needs to call MPI_Init
global mpi
mpikey = kernel_config['mpi']['default']
mpi_import_statement = kernel_config['mpi'].get(mpikey, None)
if mpi_import_statement is not None:
try:
exec mpi_import_statement in globals()
except:
mpi = None
else:
mpi = None
# Start logging
logfile = kernel_config['engine']['logfile']
if logfile:
logfile = logfile + str(os.getpid()) + '.log'
try:
openLogFile = open(logfile, 'w')
except:
openLogFile = sys.stdout
else:
openLogFile = sys.stdout
log.startLogging(openLogFile)
# Create the underlying shell class and EngineService
shell_class = import_item(core_config['shell']['shell_class'])
engine_service = EngineService(shell_class, mpi=mpi)
shell_import_statement = core_config['shell']['import_statement']
if shell_import_statement:
try:
engine_service.execute(shell_import_statement)
except:
log.msg("Error running import_statement: %s" % sis)
# Create the service hierarchy
main_service = service.MultiService()
engine_service.setServiceParent(main_service)
tub_service = Tub()
tub_service.setServiceParent(main_service)
# This needs to be called before the connection is initiated
main_service.startService()
# This initiates the connection to the controller and calls
# register_engine to tell the controller we are ready to do work
engine_connector = EngineConnector(tub_service)
furl_file = kernel_config['engine']['furl_file']
log.msg("Using furl file: %s" % furl_file)
d = engine_connector.connect_to_controller(engine_service, furl_file)
d.addErrback(lambda _: reactor.stop())
reactor.run()
def start_engine():
"""
Start the engine, by creating it and starting the Twisted reactor.
This method does:
* If it exists, runs the `mpi_import_statement` to call `MPI_Init`
* Starts the engine logging
* Creates an IPython shell and wraps it in an `EngineService`
* Creates a `foolscap.Tub` to use in connecting to a controller.
* Uses the tub and the `EngineService` along with a Foolscap URL
(or FURL) to connect to the controller and register the engine
with the controller
"""
kernel_config = kernel_config_manager.get_config_obj()
core_config = core_config_manager.get_config_obj()
# Execute the mpi import statement that needs to call MPI_Init
global mpi
mpikey = kernel_config['mpi']['default']
mpi_import_statement = kernel_config['mpi'].get(mpikey, None)
if mpi_import_statement is not None:
try:
exec mpi_import_statement in globals()
except:
mpi = None
else:
mpi = None
# Start logging
logfile = kernel_config['engine']['logfile']
if logfile:
logfile = logfile + str(os.getpid()) + '.log'
try:
openLogFile = open(logfile, 'w')
except:
openLogFile = sys.stdout
else:
openLogFile = sys.stdout
log.startLogging(openLogFile)
# Create the underlying shell class and EngineService
shell_class = import_item(core_config['shell']['shell_class'])
engine_service = EngineService(shell_class, mpi=mpi)
shell_import_statement = core_config['shell']['import_statement']
if shell_import_statement:
try:
engine_service.execute(shell_import_statement)
except:
log.msg("Error running import_statement: %s" %
shell_import_statement)
# Create the service hierarchy
main_service = service.MultiService()
engine_service.setServiceParent(main_service)
tub_service = Tub()
tub_service.setServiceParent(main_service)
# This needs to be called before the connection is initiated
main_service.startService()
# This initiates the connection to the controller and calls
# register_engine to tell the controller we are ready to do work
engine_connector = EngineConnector(tub_service)
furl_file = kernel_config['engine']['furl_file']
log.msg("Using furl file: %s" % furl_file)
def call_connect(engine_service, furl_file):
d = engine_connector.connect_to_controller(engine_service, furl_file)
def handle_error(f):
# If this print statement is replaced by a log.err(f) I get
# an unhandled error, which makes no sense. I shouldn't have
# to use a print statement here. My only thought is that
# at the beginning of the process the logging is still starting up
print "error connecting to controller:", f.getErrorMessage()
reactor.callLater(0.1, reactor.stop)
d.addErrback(handle_error)
reactor.callWhenRunning(call_connect, engine_service, furl_file)
reactor.run()
