def main_pbs(args):
cont_args = []
cont_args.append('--logfile=%s' % pjoin(args.logdir, 'ipcontroller'))
# Check security settings before proceeding
if not check_security(args, cont_args):
return
# See if we are reusing FURL files
if not check_reuse(args, cont_args):
return
cl = ControllerLauncher(extra_args=cont_args)
dstart = cl.start()
def start_engines(r):
pbs_set = PBSEngineSet(args.pbsscript)
def shutdown(signum, frame):
log.msg('Stopping pbs cluster')
d = pbs_set.kill()
d.addBoth(lambda _: cl.interrupt_then_kill(1.0))
d.addBoth(lambda _: reactor.callLater(2.0, reactor.stop))
signal.signal(signal.SIGINT, shutdown)
d = pbs_set.start(args.n)
return d
config = kernel_config_manager.get_config_obj()
furl_file = config['controller']['engine_furl_file']
dstart.addCallback(_delay_start, start_engines, furl_file, args.r)
dstart.addErrback(_err_and_stop)
def main_local(args):
cont_args = []
cont_args.append('--logfile=%s' % pjoin(args.logdir,'ipcontroller'))
# Check security settings before proceeding
if not check_security(args, cont_args):
return
# See if we are reusing FURL files
if not check_reuse(args, cont_args):
return
cl = ControllerLauncher(extra_args=cont_args)
dstart = cl.start()
def start_engines(cont_pid):
engine_args = []
engine_args.append('--logfile=%s' % \
pjoin(args.logdir,'ipengine%s-' % cont_pid))
eset = LocalEngineSet(extra_args=engine_args)
def shutdown(signum, frame):
log.msg('Stopping local cluster')
# We are still playing with the times here, but these seem
# to be reliable in allowing everything to exit cleanly.
eset.interrupt_then_kill(0.5)
cl.interrupt_then_kill(0.5)
reactor.callLater(1.0, reactor.stop)
signal.signal(signal.SIGINT,shutdown)
d = eset.start(args.n)
return d
config = kernel_config_manager.get_config_obj()
furl_file = config['controller']['engine_furl_file']
dstart.addCallback(_delay_start, start_engines, furl_file, args.r)
dstart.addErrback(_err_and_stop)
def main_pbs(args):
cont_args = []
cont_args.append('--logfile=%s' % pjoin(args.logdir,'ipcontroller'))
# Check security settings before proceeding
if not check_security(args, cont_args):
return
# See if we are reusing FURL files
if not check_reuse(args, cont_args):
return
cl = ControllerLauncher(extra_args=cont_args)
dstart = cl.start()
def start_engines(r):
pbs_set = PBSEngineSet(args.pbsscript)
def shutdown(signum, frame):
log.msg('Stopping pbs cluster')
d = pbs_set.kill()
d.addBoth(lambda _: cl.interrupt_then_kill(1.0))
d.addBoth(lambda _: reactor.callLater(2.0, reactor.stop))
signal.signal(signal.SIGINT,shutdown)
d = pbs_set.start(args.n)
return d
config = kernel_config_manager.get_config_obj()
furl_file = config['controller']['engine_furl_file']
dstart.addCallback(_delay_start, start_engines, furl_file, args.r)
dstart.addErrback(_err_and_stop)
def start_controller():
"""
Start the controller by creating the service hierarchy and starting the reactor.
This method does the following:
* It starts the controller logging
* In execute an import statement for the controller
* It creates 2 `foolscap.Tub` instances for the client and the engines
and registers `foolscap.Referenceables` with the tubs to expose the
controller to engines and clients.
"""
config = kernel_config_manager.get_config_obj()
# Start logging
logfile = config['controller']['logfile']
if logfile:
logfile = logfile + str(os.getpid()) + '.log'
try:
openLogFile = open(logfile, 'w')
except:
openLogFile = sys.stdout
else:
openLogFile = sys.stdout
log.startLogging(openLogFile)
# Execute any user defined import statements
cis = config['controller']['import_statement']
if cis:
try:
exec cis in globals(), locals()
except:
log.msg("Error running import_statement: %s" % cis)
# Delete old furl files unless the reuse_furls is set
reuse = config['controller']['reuse_furls']
if not reuse:
paths = (config['controller']['engine_furl_file'],
config['controller']['controller_interfaces']['task']['furl_file'],
config['controller']['controller_interfaces']['multiengine']['furl_file']
)
for p in paths:
if os.path.isfile(p):
os.remove(p)
# Create the service hierarchy
main_service = service.MultiService()
# The controller service
controller_service = controllerservice.ControllerService()
controller_service.setServiceParent(main_service)
# The client tub and all its refereceables
client_service = make_client_service(controller_service, config)
client_service.setServiceParent(main_service)
# The engine tub
engine_service = make_engine_service(controller_service, config)
engine_service.setServiceParent(main_service)
# Start the controller service and set things running
main_service.startService()
reactor.run()
def main_ssh(args):
"""Start a controller on localhost and engines using ssh.
Your clusterfile should look like::
send_furl = False # True, if you want
engines = {
'engine_host1' : engine_count,
'engine_host2' : engine_count2
}
"""
clusterfile = {}
execfile(args.clusterfile, clusterfile)
if not clusterfile.has_key('send_furl'):
clusterfile['send_furl'] = False
cont_args = []
cont_args.append('--logfile=%s' % pjoin(args.logdir, 'ipcontroller'))
# Check security settings before proceeding
if not check_security(args, cont_args):
return
# See if we are reusing FURL files
if not check_reuse(args, cont_args):
return
cl = ControllerLauncher(extra_args=cont_args)
dstart = cl.start()
def start_engines(cont_pid):
ssh_set = SSHEngineSet(clusterfile['engines'],
sshx=args.sshx,
copyenvs=args.copyenvs)
def shutdown(signum, frame):
d = ssh_set.kill()
cl.interrupt_then_kill(1.0)
reactor.callLater(2.0, reactor.stop)
signal.signal(signal.SIGINT, shutdown)
d = ssh_set.start(clusterfile['send_furl'])
return d
config = kernel_config_manager.get_config_obj()
furl_file = config['controller']['engine_furl_file']
dstart.addCallback(_delay_start, start_engines, furl_file, args.r)
dstart.addErrback(_err_and_stop)
def init_config():
"""
Initialize the configuration using default and command line options.
"""
parser = OptionParser()
parser.add_option(
"--furl-file",
type="string",
dest="furl_file",
help="The filename containing the FURL of the controller"
)
parser.add_option(
"--mpi",
type="string",
dest="mpi",
help="How to enable MPI (mpi4py, pytrilinos, or empty string to disable)"
)
parser.add_option(
"-l",
"--logfile",
type="string",
dest="logfile",
help="log file name (default is stdout)"
)
parser.add_option(
"--ipythondir",
type="string",
dest="ipythondir",
help="look for config files and profiles in this directory"
)
(options, args) = parser.parse_args()
kernel_config_manager.update_config_obj_from_default_file(options.ipythondir)
core_config_manager.update_config_obj_from_default_file(options.ipythondir)
kernel_config = kernel_config_manager.get_config_obj()
# Now override with command line options
if options.furl_file is not None:
kernel_config['engine']['furl_file'] = options.furl_file
if options.logfile is not None:
kernel_config['engine']['logfile'] = options.logfile
if options.mpi is not None:
kernel_config['mpi']['default'] = options.mpi
def init_config():
"""
Initialize the configuration using default and command line options.
"""
parser = OptionParser("""ipengine [options]
Start an IPython engine.
Use the IPYTHONDIR environment variable to change your IPython directory
from the default of .ipython or _ipython. The log and security
subdirectories of your IPython directory will be used by this script
for log files and security files.""")
parser.add_option(
"--furl-file",
type="string",
dest="furl_file",
help="The filename containing the FURL of the controller"
)
parser.add_option(
"--mpi",
type="string",
dest="mpi",
help="How to enable MPI (mpi4py, pytrilinos, or empty string to disable)"
)
parser.add_option(
"-l",
"--logfile",
type="string",
dest="logfile",
help="log file name (default is stdout)"
)
(options, args) = parser.parse_args()
kernel_config = kernel_config_manager.get_config_obj()
# Now override with command line options
if options.furl_file is not None:
kernel_config['engine']['furl_file'] = options.furl_file
if options.logfile is not None:
kernel_config['engine']['logfile'] = options.logfile
if options.mpi is not None:
kernel_config['mpi']['default'] = options.mpi
def main_ssh(args):
"""Start a controller on localhost and engines using ssh.
Your clusterfile should look like::
send_furl = False # True, if you want
engines = {
'engine_host1' : engine_count,
'engine_host2' : engine_count2
}
"""
clusterfile = {}
execfile(args.clusterfile, clusterfile)
if not clusterfile.has_key('send_furl'):
clusterfile['send_furl'] = False
cont_args = []
cont_args.append('--logfile=%s' % pjoin(args.logdir,'ipcontroller'))
# Check security settings before proceeding
if not check_security(args, cont_args):
return
# See if we are reusing FURL files
if not check_reuse(args, cont_args):
return
cl = ControllerLauncher(extra_args=cont_args)
dstart = cl.start()
def start_engines(cont_pid):
ssh_set = SSHEngineSet(clusterfile['engines'], sshx=args.sshx,
copyenvs=args.copyenvs)
def shutdown(signum, frame):
d = ssh_set.kill()
cl.interrupt_then_kill(1.0)
reactor.callLater(2.0, reactor.stop)
signal.signal(signal.SIGINT,shutdown)
d = ssh_set.start(clusterfile['send_furl'])
return d
config = kernel_config_manager.get_config_obj()
furl_file = config['controller']['engine_furl_file']
dstart.addCallback(_delay_start, start_engines, furl_file, args.r)
dstart.addErrback(_err_and_stop)
def init_config():
"""
Initialize the configuration using default and command line options.
"""
parser = OptionParser()
parser.add_option(
"--furl-file",
type="string",
dest="furl_file",
help="The filename containing the FURL of the controller")
parser.add_option(
"--mpi",
type="string",
dest="mpi",
help=
"How to enable MPI (mpi4py, pytrilinos, or empty string to disable)")
parser.add_option("-l",
"--logfile",
type="string",
dest="logfile",
help="log file name (default is stdout)")
parser.add_option(
"--ipythondir",
type="string",
dest="ipythondir",
help="look for config files and profiles in this directory")
(options, args) = parser.parse_args()
kernel_config_manager.update_config_obj_from_default_file(
options.ipythondir)
core_config_manager.update_config_obj_from_default_file(options.ipythondir)
kernel_config = kernel_config_manager.get_config_obj()
# Now override with command line options
if options.furl_file is not None:
kernel_config['engine']['furl_file'] = options.furl_file
if options.logfile is not None:
kernel_config['engine']['logfile'] = options.logfile
if options.mpi is not None:
kernel_config['mpi']['default'] = options.mpi
def main_mpi(args):
cont_args = []
cont_args.append('--logfile=%s' % pjoin(args.logdir, 'ipcontroller'))
# Check security settings before proceeding
if not check_security(args, cont_args):
return
# See if we are reusing FURL files
if not check_reuse(args, cont_args):
return
cl = ControllerLauncher(extra_args=cont_args)
dstart = cl.start()
def start_engines(cont_pid):
raw_args = [args.cmd]
raw_args.extend(['-n', str(args.n)])
raw_args.append('ipengine')
raw_args.append('-l')
raw_args.append(pjoin(args.logdir, 'ipengine%s-' % cont_pid))
if args.mpi:
raw_args.append('--mpi=%s' % args.mpi)
eset = ProcessLauncher(raw_args)
def shutdown(signum, frame):
log.msg('Stopping local cluster')
# We are still playing with the times here, but these seem
# to be reliable in allowing everything to exit cleanly.
eset.interrupt_then_kill(1.0)
cl.interrupt_then_kill(1.0)
reactor.callLater(2.0, reactor.stop)
signal.signal(signal.SIGINT, shutdown)
d = eset.start()
return d
config = kernel_config_manager.get_config_obj()
furl_file = config['controller']['engine_furl_file']
dstart.addCallback(_delay_start, start_engines, furl_file, args.r)
dstart.addErrback(_err_and_stop)
def init_config():
"""
Initialize the configuration using default and command line options.
"""
parser = OptionParser("""ipengine [options]
Start an IPython engine.
Use the IPYTHONDIR environment variable to change your IPython directory
from the default of .ipython or _ipython. The log and security
subdirectories of your IPython directory will be used by this script
for log files and security files.""")
parser.add_option(
"--furl-file",
type="string",
dest="furl_file",
help="The filename containing the FURL of the controller")
parser.add_option(
"--mpi",
type="string",
dest="mpi",
help=
"How to enable MPI (mpi4py, pytrilinos, or empty string to disable)")
parser.add_option("-l",
"--logfile",
type="string",
dest="logfile",
help="log file name (default is stdout)")
(options, args) = parser.parse_args()
kernel_config = kernel_config_manager.get_config_obj()
# Now override with command line options
if options.furl_file is not None:
kernel_config['engine']['furl_file'] = options.furl_file
if options.logfile is not None:
kernel_config['engine']['logfile'] = options.logfile
if options.mpi is not None:
kernel_config['mpi']['default'] = options.mpi
def init_config():
"""
Initialize the configuration using default and command line options.
"""
parser = OptionParser("""ipcontroller [options]
Start an IPython controller.
Use the IPYTHONDIR environment variable to change your IPython directory
from the default of .ipython or _ipython. The log and security
subdirectories of your IPython directory will be used by this script
for log files and security files.""")
# Client related options
parser.add_option(
"--client-ip",
type="string",
dest="client_ip",
help="the IP address or hostname the controller will listen on for client connections"
)
parser.add_option(
"--client-port",
type="int",
dest="client_port",
help="the port the controller will listen on for client connections"
)
parser.add_option(
'--client-location',
type="string",
dest="client_location",
help="hostname or ip for clients to connect to"
)
parser.add_option(
"-x",
action="store_false",
dest="client_secure",
help="turn off all client security"
)
parser.add_option(
'--client-cert-file',
type="string",
dest="client_cert_file",
help="file to store the client SSL certificate"
)
parser.add_option(
'--task-furl-file',
type="string",
dest="task_furl_file",
help="file to store the FURL for task clients to connect with"
)
parser.add_option(
'--multiengine-furl-file',
type="string",
dest="multiengine_furl_file",
help="file to store the FURL for multiengine clients to connect with"
)
# Engine related options
parser.add_option(
"--engine-ip",
type="string",
dest="engine_ip",
help="the IP address or hostname the controller will listen on for engine connections"
)
parser.add_option(
"--engine-port",
type="int",
dest="engine_port",
help="the port the controller will listen on for engine connections"
)
parser.add_option(
'--engine-location',
type="string",
dest="engine_location",
help="hostname or ip for engines to connect to"
)
parser.add_option(
"-y",
action="store_false",
dest="engine_secure",
help="turn off all engine security"
)
parser.add_option(
'--engine-cert-file',
type="string",
dest="engine_cert_file",
help="file to store the engine SSL certificate"
)
parser.add_option(
'--engine-furl-file',
type="string",
dest="engine_furl_file",
help="file to store the FURL for engines to connect with"
)
parser.add_option(
"-l", "--logfile",
type="string",
dest="logfile",
help="log file name (default is stdout)"
)
parser.add_option(
"-r",
action="store_true",
dest="reuse_furls",
help="try to reuse all furl files"
)
(options, args) = parser.parse_args()
config = kernel_config_manager.get_config_obj()
# Update with command line options
if options.client_ip is not None:
config['controller']['client_tub']['ip'] = options.client_ip
if options.client_port is not None:
config['controller']['client_tub']['port'] = options.client_port
if options.client_location is not None:
config['controller']['client_tub']['location'] = options.client_location
if options.client_secure is not None:
config['controller']['client_tub']['secure'] = options.client_secure
if options.client_cert_file is not None:
config['controller']['client_tub']['cert_file'] = options.client_cert_file
if options.task_furl_file is not None:
config['controller']['controller_interfaces']['task']['furl_file'] = options.task_furl_file
if options.multiengine_furl_file is not None:
config['controller']['controller_interfaces']['multiengine']['furl_file'] = options.multiengine_furl_file
if options.engine_ip is not None:
config['controller']['engine_tub']['ip'] = options.engine_ip
if options.engine_port is not None:
config['controller']['engine_tub']['port'] = options.engine_port
if options.engine_location is not None:
config['controller']['engine_tub']['location'] = options.engine_location
if options.engine_secure is not None:
config['controller']['engine_tub']['secure'] = options.engine_secure
if options.engine_cert_file is not None:
config['controller']['engine_tub']['cert_file'] = options.engine_cert_file
if options.engine_furl_file is not None:
config['controller']['engine_furl_file'] = options.engine_furl_file
if options.reuse_furls is not None:
config['controller']['reuse_furls'] = options.reuse_furls
if options.logfile is not None:
config['controller']['logfile'] = options.logfile
kernel_config_manager.update_config_obj(config)
# the file COPYING, distributed as part of this software.
#-------------------------------------------------------------------------------
#-------------------------------------------------------------------------------
# Imports
#-------------------------------------------------------------------------------
from twisted.internet import defer
from IPython.kernel.fcutil import Tub, UnauthenticatedTub
from IPython.kernel.config import config_manager as kernel_config_manager
from IPython.config.cutils import import_item
from IPython.kernel.fcutil import find_furl
co = kernel_config_manager.get_config_obj()
client_co = co['client']
#-------------------------------------------------------------------------------
# The ClientConnector class
#-------------------------------------------------------------------------------
class ClientConnector(object):
"""
This class gets remote references from furls and returns the wrapped clients.
This class is also used in `client.py` and `asyncclient.py` to create
a single per client-process Tub.
"""
def __init__(self):
def start_engine():
"""
Start the engine, by creating it and starting the Twisted reactor.
This method does:
* If it exists, runs the `mpi_import_statement` to call `MPI_Init`
* Starts the engine logging
* Creates an IPython shell and wraps it in an `EngineService`
* Creates a `foolscap.Tub` to use in connecting to a controller.
* Uses the tub and the `EngineService` along with a Foolscap URL
(or FURL) to connect to the controller and register the engine
with the controller
"""
kernel_config = kernel_config_manager.get_config_obj()
core_config = core_config_manager.get_config_obj()
# Execute the mpi import statement that needs to call MPI_Init
global mpi
mpikey = kernel_config['mpi']['default']
mpi_import_statement = kernel_config['mpi'].get(mpikey, None)
if mpi_import_statement is not None:
try:
exec mpi_import_statement in globals()
except:
mpi = None
else:
mpi = None
# Start logging
logfile = kernel_config['engine']['logfile']
if logfile:
logfile = logfile + str(os.getpid()) + '.log'
try:
openLogFile = open(logfile, 'w')
except:
openLogFile = sys.stdout
else:
openLogFile = sys.stdout
log.startLogging(openLogFile)
# Create the underlying shell class and EngineService
shell_class = import_item(core_config['shell']['shell_class'])
engine_service = EngineService(shell_class, mpi=mpi)
shell_import_statement = core_config['shell']['import_statement']
if shell_import_statement:
try:
engine_service.execute(shell_import_statement)
except:
log.msg("Error running import_statement: %s" % sis)
# Create the service hierarchy
main_service = service.MultiService()
engine_service.setServiceParent(main_service)
tub_service = Tub()
tub_service.setServiceParent(main_service)
# This needs to be called before the connection is initiated
main_service.startService()
# This initiates the connection to the controller and calls
# register_engine to tell the controller we are ready to do work
engine_connector = EngineConnector(tub_service)
furl_file = kernel_config['engine']['furl_file']
log.msg("Using furl file: %s" % furl_file)
d = engine_connector.connect_to_controller(engine_service, furl_file)
d.addErrback(lambda _: reactor.stop())
reactor.run()
def init_config():
"""
Initialize the configuration using default and command line options.
"""
parser = OptionParser()
# Client related options
parser.add_option(
"--client-ip",
type="string",
dest="client_ip",
help="the IP address or hostname the controller will listen on for client connections"
)
parser.add_option(
"--client-port",
type="int",
dest="client_port",
help="the port the controller will listen on for client connections"
)
parser.add_option(
'--client-location',
type="string",
dest="client_location",
help="hostname or ip for clients to connect to"
)
parser.add_option(
"-x",
action="store_false",
dest="client_secure",
help="turn off all client security"
)
parser.add_option(
'--client-cert-file',
type="string",
dest="client_cert_file",
help="file to store the client SSL certificate"
)
parser.add_option(
'--task-furl-file',
type="string",
dest="task_furl_file",
help="file to store the FURL for task clients to connect with"
)
parser.add_option(
'--multiengine-furl-file',
type="string",
dest="multiengine_furl_file",
help="file to store the FURL for multiengine clients to connect with"
)
# Engine related options
parser.add_option(
"--engine-ip",
type="string",
dest="engine_ip",
help="the IP address or hostname the controller will listen on for engine connections"
)
parser.add_option(
"--engine-port",
type="int",
dest="engine_port",
help="the port the controller will listen on for engine connections"
)
parser.add_option(
'--engine-location',
type="string",
dest="engine_location",
help="hostname or ip for engines to connect to"
)
parser.add_option(
"-y",
action="store_false",
dest="engine_secure",
help="turn off all engine security"
)
parser.add_option(
'--engine-cert-file',
type="string",
dest="engine_cert_file",
help="file to store the engine SSL certificate"
)
parser.add_option(
'--engine-furl-file',
type="string",
dest="engine_furl_file",
help="file to store the FURL for engines to connect with"
)
parser.add_option(
"-l", "--logfile",
type="string",
dest="logfile",
help="log file name (default is stdout)"
)
parser.add_option(
"--ipythondir",
type="string",
dest="ipythondir",
help="look for config files and profiles in this directory"
)
(options, args) = parser.parse_args()
kernel_config_manager.update_config_obj_from_default_file(options.ipythondir)
config = kernel_config_manager.get_config_obj()
# Update with command line options
if options.client_ip is not None:
config['controller']['client_tub']['ip'] = options.client_ip
if options.client_port is not None:
config['controller']['client_tub']['port'] = options.client_port
if options.client_location is not None:
config['controller']['client_tub']['location'] = options.client_location
if options.client_secure is not None:
config['controller']['client_tub']['secure'] = options.client_secure
if options.client_cert_file is not None:
config['controller']['client_tub']['cert_file'] = options.client_cert_file
if options.task_furl_file is not None:
config['controller']['controller_interfaces']['task']['furl_file'] = options.task_furl_file
if options.multiengine_furl_file is not None:
config['controller']['controller_interfaces']['multiengine']['furl_file'] = options.multiengine_furl_file
if options.engine_ip is not None:
config['controller']['engine_tub']['ip'] = options.engine_ip
if options.engine_port is not None:
config['controller']['engine_tub']['port'] = options.engine_port
if options.engine_location is not None:
config['controller']['engine_tub']['location'] = options.engine_location
if options.engine_secure is not None:
config['controller']['engine_tub']['secure'] = options.engine_secure
if options.engine_cert_file is not None:
config['controller']['engine_tub']['cert_file'] = options.engine_cert_file
if options.engine_furl_file is not None:
config['controller']['engine_furl_file'] = options.engine_furl_file
if options.logfile is not None:
config['controller']['logfile'] = options.logfile
kernel_config_manager.update_config_obj(config)
def start_engine():
"""
Start the engine, by creating it and starting the Twisted reactor.
This method does:
* If it exists, runs the `mpi_import_statement` to call `MPI_Init`
* Starts the engine logging
* Creates an IPython shell and wraps it in an `EngineService`
* Creates a `foolscap.Tub` to use in connecting to a controller.
* Uses the tub and the `EngineService` along with a Foolscap URL
(or FURL) to connect to the controller and register the engine
with the controller
"""
kernel_config = kernel_config_manager.get_config_obj()
core_config = core_config_manager.get_config_obj()
# Execute the mpi import statement that needs to call MPI_Init
global mpi
mpikey = kernel_config['mpi']['default']
mpi_import_statement = kernel_config['mpi'].get(mpikey, None)
if mpi_import_statement is not None:
try:
exec mpi_import_statement in globals()
except:
mpi = None
else:
mpi = None
# Start logging
logfile = kernel_config['engine']['logfile']
if logfile:
logfile = logfile + str(os.getpid()) + '.log'
try:
openLogFile = open(logfile, 'w')
except:
openLogFile = sys.stdout
else:
openLogFile = sys.stdout
log.startLogging(openLogFile)
# Create the underlying shell class and EngineService
shell_class = import_item(core_config['shell']['shell_class'])
engine_service = EngineService(shell_class, mpi=mpi)
shell_import_statement = core_config['shell']['import_statement']
if shell_import_statement:
try:
engine_service.execute(shell_import_statement)
except:
log.msg("Error running import_statement: %s" % shell_import_statement)
# Create the service hierarchy
main_service = service.MultiService()
engine_service.setServiceParent(main_service)
tub_service = Tub()
tub_service.setServiceParent(main_service)
# This needs to be called before the connection is initiated
main_service.startService()
# This initiates the connection to the controller and calls
# register_engine to tell the controller we are ready to do work
engine_connector = EngineConnector(tub_service)
furl_file = kernel_config['engine']['furl_file']
log.msg("Using furl file: %s" % furl_file)
def call_connect(engine_service, furl_file):
d = engine_connector.connect_to_controller(engine_service, furl_file)
def handle_error(f):
# If this print statement is replaced by a log.err(f) I get
# an unhandled error, which makes no sense. I shouldn't have
# to use a print statement here. My only thought is that
# at the beginning of the process the logging is still starting up
print "error connecting to controller:", f.getErrorMessage()
reactor.callLater(0.1, reactor.stop)
d.addErrback(handle_error)
reactor.callWhenRunning(call_connect, engine_service, furl_file)
reactor.run()
def start_engine():
"""
Start the engine, by creating it and starting the Twisted reactor.
This method does:
* If it exists, runs the `mpi_import_statement` to call `MPI_Init`
* Starts the engine logging
* Creates an IPython shell and wraps it in an `EngineService`
* Creates a `foolscap.Tub` to use in connecting to a controller.
* Uses the tub and the `EngineService` along with a Foolscap URL
(or FURL) to connect to the controller and register the engine
with the controller
"""
kernel_config = kernel_config_manager.get_config_obj()
core_config = core_config_manager.get_config_obj()
# Execute the mpi import statement that needs to call MPI_Init
global mpi
mpikey = kernel_config['mpi']['default']
mpi_import_statement = kernel_config['mpi'].get(mpikey, None)
if mpi_import_statement is not None:
try:
exec mpi_import_statement in globals()
except:
mpi = None
else:
mpi = None
# Start logging
logfile = kernel_config['engine']['logfile']
if logfile:
logfile = logfile + str(os.getpid()) + '.log'
try:
openLogFile = open(logfile, 'w')
except:
openLogFile = sys.stdout
else:
openLogFile = sys.stdout
log.startLogging(openLogFile)
# Create the underlying shell class and EngineService
shell_class = import_item(core_config['shell']['shell_class'])
engine_service = EngineService(shell_class, mpi=mpi)
shell_import_statement = core_config['shell']['import_statement']
if shell_import_statement:
try:
engine_service.execute(shell_import_statement)
except:
log.msg("Error running import_statement: %s" % sis)
# Create the service hierarchy
main_service = service.MultiService()
engine_service.setServiceParent(main_service)
tub_service = Tub()
tub_service.setServiceParent(main_service)
# This needs to be called before the connection is initiated
main_service.startService()
# This initiates the connection to the controller and calls
# register_engine to tell the controller we are ready to do work
engine_connector = EngineConnector(tub_service)
furl_file = kernel_config['engine']['furl_file']
log.msg("Using furl file: %s" % furl_file)
d = engine_connector.connect_to_controller(engine_service, furl_file)
d.addErrback(lambda _: reactor.stop())
reactor.run()
def start_engine():
"""
Start the engine, by creating it and starting the Twisted reactor.
This method does:
* If it exists, runs the `mpi_import_statement` to call `MPI_Init`
* Starts the engine logging
* Creates an IPython shell and wraps it in an `EngineService`
* Creates a `foolscap.Tub` to use in connecting to a controller.
* Uses the tub and the `EngineService` along with a Foolscap URL
(or FURL) to connect to the controller and register the engine
with the controller
"""
kernel_config = kernel_config_manager.get_config_obj()
core_config = core_config_manager.get_config_obj()
# Execute the mpi import statement that needs to call MPI_Init
global mpi
mpikey = kernel_config['mpi']['default']
mpi_import_statement = kernel_config['mpi'].get(mpikey, None)
if mpi_import_statement is not None:
try:
exec mpi_import_statement in globals()
except:
mpi = None
else:
mpi = None
# Start logging
logfile = kernel_config['engine']['logfile']
if logfile:
logfile = logfile + str(os.getpid()) + '.log'
try:
openLogFile = open(logfile, 'w')
except:
openLogFile = sys.stdout
else:
openLogFile = sys.stdout
log.startLogging(openLogFile)
# Create the underlying shell class and EngineService
shell_class = import_item(core_config['shell']['shell_class'])
engine_service = EngineService(shell_class, mpi=mpi)
shell_import_statement = core_config['shell']['import_statement']
if shell_import_statement:
try:
engine_service.execute(shell_import_statement)
except:
log.msg("Error running import_statement: %s" %
shell_import_statement)
# Create the service hierarchy
main_service = service.MultiService()
engine_service.setServiceParent(main_service)
tub_service = Tub()
tub_service.setServiceParent(main_service)
# This needs to be called before the connection is initiated
main_service.startService()
# This initiates the connection to the controller and calls
# register_engine to tell the controller we are ready to do work
engine_connector = EngineConnector(tub_service)
furl_file = kernel_config['engine']['furl_file']
log.msg("Using furl file: %s" % furl_file)
def call_connect(engine_service, furl_file):
d = engine_connector.connect_to_controller(engine_service, furl_file)
def handle_error(f):
# If this print statement is replaced by a log.err(f) I get
# an unhandled error, which makes no sense. I shouldn't have
# to use a print statement here. My only thought is that
# at the beginning of the process the logging is still starting up
print "error connecting to controller:", f.getErrorMessage()
reactor.callLater(0.1, reactor.stop)
d.addErrback(handle_error)
reactor.callWhenRunning(call_connect, engine_service, furl_file)
reactor.run()
