def construct(self):
# This is the working dir by now.
sys.path.insert(0, '')
self.start_mpi()
self.start_logging()
# Create the underlying shell class and EngineService
shell_class = import_item(self.master_config.Global.shell_class)
self.engine_service = EngineService(shell_class, mpi=mpi)
self.exec_lines()
# Create the service hierarchy
self.main_service = service.MultiService()
self.engine_service.setServiceParent(self.main_service)
self.tub_service = Tub()
self.tub_service.setServiceParent(self.main_service)
# This needs to be called before the connection is initiated
self.main_service.startService()
# This initiates the connection to the controller and calls
# register_engine to tell the controller we are ready to do work
self.engine_connector = EngineConnector(self.tub_service)
log.msg("Using furl file: %s" % self.master_config.Global.furl_file)
reactor.callWhenRunning(self.call_connect)
def setUp(self):
self.engines = []
self.controller = cs.ControllerService()
self.controller.startService()
self.imultiengine = me.IMultiEngine(self.controller)
self.itc = taskmodule.ITaskController(self.controller)
self.itc.failurePenalty = 0
self.mec_referenceable = IFCSynchronousMultiEngine(self.imultiengine)
self.tc_referenceable = IFCTaskController(self.itc)
self.controller_tub = Tub()
self.controller_tub.listenOn('tcp:10105:interface=127.0.0.1')
self.controller_tub.setLocation('127.0.0.1:10105')
mec_furl = self.controller_tub.registerReference(self.mec_referenceable)
tc_furl = self.controller_tub.registerReference(self.tc_referenceable)
self.controller_tub.startService()
self.client_tub = ClientConnector()
d = self.client_tub.get_multiengine_client(mec_furl)
d.addCallback(self.handle_mec_client)
d.addCallback(lambda _: self.client_tub.get_task_client(tc_furl))
d.addCallback(self.handle_tc_client)
return d
def make_tub(ip, port, secure, cert_file):
"""
Create a listening tub given an ip, port, and cert_file location.
:Parameters:
ip : str
The ip address that the tub should listen on. Empty means all
port : int
The port that the tub should listen on. A value of 0 means
pick a random port
secure: boolean
Will the connection be secure (in the foolscap sense)
cert_file:
A filename of a file to be used for theSSL certificate
"""
if secure:
if have_crypto:
tub = Tub(certFile=cert_file)
else:
raise SecurityError("OpenSSL is not available, so we can't run in secure mode, aborting")
else:
tub = UnauthenticatedTub()
# Set the strport based on the ip and port and start listening
if ip == '':
strport = "tcp:%i" % port
else:
strport = "tcp:%i:interface=%s" % (port, ip)
listener = tub.listenOn(strport)
return tub, listener
def setUp(self):
self.engines = []
self.controller = cs.ControllerService()
self.controller.startService()
self.imultiengine = me.IMultiEngine(self.controller)
self.itc = taskmodule.ITaskController(self.controller)
self.itc.failurePenalty = 0
self.mec_referenceable = IFCSynchronousMultiEngine(self.imultiengine)
self.tc_referenceable = IFCTaskController(self.itc)
self.controller_tub = Tub()
self.controller_tub.listenOn('tcp:10105:interface=127.0.0.1')
self.controller_tub.setLocation('127.0.0.1:10105')
mec_furl = self.controller_tub.registerReference(self.mec_referenceable)
tc_furl = self.controller_tub.registerReference(self.tc_referenceable)
self.controller_tub.startService()
self.client_tub = ClientConnector()
d = self.client_tub.get_multiengine_client(mec_furl)
d.addCallback(self.handle_mec_client)
d.addCallback(lambda _: self.client_tub.get_task_client(tc_furl))
d.addCallback(self.handle_tc_client)
return d
def construct(self):
# This is the working dir by now.
sys.path.insert(0, '')
self.start_mpi()
self.start_logging()
# Create the underlying shell class and EngineService
shell_class = import_item(self.master_config.Global.shell_class)
self.engine_service = EngineService(shell_class, mpi=mpi)
self.exec_lines()
# Create the service hierarchy
self.main_service = service.MultiService()
self.engine_service.setServiceParent(self.main_service)
self.tub_service = Tub()
self.tub_service.setServiceParent(self.main_service)
# This needs to be called before the connection is initiated
self.main_service.startService()
# This initiates the connection to the controller and calls
# register_engine to tell the controller we are ready to do work
self.engine_connector = EngineConnector(self.tub_service)
log.msg("Using furl file: %s" % self.master_config.Global.furl_file)
reactor.callWhenRunning(self.call_connect)
def setUp(self):
self.engines = []
self.controller = ControllerService()
self.controller.startService()
self.imultiengine = IMultiEngine(self.controller)
self.mec_referenceable = IFCSynchronousMultiEngine(self.imultiengine)
self.controller_tub = Tub()
self.controller_tub.listenOn('tcp:10105:interface=127.0.0.1')
self.controller_tub.setLocation('127.0.0.1:10105')
furl = self.controller_tub.registerReference(self.mec_referenceable)
self.controller_tub.startService()
self.client_tub = ClientConnector()
d = self.client_tub.get_multiengine_client(furl)
d.addCallback(self.handle_got_client)
return d
def setUp(self):
# Start a server and append to self.servers
self.controller_reference = FCRemoteEngineRefFromService(self)
self.controller_tub = Tub()
self.controller_tub.listenOn('tcp:10111:interface=127.0.0.1')
self.controller_tub.setLocation('127.0.0.1:10111')
furl = self.controller_tub.registerReference(self.controller_reference)
self.controller_tub.startService()
# Start an EngineService and append to services/client
self.engine_service = es.EngineService()
self.engine_service.startService()
self.engine_tub = Tub()
self.engine_tub.startService()
engine_connector = EngineConnector(self.engine_tub)
d = engine_connector.connect_to_controller(self.engine_service, furl)
# This deferred doesn't fire until after register_engine has returned and
# thus, self.engine has been defined and the tets can proceed.
return d
def setUp(self):
# Start a server and append to self.servers
self.controller_reference = FCRemoteEngineRefFromService(self)
self.controller_tub = Tub()
self.controller_tub.listenOn('tcp:10111:interface=127.0.0.1')
self.controller_tub.setLocation('127.0.0.1:10111')
furl = self.controller_tub.registerReference(self.controller_reference)
self.controller_tub.startService()
# Start an EngineService and append to services/client
self.engine_service = es.EngineService()
self.engine_service.startService()
self.engine_tub = Tub()
self.engine_tub.startService()
engine_connector = EngineConnector(self.engine_tub)
d = engine_connector.connect_to_controller(self.engine_service, furl)
# This deferred doesn't fire until after register_engine has returned and
# thus, self.engine has been defined and the tets can proceed.
return d
def setUp(self):
self.engines = []
self.controller = ControllerService()
self.controller.startService()
self.imultiengine = IMultiEngine(self.controller)
self.mec_referenceable = IFCSynchronousMultiEngine(self.imultiengine)
self.controller_tub = Tub()
self.controller_tub.listenOn('tcp:10111:interface=127.0.0.1')
self.controller_tub.setLocation('127.0.0.1:10111')
furl = self.controller_tub.registerReference(self.mec_referenceable)
self.controller_tub.startService()
self.client_tub = AsyncClientConnector()
d = self.client_tub.get_multiengine_client(furl_or_file=furl)
d.addCallback(self.handle_got_client)
return d
class IPEngineApp(ApplicationWithClusterDir):
name = u'ipengine'
description = _description
command_line_loader = IPEngineAppConfigLoader
default_config_file_name = default_config_file_name
auto_create_cluster_dir = True
def create_default_config(self):
super(IPEngineApp, self).create_default_config()
# The engine should not clean logs as we don't want to remove the
# active log files of other running engines.
self.default_config.Global.clean_logs = False
# Global config attributes
self.default_config.Global.exec_lines = []
self.default_config.Global.shell_class = 'IPython.kernel.core.interpreter.Interpreter'
# Configuration related to the controller
# This must match the filename (path not included) that the controller
# used for the FURL file.
self.default_config.Global.furl_file_name = u'ipcontroller-engine.furl'
# If given, this is the actual location of the controller's FURL file.
# If not, this is computed using the profile, app_dir and furl_file_name
self.default_config.Global.furl_file = u''
# The max number of connection attemps and the initial delay between
# those attemps.
self.default_config.Global.connect_delay = 0.1
self.default_config.Global.connect_max_tries = 15
# MPI related config attributes
self.default_config.MPI.use = ''
self.default_config.MPI.mpi4py = mpi4py_init
self.default_config.MPI.pytrilinos = pytrilinos_init
def post_load_command_line_config(self):
pass
def pre_construct(self):
super(IPEngineApp, self).pre_construct()
self.find_cont_furl_file()
def find_cont_furl_file(self):
"""Set the furl file.
Here we don't try to actually see if it exists for is valid as that
is hadled by the connection logic.
"""
config = self.master_config
# Find the actual controller FURL file
if not config.Global.furl_file:
try_this = os.path.join(
config.Global.cluster_dir,
config.Global.security_dir,
config.Global.furl_file_name
)
config.Global.furl_file = try_this
def construct(self):
# This is the working dir by now.
sys.path.insert(0, '')
self.start_mpi()
self.start_logging()
# Create the underlying shell class and EngineService
shell_class = import_item(self.master_config.Global.shell_class)
self.engine_service = EngineService(shell_class, mpi=mpi)
self.exec_lines()
# Create the service hierarchy
self.main_service = service.MultiService()
self.engine_service.setServiceParent(self.main_service)
self.tub_service = Tub()
self.tub_service.setServiceParent(self.main_service)
# This needs to be called before the connection is initiated
self.main_service.startService()
# This initiates the connection to the controller and calls
# register_engine to tell the controller we are ready to do work
self.engine_connector = EngineConnector(self.tub_service)
log.msg("Using furl file: %s" % self.master_config.Global.furl_file)
reactor.callWhenRunning(self.call_connect)
def call_connect(self):
d = self.engine_connector.connect_to_controller(
self.engine_service,
self.master_config.Global.furl_file,
self.master_config.Global.connect_delay,
self.master_config.Global.connect_max_tries
)
def handle_error(f):
log.msg('Error connecting to controller. This usually means that '
'i) the controller was not started, ii) a firewall was blocking '
'the engine from connecting to the controller or iii) the engine '
' was not pointed at the right FURL file:')
log.msg(f.getErrorMessage())
reactor.callLater(0.1, reactor.stop)
d.addErrback(handle_error)
def start_mpi(self):
global mpi
mpikey = self.master_config.MPI.use
mpi_import_statement = self.master_config.MPI.get(mpikey, None)
if mpi_import_statement is not None:
try:
self.log.info("Initializing MPI:")
self.log.info(mpi_import_statement)
exec mpi_import_statement in globals()
except:
mpi = None
else:
mpi = None
def exec_lines(self):
for line in self.master_config.Global.exec_lines:
try:
log.msg("Executing statement: '%s'" % line)
self.engine_service.execute(line)
except:
log.msg("Error executing statement: %s" % line)
def start_app(self):
reactor.run()
class EngineFCTest(DeferredTestCase, IEngineCoreTestCase,
IEngineSerializedTestCase, IEngineQueuedTestCase):
zi.implements(IControllerBase)
def setUp(self):
# Start a server and append to self.servers
self.controller_reference = FCRemoteEngineRefFromService(self)
self.controller_tub = Tub()
self.controller_tub.listenOn('tcp:10111:interface=127.0.0.1')
self.controller_tub.setLocation('127.0.0.1:10111')
furl = self.controller_tub.registerReference(self.controller_reference)
self.controller_tub.startService()
# Start an EngineService and append to services/client
self.engine_service = es.EngineService()
self.engine_service.startService()
self.engine_tub = Tub()
self.engine_tub.startService()
engine_connector = EngineConnector(self.engine_tub)
d = engine_connector.connect_to_controller(self.engine_service, furl)
# This deferred doesn't fire until after register_engine has returned and
# thus, self.engine has been defined and the tets can proceed.
return d
def tearDown(self):
dlist = []
# Shut down the engine
d = self.engine_tub.stopService()
dlist.append(d)
# Shut down the controller
d = self.controller_tub.stopService()
dlist.append(d)
return defer.DeferredList(dlist)
#---------------------------------------------------------------------------
# Make me look like a basic controller
#---------------------------------------------------------------------------
def register_engine(self,
engine_ref,
id=None,
ip=None,
port=None,
pid=None):
self.engine = IEngineQueued(IEngineBase(engine_ref))
return {'id': id}
def unregister_engine(self, id):
pass
def __init__(self):
self._remote_refs = {}
self.tub = Tub()
self.tub.startService()
class ClientConnector(object):
"""
This class gets remote references from furls and returns the wrapped clients.
This class is also used in `client.py` and `asyncclient.py` to create
a single per client-process Tub.
"""
def __init__(self):
self._remote_refs = {}
self.tub = Tub()
self.tub.startService()
def get_reference(self, furl_or_file):
"""
Get a remote reference using a furl or a file containing a furl.
Remote references are cached locally so once a remote reference
has been retrieved for a given furl, the cached version is
returned.
:Parameters:
furl_or_file : str
A furl or a filename containing a furl
:Returns:
A deferred to a remote reference
"""
furl = find_furl(furl_or_file)
if furl in self._remote_refs:
d = defer.succeed(self._remote_refs[furl])
else:
d = self.tub.getReference(furl)
d.addCallback(self.save_ref, furl)
return d
def save_ref(self, ref, furl):
"""
Cache a remote reference by its furl.
"""
self._remote_refs[furl] = ref
return ref
def get_task_client(self, furl_or_file=''):
"""
Get the task controller client.
This method is a simple wrapper around `get_client` that allow
`furl_or_file` to be empty, in which case, the furls is taken
from the default furl file given in the configuration.
:Parameters:
furl_or_file : str
A furl or a filename containing a furl. If empty, the
default furl_file will be used
:Returns:
A deferred to the actual client class
"""
task_co = client_co['client_interfaces']['task']
if furl_or_file:
ff = furl_or_file
else:
ff = task_co['furl_file']
return self.get_client(ff)
def get_multiengine_client(self, furl_or_file=''):
"""
Get the multiengine controller client.
This method is a simple wrapper around `get_client` that allow
`furl_or_file` to be empty, in which case, the furls is taken
from the default furl file given in the configuration.
:Parameters:
furl_or_file : str
A furl or a filename containing a furl. If empty, the
default furl_file will be used
:Returns:
A deferred to the actual client class
"""
task_co = client_co['client_interfaces']['multiengine']
if furl_or_file:
ff = furl_or_file
else:
ff = task_co['furl_file']
return self.get_client(ff)
def get_client(self, furl_or_file):
"""
Get a remote reference and wrap it in a client by furl.
This method first gets a remote reference and then calls its
`get_client_name` method to find the apprpriate client class
that should be used to wrap the remote reference.
:Parameters:
furl_or_file : str
A furl or a filename containing a furl
:Returns:
A deferred to the actual client class
"""
furl = find_furl(furl_or_file)
d = self.get_reference(furl)
def wrap_remote_reference(rr):
d = rr.callRemote('get_client_name')
d.addCallback(lambda name: import_item(name))
def adapt(client_interface):
client = client_interface(rr)
client.tub = self.tub
return client
d.addCallback(adapt)
return d
d.addCallback(wrap_remote_reference)
return d
def start_engine():
"""
Start the engine, by creating it and starting the Twisted reactor.
This method does:
* If it exists, runs the `mpi_import_statement` to call `MPI_Init`
* Starts the engine logging
* Creates an IPython shell and wraps it in an `EngineService`
* Creates a `foolscap.Tub` to use in connecting to a controller.
* Uses the tub and the `EngineService` along with a Foolscap URL
(or FURL) to connect to the controller and register the engine
with the controller
"""
kernel_config = kernel_config_manager.get_config_obj()
core_config = core_config_manager.get_config_obj()
# Execute the mpi import statement that needs to call MPI_Init
global mpi
mpikey = kernel_config['mpi']['default']
mpi_import_statement = kernel_config['mpi'].get(mpikey, None)
if mpi_import_statement is not None:
try:
exec mpi_import_statement in globals()
except:
mpi = None
else:
mpi = None
# Start logging
logfile = kernel_config['engine']['logfile']
if logfile:
logfile = logfile + str(os.getpid()) + '.log'
try:
openLogFile = open(logfile, 'w')
except:
openLogFile = sys.stdout
else:
openLogFile = sys.stdout
log.startLogging(openLogFile)
# Create the underlying shell class and EngineService
shell_class = import_item(core_config['shell']['shell_class'])
engine_service = EngineService(shell_class, mpi=mpi)
shell_import_statement = core_config['shell']['import_statement']
if shell_import_statement:
try:
engine_service.execute(shell_import_statement)
except:
log.msg("Error running import_statement: %s" % sis)
# Create the service hierarchy
main_service = service.MultiService()
engine_service.setServiceParent(main_service)
tub_service = Tub()
tub_service.setServiceParent(main_service)
# This needs to be called before the connection is initiated
main_service.startService()
# This initiates the connection to the controller and calls
# register_engine to tell the controller we are ready to do work
engine_connector = EngineConnector(tub_service)
furl_file = kernel_config['engine']['furl_file']
log.msg("Using furl file: %s" % furl_file)
d = engine_connector.connect_to_controller(engine_service, furl_file)
d.addErrback(lambda _: reactor.stop())
reactor.run()
class EngineFCTest(DeferredTestCase,
IEngineCoreTestCase,
IEngineSerializedTestCase,
IEngineQueuedTestCase
):
zi.implements(IControllerBase)
def setUp(self):
# Start a server and append to self.servers
self.controller_reference = FCRemoteEngineRefFromService(self)
self.controller_tub = Tub()
self.controller_tub.listenOn('tcp:10111:interface=127.0.0.1')
self.controller_tub.setLocation('127.0.0.1:10111')
furl = self.controller_tub.registerReference(self.controller_reference)
self.controller_tub.startService()
# Start an EngineService and append to services/client
self.engine_service = es.EngineService()
self.engine_service.startService()
self.engine_tub = Tub()
self.engine_tub.startService()
engine_connector = EngineConnector(self.engine_tub)
d = engine_connector.connect_to_controller(self.engine_service, furl)
# This deferred doesn't fire until after register_engine has returned and
# thus, self.engine has been defined and the tets can proceed.
return d
def tearDown(self):
dlist = []
# Shut down the engine
d = self.engine_tub.stopService()
dlist.append(d)
# Shut down the controller
d = self.controller_tub.stopService()
dlist.append(d)
return defer.DeferredList(dlist)
#---------------------------------------------------------------------------
# Make me look like a basic controller
#---------------------------------------------------------------------------
def register_engine(self, engine_ref, id=None, ip=None, port=None, pid=None):
self.engine = IEngineQueued(IEngineBase(engine_ref))
return {'id':id}
def unregister_engine(self, id):
pass
class TaskTest(DeferredTestCase, ITaskControllerTestCase):
def setUp(self):
self.engines = []
self.controller = cs.ControllerService()
self.controller.startService()
self.imultiengine = me.IMultiEngine(self.controller)
self.itc = taskmodule.ITaskController(self.controller)
self.itc.failurePenalty = 0
self.mec_referenceable = IFCSynchronousMultiEngine(self.imultiengine)
self.tc_referenceable = IFCTaskController(self.itc)
self.controller_tub = Tub()
self.controller_tub.listenOn('tcp:10105:interface=127.0.0.1')
self.controller_tub.setLocation('127.0.0.1:10105')
mec_furl = self.controller_tub.registerReference(self.mec_referenceable)
tc_furl = self.controller_tub.registerReference(self.tc_referenceable)
self.controller_tub.startService()
self.client_tub = ClientConnector()
d = self.client_tub.get_multiengine_client(mec_furl)
d.addCallback(self.handle_mec_client)
d.addCallback(lambda _: self.client_tub.get_task_client(tc_furl))
d.addCallback(self.handle_tc_client)
return d
def handle_mec_client(self, client):
self.multiengine = client
def handle_tc_client(self, client):
self.tc = client
def tearDown(self):
dlist = []
# Shut down the multiengine client
d = self.client_tub.tub.stopService()
dlist.append(d)
# Shut down the engines
for e in self.engines:
e.stopService()
# Shut down the controller
d = self.controller_tub.stopService()
d.addBoth(lambda _: self.controller.stopService())
dlist.append(d)
return defer.DeferredList(dlist)
def test_mapper(self):
self.addEngine(1)
m = self.tc.mapper()
self.assertEquals(m.task_controller,self.tc)
self.assertEquals(m.clear_before,False)
self.assertEquals(m.clear_after,False)
self.assertEquals(m.retries,0)
self.assertEquals(m.recovery_task,None)
self.assertEquals(m.depend,None)
self.assertEquals(m.block,True)
def test_map_default(self):
self.addEngine(1)
m = self.tc.mapper()
d = m.map(lambda x: 2*x, range(10))
d.addCallback(lambda r: self.assertEquals(r,[2*x for x in range(10)]))
d.addCallback(lambda _: self.tc.map(lambda x: 2*x, range(10)))
d.addCallback(lambda r: self.assertEquals(r,[2*x for x in range(10)]))
return d
def test_map_noblock(self):
self.addEngine(1)
m = self.tc.mapper(block=False)
d = m.map(lambda x: 2*x, range(10))
d.addCallback(lambda r: self.assertEquals(r,[x for x in range(10)]))
return d
def test_mapper_fail(self):
self.addEngine(1)
m = self.tc.mapper()
d = m.map(lambda x: 1/0, range(10))
d.addBoth(lambda f: self.assertRaises(ZeroDivisionError, _raise_it, f))
return d
def test_parallel(self):
self.addEngine(1)
p = self.tc.parallel()
self.assert_(isinstance(p, ParallelFunction))
@p
def f(x): return 2*x
d = f(range(10))
d.addCallback(lambda r: self.assertEquals(r,[2*x for x in range(10)]))
return d
def test_parallel_noblock(self):
self.addEngine(1)
p = self.tc.parallel(block=False)
self.assert_(isinstance(p, ParallelFunction))
@p
def f(x): return 2*x
d = f(range(10))
d.addCallback(lambda r: self.assertEquals(r,[x for x in range(10)]))
return d
def test_parallel_fail(self):
self.addEngine(1)
p = self.tc.parallel()
self.assert_(isinstance(p, ParallelFunction))
@p
def f(x): return 1/0
d = f(range(10))
d.addBoth(lambda f: self.assertRaises(ZeroDivisionError, _raise_it, f))
return d
def start_engine():
"""
Start the engine, by creating it and starting the Twisted reactor.
This method does:
* If it exists, runs the `mpi_import_statement` to call `MPI_Init`
* Starts the engine logging
* Creates an IPython shell and wraps it in an `EngineService`
* Creates a `foolscap.Tub` to use in connecting to a controller.
* Uses the tub and the `EngineService` along with a Foolscap URL
(or FURL) to connect to the controller and register the engine
with the controller
"""
kernel_config = kernel_config_manager.get_config_obj()
core_config = core_config_manager.get_config_obj()
# Execute the mpi import statement that needs to call MPI_Init
global mpi
mpikey = kernel_config['mpi']['default']
mpi_import_statement = kernel_config['mpi'].get(mpikey, None)
if mpi_import_statement is not None:
try:
exec mpi_import_statement in globals()
except:
mpi = None
else:
mpi = None
# Start logging
logfile = kernel_config['engine']['logfile']
if logfile:
logfile = logfile + str(os.getpid()) + '.log'
try:
openLogFile = open(logfile, 'w')
except:
openLogFile = sys.stdout
else:
openLogFile = sys.stdout
log.startLogging(openLogFile)
# Create the underlying shell class and EngineService
shell_class = import_item(core_config['shell']['shell_class'])
engine_service = EngineService(shell_class, mpi=mpi)
shell_import_statement = core_config['shell']['import_statement']
if shell_import_statement:
try:
engine_service.execute(shell_import_statement)
except:
log.msg("Error running import_statement: %s" %
shell_import_statement)
# Create the service hierarchy
main_service = service.MultiService()
engine_service.setServiceParent(main_service)
tub_service = Tub()
tub_service.setServiceParent(main_service)
# This needs to be called before the connection is initiated
main_service.startService()
# This initiates the connection to the controller and calls
# register_engine to tell the controller we are ready to do work
engine_connector = EngineConnector(tub_service)
furl_file = kernel_config['engine']['furl_file']
log.msg("Using furl file: %s" % furl_file)
def call_connect(engine_service, furl_file):
d = engine_connector.connect_to_controller(engine_service, furl_file)
def handle_error(f):
# If this print statement is replaced by a log.err(f) I get
# an unhandled error, which makes no sense. I shouldn't have
# to use a print statement here. My only thought is that
# at the beginning of the process the logging is still starting up
print "error connecting to controller:", f.getErrorMessage()
reactor.callLater(0.1, reactor.stop)
d.addErrback(handle_error)
reactor.callWhenRunning(call_connect, engine_service, furl_file)
reactor.run()
class FullSynchronousMultiEngineTestCase(DeferredTestCase,
IFullSynchronousMultiEngineTestCase):
def setUp(self):
self.engines = []
self.controller = ControllerService()
self.controller.startService()
self.imultiengine = IMultiEngine(self.controller)
self.mec_referenceable = IFCSynchronousMultiEngine(self.imultiengine)
self.controller_tub = Tub()
self.controller_tub.listenOn('tcp:10105:interface=127.0.0.1')
self.controller_tub.setLocation('127.0.0.1:10105')
furl = self.controller_tub.registerReference(self.mec_referenceable)
self.controller_tub.startService()
self.client_tub = ClientConnector()
d = self.client_tub.get_multiengine_client(furl)
d.addCallback(self.handle_got_client)
return d
def handle_got_client(self, client):
self.multiengine = client
def tearDown(self):
dlist = []
# Shut down the multiengine client
d = self.client_tub.tub.stopService()
dlist.append(d)
# Shut down the engines
for e in self.engines:
e.stopService()
# Shut down the controller
d = self.controller_tub.stopService()
d.addBoth(lambda _: self.controller.stopService())
dlist.append(d)
return defer.DeferredList(dlist)
def test_mapper(self):
self.addEngine(4)
m = self.multiengine.mapper()
self.assertEquals(m.multiengine, self.multiengine)
self.assertEquals(m.dist, 'b')
self.assertEquals(m.targets, 'all')
self.assertEquals(m.block, True)
def test_map_default(self):
self.addEngine(4)
m = self.multiengine.mapper()
d = m.map(lambda x: 2 * x, range(10))
d.addCallback(
lambda r: self.assertEquals(r, [2 * x for x in range(10)]))
d.addCallback(
lambda _: self.multiengine.map(lambda x: 2 * x, range(10)))
d.addCallback(
lambda r: self.assertEquals(r, [2 * x for x in range(10)]))
return d
def test_map_noblock(self):
self.addEngine(4)
m = self.multiengine.mapper(block=False)
d = m.map(lambda x: 2 * x, range(10))
d.addCallback(
lambda did: self.multiengine.get_pending_deferred(did, True))
d.addCallback(
lambda r: self.assertEquals(r, [2 * x for x in range(10)]))
return d
def test_mapper_fail(self):
self.addEngine(4)
m = self.multiengine.mapper()
d = m.map(lambda x: 1 / 0, range(10))
d.addBoth(lambda f: self.assertRaises(ZeroDivisionError, _raise_it, f))
return d
def test_parallel(self):
self.addEngine(4)
p = self.multiengine.parallel()
self.assert_(isinstance(p, ParallelFunction))
@p
def f(x):
return 2 * x
d = f(range(10))
d.addCallback(
lambda r: self.assertEquals(r, [2 * x for x in range(10)]))
return d
def test_parallel_noblock(self):
self.addEngine(1)
p = self.multiengine.parallel(block=False)
self.assert_(isinstance(p, ParallelFunction))
@p
def f(x):
return 2 * x
d = f(range(10))
d.addCallback(
lambda did: self.multiengine.get_pending_deferred(did, True))
d.addCallback(
lambda r: self.assertEquals(r, [2 * x for x in range(10)]))
return d
def test_parallel_fail(self):
self.addEngine(4)
p = self.multiengine.parallel()
self.assert_(isinstance(p, ParallelFunction))
@p
def f(x):
return 1 / 0
d = f(range(10))
d.addBoth(lambda f: self.assertRaises(ZeroDivisionError, _raise_it, f))
return d
class AsyncClientConnector(object):
"""A class for getting remote references and clients from furls.
This start a single :class:`Tub` for all remote reference and caches
references.
"""
def __init__(self):
self._remote_refs = {}
self.tub = Tub()
self.tub.startService()
def _find_furl(self, profile='default', cluster_dir=None,
furl_or_file=None, furl_file_name=None,
ipython_dir=None):
"""Find a FURL file.
If successful, this returns a FURL file that exists on the file
system. The contents of the file have not been checked though. This
is because we often have to deal with FURL file whose buffers have
not been flushed.
This raises an :exc:`~IPython.kernel.fcutil.FURLError` exception
if a FURL file can't be found.
This tries the following:
1. By the name ``furl_or_file``.
2. By ``cluster_dir`` and ``furl_file_name``.
3. By cluster profile with a default of ``default``. This uses
``ipython_dir``.
"""
# Try by furl_or_file
if furl_or_file is not None:
if is_valid_furl_or_file(furl_or_file):
return furl_or_file
if furl_file_name is None:
raise FURLError('A furl_file_name must be provided if furl_or_file is not')
# Try by cluster_dir
if cluster_dir is not None:
cluster_dir_obj = ClusterDir.find_cluster_dir(cluster_dir)
sdir = cluster_dir_obj.security_dir
furl_file = os.path.join(sdir, furl_file_name)
validate_furl_or_file(furl_file)
return furl_file
# Try by profile
if ipython_dir is None:
ipython_dir = get_ipython_dir()
if profile is not None:
cluster_dir_obj = ClusterDir.find_cluster_dir_by_profile(
ipython_dir, profile)
sdir = cluster_dir_obj.security_dir
furl_file = os.path.join(sdir, furl_file_name)
validate_furl_or_file(furl_file)
return furl_file
raise FURLError('Could not find a valid FURL file.')
def get_reference(self, furl_or_file):
"""Get a remote reference using a furl or a file containing a furl.
Remote references are cached locally so once a remote reference
has been retrieved for a given furl, the cached version is
returned.
Parameters
----------
furl_or_file : str
A furl or a filename containing a furl. This should already be
validated, but might not yet exist.
Returns
-------
A deferred to a remote reference
"""
furl = furl_or_file
if furl in self._remote_refs:
d = defer.succeed(self._remote_refs[furl])
else:
d = self.tub.getReference(furl)
d.addCallback(self._save_ref, furl)
return d
def _save_ref(self, ref, furl):
"""Cache a remote reference by its furl."""
self._remote_refs[furl] = ref
return ref
def get_task_client(self, profile='default', cluster_dir=None,
furl_or_file=None, ipython_dir=None,
delay=DELAY, max_tries=MAX_TRIES):
"""Get the task controller client.
This method is a simple wrapper around `get_client` that passes in
the default name of the task client FURL file. Usually only
the ``profile`` option will be needed. If a FURL file can't be
found by its profile, use ``cluster_dir`` or ``furl_or_file``.
Parameters
----------
profile : str
The name of a cluster directory profile (default="default"). The
cluster directory "cluster_<profile>" will be searched for
in ``os.getcwd()``, the ipython_dir and then in the directories
listed in the :env:`IPCLUSTER_DIR_PATH` environment variable.
cluster_dir : str
The full path to a cluster directory. This is useful if profiles
are not being used.
furl_or_file : str
A furl or a filename containing a FURL. This is useful if you
simply know the location of the FURL file.
ipython_dir : str
The location of the ipython_dir if different from the default.
This is used if the cluster directory is being found by profile.
delay : float
The initial delay between re-connection attempts. Susequent delays
get longer according to ``delay[i] = 1.5*delay[i-1]``.
max_tries : int
The max number of re-connection attempts.
Returns
-------
A deferred to the actual client class.
"""
return self.get_client(
profile, cluster_dir, furl_or_file,
'ipcontroller-tc.furl', ipython_dir,
delay, max_tries
)
def get_multiengine_client(self, profile='default', cluster_dir=None,
furl_or_file=None, ipython_dir=None,
delay=DELAY, max_tries=MAX_TRIES):
"""Get the multiengine controller client.
This method is a simple wrapper around `get_client` that passes in
the default name of the task client FURL file. Usually only
the ``profile`` option will be needed. If a FURL file can't be
found by its profile, use ``cluster_dir`` or ``furl_or_file``.
Parameters
----------
profile : str
The name of a cluster directory profile (default="default"). The
cluster directory "cluster_<profile>" will be searched for
in ``os.getcwd()``, the ipython_dir and then in the directories
listed in the :env:`IPCLUSTER_DIR_PATH` environment variable.
cluster_dir : str
The full path to a cluster directory. This is useful if profiles
are not being used.
furl_or_file : str
A furl or a filename containing a FURL. This is useful if you
simply know the location of the FURL file.
ipython_dir : str
The location of the ipython_dir if different from the default.
This is used if the cluster directory is being found by profile.
delay : float
The initial delay between re-connection attempts. Susequent delays
get longer according to ``delay[i] = 1.5*delay[i-1]``.
max_tries : int
The max number of re-connection attempts.
Returns
-------
A deferred to the actual client class.
"""
return self.get_client(
profile, cluster_dir, furl_or_file,
'ipcontroller-mec.furl', ipython_dir,
delay, max_tries
)
def get_client(self, profile='default', cluster_dir=None,
furl_or_file=None, furl_file_name=None, ipython_dir=None,
delay=DELAY, max_tries=MAX_TRIES):
"""Get a remote reference and wrap it in a client by furl.
This method is a simple wrapper around `get_client` that passes in
the default name of the task client FURL file. Usually only
the ``profile`` option will be needed. If a FURL file can't be
found by its profile, use ``cluster_dir`` or ``furl_or_file``.
Parameters
----------
profile : str
The name of a cluster directory profile (default="default"). The
cluster directory "cluster_<profile>" will be searched for
in ``os.getcwd()``, the ipython_dir and then in the directories
listed in the :env:`IPCLUSTER_DIR_PATH` environment variable.
cluster_dir : str
The full path to a cluster directory. This is useful if profiles
are not being used.
furl_or_file : str
A furl or a filename containing a FURL. This is useful if you
simply know the location of the FURL file.
furl_file_name : str
The filename (not the full path) of the FURL. This must be
provided if ``furl_or_file`` is not.
ipython_dir : str
The location of the ipython_dir if different from the default.
This is used if the cluster directory is being found by profile.
delay : float
The initial delay between re-connection attempts. Susequent delays
get longer according to ``delay[i] = 1.5*delay[i-1]``.
max_tries : int
The max number of re-connection attempts.
Returns
-------
A deferred to the actual client class. Or a failure to a
:exc:`FURLError`.
"""
try:
furl_file = self._find_furl(
profile, cluster_dir, furl_or_file,
furl_file_name, ipython_dir
)
# If this succeeds, we know the furl file exists and has a .furl
# extension, but it could still be empty. That is checked each
# connection attempt.
except FURLError:
return defer.fail(failure.Failure())
def _wrap_remote_reference(rr):
d = rr.callRemote('get_client_name')
d.addCallback(lambda name: import_item(name))
def adapt(client_interface):
client = client_interface(rr)
client.tub = self.tub
return client
d.addCallback(adapt)
return d
d = self._try_to_connect(furl_file, delay, max_tries, attempt=0)
d.addCallback(_wrap_remote_reference)
d.addErrback(self._handle_error, furl_file)
return d
def _handle_error(self, f, furl_file):
raise ClientConnectorError('Could not connect to the controller '
'using the FURL file. This usually means that i) the controller '
'was not started or ii) a firewall was blocking the client from '
'connecting to the controller: %s' % furl_file)
@inlineCallbacks
def _try_to_connect(self, furl_or_file, delay, max_tries, attempt):
"""Try to connect to the controller with retry logic."""
if attempt < max_tries:
log.msg("Connecting [%r]" % attempt)
try:
self.furl = find_furl(furl_or_file)
# Uncomment this to see the FURL being tried.
# log.msg("FURL: %s" % self.furl)
rr = yield self.get_reference(self.furl)
log.msg("Connected: %s" % furl_or_file)
except:
if attempt==max_tries-1:
# This will propagate the exception all the way to the top
# where it can be handled.
raise
else:
yield sleep_deferred(delay)
rr = yield self._try_to_connect(
furl_or_file, 1.5*delay, max_tries, attempt+1
)
returnValue(rr)
else:
returnValue(rr)
else:
raise ClientConnectorError(
'Could not connect to controller, max_tries (%r) exceeded. '
'This usually means that i) the controller was not started, '
'or ii) a firewall was blocking the client from connecting '
'to the controller.' % max_tries
)
class FullSynchronousMultiEngineTestCase(
DeferredTestCase, IFullSynchronousMultiEngineTestCase):
def setUp(self):
self.engines = []
self.controller = ControllerService()
self.controller.startService()
self.imultiengine = IMultiEngine(self.controller)
self.mec_referenceable = IFCSynchronousMultiEngine(self.imultiengine)
self.controller_tub = Tub()
self.controller_tub.listenOn('tcp:10111:interface=127.0.0.1')
self.controller_tub.setLocation('127.0.0.1:10111')
furl = self.controller_tub.registerReference(self.mec_referenceable)
self.controller_tub.startService()
self.client_tub = AsyncClientConnector()
d = self.client_tub.get_multiengine_client(furl_or_file=furl)
d.addCallback(self.handle_got_client)
return d
def handle_got_client(self, client):
self.multiengine = client
def tearDown(self):
dlist = []
# Shut down the multiengine client
d = self.client_tub.tub.stopService()
dlist.append(d)
# Shut down the engines
for e in self.engines:
e.stopService()
# Shut down the controller
d = self.controller_tub.stopService()
d.addBoth(lambda _: self.controller.stopService())
dlist.append(d)
return defer.DeferredList(dlist)
def test_mapper(self):
self.addEngine(4)
m = self.multiengine.mapper()
self.assertEquals(m.multiengine,self.multiengine)
self.assertEquals(m.dist,'b')
self.assertEquals(m.targets,'all')
self.assertEquals(m.block,True)
def test_map_default(self):
self.addEngine(4)
m = self.multiengine.mapper()
d = m.map(lambda x: 2*x, list(range(10)))
d.addCallback(lambda r: self.assertEquals(r,[2*x for x in range(10)]))
d.addCallback(lambda _: self.multiengine.map(lambda x: 2*x, list(range(10))))
d.addCallback(lambda r: self.assertEquals(r,[2*x for x in range(10)]))
return d
def test_map_noblock(self):
self.addEngine(4)
m = self.multiengine.mapper(block=False)
d = m.map(lambda x: 2*x, list(range(10)))
d.addCallback(lambda did: self.multiengine.get_pending_deferred(did, True))
d.addCallback(lambda r: self.assertEquals(r,[2*x for x in range(10)]))
return d
def test_mapper_fail(self):
self.addEngine(4)
m = self.multiengine.mapper()
d = m.map(lambda x: 1/0, list(range(10)))
d.addBoth(lambda f: self.assertRaises(ZeroDivisionError, _raise_it, f))
return d
def test_parallel(self):
self.addEngine(4)
p = self.multiengine.parallel()
self.assert_(isinstance(p, ParallelFunction))
@p
def f(x): return 2*x
d = f(list(range(10)))
d.addCallback(lambda r: self.assertEquals(r,[2*x for x in range(10)]))
return d
def test_parallel_noblock(self):
self.addEngine(1)
p = self.multiengine.parallel(block=False)
self.assert_(isinstance(p, ParallelFunction))
@p
def f(x): return 2*x
d = f(list(range(10)))
d.addCallback(lambda did: self.multiengine.get_pending_deferred(did, True))
d.addCallback(lambda r: self.assertEquals(r,[2*x for x in range(10)]))
return d
def test_parallel_fail(self):
self.addEngine(4)
p = self.multiengine.parallel()
self.assert_(isinstance(p, ParallelFunction))
@p
def f(x): return 1/0
d = f(list(range(10)))
d.addBoth(lambda f: self.assertRaises(ZeroDivisionError, _raise_it, f))
return d
class TaskTest(DeferredTestCase, ITaskControllerTestCase):
def setUp(self):
self.engines = []
self.controller = cs.ControllerService()
self.controller.startService()
self.imultiengine = me.IMultiEngine(self.controller)
self.itc = taskmodule.ITaskController(self.controller)
self.itc.failurePenalty = 0
self.mec_referenceable = IFCSynchronousMultiEngine(self.imultiengine)
self.tc_referenceable = IFCTaskController(self.itc)
self.controller_tub = Tub()
self.controller_tub.listenOn('tcp:10105:interface=127.0.0.1')
self.controller_tub.setLocation('127.0.0.1:10105')
mec_furl = self.controller_tub.registerReference(self.mec_referenceable)
tc_furl = self.controller_tub.registerReference(self.tc_referenceable)
self.controller_tub.startService()
self.client_tub = ClientConnector()
d = self.client_tub.get_multiengine_client(mec_furl)
d.addCallback(self.handle_mec_client)
d.addCallback(lambda _: self.client_tub.get_task_client(tc_furl))
d.addCallback(self.handle_tc_client)
return d
def handle_mec_client(self, client):
self.multiengine = client
def handle_tc_client(self, client):
self.tc = client
def tearDown(self):
dlist = []
# Shut down the multiengine client
d = self.client_tub.tub.stopService()
dlist.append(d)
# Shut down the engines
for e in self.engines:
e.stopService()
# Shut down the controller
d = self.controller_tub.stopService()
d.addBoth(lambda _: self.controller.stopService())
dlist.append(d)
return defer.DeferredList(dlist)
def test_mapper(self):
self.addEngine(1)
m = self.tc.mapper()
self.assertEquals(m.task_controller,self.tc)
self.assertEquals(m.clear_before,False)
self.assertEquals(m.clear_after,False)
self.assertEquals(m.retries,0)
self.assertEquals(m.recovery_task,None)
self.assertEquals(m.depend,None)
self.assertEquals(m.block,True)
def test_map_default(self):
self.addEngine(1)
m = self.tc.mapper()
d = m.map(lambda x: 2*x, range(10))
d.addCallback(lambda r: self.assertEquals(r,[2*x for x in range(10)]))
d.addCallback(lambda _: self.tc.map(lambda x: 2*x, range(10)))
d.addCallback(lambda r: self.assertEquals(r,[2*x for x in range(10)]))
return d
def test_map_noblock(self):
self.addEngine(1)
m = self.tc.mapper(block=False)
d = m.map(lambda x: 2*x, range(10))
d.addCallback(lambda r: self.assertEquals(r,[x for x in range(10)]))
return d
def test_mapper_fail(self):
self.addEngine(1)
m = self.tc.mapper()
d = m.map(lambda x: 1/0, range(10))
d.addBoth(lambda f: self.assertRaises(ZeroDivisionError, _raise_it, f))
return d
def test_parallel(self):
self.addEngine(1)
p = self.tc.parallel()
self.assert_(isinstance(p, ParallelFunction))
@p
def f(x): return 2*x
d = f(range(10))
d.addCallback(lambda r: self.assertEquals(r,[2*x for x in range(10)]))
return d
def test_parallel_noblock(self):
self.addEngine(1)
p = self.tc.parallel(block=False)
self.assert_(isinstance(p, ParallelFunction))
@p
def f(x): return 2*x
d = f(range(10))
d.addCallback(lambda r: self.assertEquals(r,[x for x in range(10)]))
return d
def test_parallel_fail(self):
self.addEngine(1)
p = self.tc.parallel()
self.assert_(isinstance(p, ParallelFunction))
@p
def f(x): return 1/0
d = f(range(10))
d.addBoth(lambda f: self.assertRaises(ZeroDivisionError, _raise_it, f))
return d
def start_engine():
"""
Start the engine, by creating it and starting the Twisted reactor.
This method does:
* If it exists, runs the `mpi_import_statement` to call `MPI_Init`
* Starts the engine logging
* Creates an IPython shell and wraps it in an `EngineService`
* Creates a `foolscap.Tub` to use in connecting to a controller.
* Uses the tub and the `EngineService` along with a Foolscap URL
(or FURL) to connect to the controller and register the engine
with the controller
"""
kernel_config = kernel_config_manager.get_config_obj()
core_config = core_config_manager.get_config_obj()
# Execute the mpi import statement that needs to call MPI_Init
global mpi
mpikey = kernel_config['mpi']['default']
mpi_import_statement = kernel_config['mpi'].get(mpikey, None)
if mpi_import_statement is not None:
try:
exec mpi_import_statement in globals()
except:
mpi = None
else:
mpi = None
# Start logging
logfile = kernel_config['engine']['logfile']
if logfile:
logfile = logfile + str(os.getpid()) + '.log'
try:
openLogFile = open(logfile, 'w')
except:
openLogFile = sys.stdout
else:
openLogFile = sys.stdout
log.startLogging(openLogFile)
# Create the underlying shell class and EngineService
shell_class = import_item(core_config['shell']['shell_class'])
engine_service = EngineService(shell_class, mpi=mpi)
shell_import_statement = core_config['shell']['import_statement']
if shell_import_statement:
try:
engine_service.execute(shell_import_statement)
except:
log.msg("Error running import_statement: %s" % shell_import_statement)
# Create the service hierarchy
main_service = service.MultiService()
engine_service.setServiceParent(main_service)
tub_service = Tub()
tub_service.setServiceParent(main_service)
# This needs to be called before the connection is initiated
main_service.startService()
# This initiates the connection to the controller and calls
# register_engine to tell the controller we are ready to do work
engine_connector = EngineConnector(tub_service)
furl_file = kernel_config['engine']['furl_file']
log.msg("Using furl file: %s" % furl_file)
def call_connect(engine_service, furl_file):
d = engine_connector.connect_to_controller(engine_service, furl_file)
def handle_error(f):
# If this print statement is replaced by a log.err(f) I get
# an unhandled error, which makes no sense. I shouldn't have
# to use a print statement here. My only thought is that
# at the beginning of the process the logging is still starting up
print "error connecting to controller:", f.getErrorMessage()
reactor.callLater(0.1, reactor.stop)
d.addErrback(handle_error)
reactor.callWhenRunning(call_connect, engine_service, furl_file)
reactor.run()
def start_engine():
"""
Start the engine, by creating it and starting the Twisted reactor.
This method does:
* If it exists, runs the `mpi_import_statement` to call `MPI_Init`
* Starts the engine logging
* Creates an IPython shell and wraps it in an `EngineService`
* Creates a `foolscap.Tub` to use in connecting to a controller.
* Uses the tub and the `EngineService` along with a Foolscap URL
(or FURL) to connect to the controller and register the engine
with the controller
"""
kernel_config = kernel_config_manager.get_config_obj()
core_config = core_config_manager.get_config_obj()
# Execute the mpi import statement that needs to call MPI_Init
global mpi
mpikey = kernel_config['mpi']['default']
mpi_import_statement = kernel_config['mpi'].get(mpikey, None)
if mpi_import_statement is not None:
try:
exec mpi_import_statement in globals()
except:
mpi = None
else:
mpi = None
# Start logging
logfile = kernel_config['engine']['logfile']
if logfile:
logfile = logfile + str(os.getpid()) + '.log'
try:
openLogFile = open(logfile, 'w')
except:
openLogFile = sys.stdout
else:
openLogFile = sys.stdout
log.startLogging(openLogFile)
# Create the underlying shell class and EngineService
shell_class = import_item(core_config['shell']['shell_class'])
engine_service = EngineService(shell_class, mpi=mpi)
shell_import_statement = core_config['shell']['import_statement']
if shell_import_statement:
try:
engine_service.execute(shell_import_statement)
except:
log.msg("Error running import_statement: %s" % sis)
# Create the service hierarchy
main_service = service.MultiService()
engine_service.setServiceParent(main_service)
tub_service = Tub()
tub_service.setServiceParent(main_service)
# This needs to be called before the connection is initiated
main_service.startService()
# This initiates the connection to the controller and calls
# register_engine to tell the controller we are ready to do work
engine_connector = EngineConnector(tub_service)
furl_file = kernel_config['engine']['furl_file']
log.msg("Using furl file: %s" % furl_file)
d = engine_connector.connect_to_controller(engine_service, furl_file)
d.addErrback(lambda _: reactor.stop())
reactor.run()
class IPEngineApp(ApplicationWithClusterDir):
name = u'ipengine'
description = _description
command_line_loader = IPEngineAppConfigLoader
default_config_file_name = default_config_file_name
auto_create_cluster_dir = True
def create_default_config(self):
super(IPEngineApp, self).create_default_config()
# The engine should not clean logs as we don't want to remove the
# active log files of other running engines.
self.default_config.Global.clean_logs = False
# Global config attributes
self.default_config.Global.exec_lines = []
self.default_config.Global.shell_class = 'IPython.kernel.core.interpreter.Interpreter'
# Configuration related to the controller
# This must match the filename (path not included) that the controller
# used for the FURL file.
self.default_config.Global.furl_file_name = u'ipcontroller-engine.furl'
# If given, this is the actual location of the controller's FURL file.
# If not, this is computed using the profile, app_dir and furl_file_name
self.default_config.Global.furl_file = u''
# The max number of connection attemps and the initial delay between
# those attemps.
self.default_config.Global.connect_delay = 0.1
self.default_config.Global.connect_max_tries = 15
# MPI related config attributes
self.default_config.MPI.use = ''
self.default_config.MPI.mpi4py = mpi4py_init
self.default_config.MPI.pytrilinos = pytrilinos_init
def post_load_command_line_config(self):
pass
def pre_construct(self):
super(IPEngineApp, self).pre_construct()
self.find_cont_furl_file()
def find_cont_furl_file(self):
"""Set the furl file.
Here we don't try to actually see if it exists for is valid as that
is hadled by the connection logic.
"""
config = self.master_config
# Find the actual controller FURL file
if not config.Global.furl_file:
try_this = os.path.join(config.Global.cluster_dir,
config.Global.security_dir,
config.Global.furl_file_name)
config.Global.furl_file = try_this
def construct(self):
# This is the working dir by now.
sys.path.insert(0, '')
self.start_mpi()
self.start_logging()
# Create the underlying shell class and EngineService
shell_class = import_item(self.master_config.Global.shell_class)
self.engine_service = EngineService(shell_class, mpi=mpi)
self.exec_lines()
# Create the service hierarchy
self.main_service = service.MultiService()
self.engine_service.setServiceParent(self.main_service)
self.tub_service = Tub()
self.tub_service.setServiceParent(self.main_service)
# This needs to be called before the connection is initiated
self.main_service.startService()
# This initiates the connection to the controller and calls
# register_engine to tell the controller we are ready to do work
self.engine_connector = EngineConnector(self.tub_service)
log.msg("Using furl file: %s" % self.master_config.Global.furl_file)
reactor.callWhenRunning(self.call_connect)
def call_connect(self):
d = self.engine_connector.connect_to_controller(
self.engine_service, self.master_config.Global.furl_file,
self.master_config.Global.connect_delay,
self.master_config.Global.connect_max_tries)
def handle_error(f):
log.msg(
'Error connecting to controller. This usually means that '
'i) the controller was not started, ii) a firewall was blocking '
'the engine from connecting to the controller or iii) the engine '
' was not pointed at the right FURL file:')
log.msg(f.getErrorMessage())
reactor.callLater(0.1, reactor.stop)
d.addErrback(handle_error)
def start_mpi(self):
global mpi
mpikey = self.master_config.MPI.use
mpi_import_statement = self.master_config.MPI.get(mpikey, None)
if mpi_import_statement is not None:
try:
self.log.info("Initializing MPI:")
self.log.info(mpi_import_statement)
exec mpi_import_statement in globals()
except:
mpi = None
else:
mpi = None
def exec_lines(self):
for line in self.master_config.Global.exec_lines:
try:
log.msg("Executing statement: '%s'" % line)
self.engine_service.execute(line)
except:
log.msg("Error executing statement: %s" % line)
def start_app(self):
reactor.run()
class AsyncClientConnector(object):
"""A class for getting remote references and clients from furls.
This start a single :class:`Tub` for all remote reference and caches
references.
"""
def __init__(self):
self._remote_refs = {}
self.tub = Tub()
self.tub.startService()
def _find_furl(self, profile='default', cluster_dir=None,
furl_or_file=None, furl_file_name=None,
ipython_dir=None):
"""Find a FURL file.
If successful, this returns a FURL file that exists on the file
system. The contents of the file have not been checked though. This
is because we often have to deal with FURL file whose buffers have
not been flushed.
This raises an :exc:`~IPython.kernel.fcutil.FURLError` exception
if a FURL file can't be found.
This tries the following:
1. By the name ``furl_or_file``.
2. By ``cluster_dir`` and ``furl_file_name``.
3. By cluster profile with a default of ``default``. This uses
``ipython_dir``.
"""
# Try by furl_or_file
if furl_or_file is not None:
if is_valid_furl_or_file(furl_or_file):
return furl_or_file
if furl_file_name is None:
raise FURLError('A furl_file_name must be provided if furl_or_file is not')
# Try by cluster_dir
if cluster_dir is not None:
cluster_dir_obj = ClusterDir.find_cluster_dir(cluster_dir)
sdir = cluster_dir_obj.security_dir
furl_file = os.path.join(sdir, furl_file_name)
validate_furl_or_file(furl_file)
return furl_file
# Try by profile
if ipython_dir is None:
ipython_dir = get_ipython_dir()
if profile is not None:
cluster_dir_obj = ClusterDir.find_cluster_dir_by_profile(
ipython_dir, profile)
sdir = cluster_dir_obj.security_dir
furl_file = os.path.join(sdir, furl_file_name)
validate_furl_or_file(furl_file)
return furl_file
raise FURLError('Could not find a valid FURL file.')
def get_reference(self, furl_or_file):
"""Get a remote reference using a furl or a file containing a furl.
Remote references are cached locally so once a remote reference
has been retrieved for a given furl, the cached version is
returned.
Parameters
----------
furl_or_file : str
A furl or a filename containing a furl. This should already be
validated, but might not yet exist.
Returns
-------
A deferred to a remote reference
"""
furl = furl_or_file
if furl in self._remote_refs:
d = defer.succeed(self._remote_refs[furl])
else:
d = self.tub.getReference(furl)
d.addCallback(self._save_ref, furl)
return d
def _save_ref(self, ref, furl):
"""Cache a remote reference by its furl."""
self._remote_refs[furl] = ref
return ref
def get_task_client(self, profile='default', cluster_dir=None,
furl_or_file=None, ipython_dir=None,
delay=DELAY, max_tries=MAX_TRIES):
"""Get the task controller client.
This method is a simple wrapper around `get_client` that passes in
the default name of the task client FURL file. Usually only
the ``profile`` option will be needed. If a FURL file can't be
found by its profile, use ``cluster_dir`` or ``furl_or_file``.
Parameters
----------
profile : str
The name of a cluster directory profile (default="default"). The
cluster directory "cluster_<profile>" will be searched for
in ``os.getcwd()``, the ipython_dir and then in the directories
listed in the :env:`IPCLUSTER_DIR_PATH` environment variable.
cluster_dir : str
The full path to a cluster directory. This is useful if profiles
are not being used.
furl_or_file : str
A furl or a filename containing a FURL. This is useful if you
simply know the location of the FURL file.
ipython_dir : str
The location of the ipython_dir if different from the default.
This is used if the cluster directory is being found by profile.
delay : float
The initial delay between re-connection attempts. Susequent delays
get longer according to ``delay[i] = 1.5*delay[i-1]``.
max_tries : int
The max number of re-connection attempts.
Returns
-------
A deferred to the actual client class.
"""
return self.get_client(
profile, cluster_dir, furl_or_file,
'ipcontroller-tc.furl', ipython_dir,
delay, max_tries
)
def get_multiengine_client(self, profile='default', cluster_dir=None,
furl_or_file=None, ipython_dir=None,
delay=DELAY, max_tries=MAX_TRIES):
"""Get the multiengine controller client.
This method is a simple wrapper around `get_client` that passes in
the default name of the task client FURL file. Usually only
the ``profile`` option will be needed. If a FURL file can't be
found by its profile, use ``cluster_dir`` or ``furl_or_file``.
Parameters
----------
profile : str
The name of a cluster directory profile (default="default"). The
cluster directory "cluster_<profile>" will be searched for
in ``os.getcwd()``, the ipython_dir and then in the directories
listed in the :env:`IPCLUSTER_DIR_PATH` environment variable.
cluster_dir : str
The full path to a cluster directory. This is useful if profiles
are not being used.
furl_or_file : str
A furl or a filename containing a FURL. This is useful if you
simply know the location of the FURL file.
ipython_dir : str
The location of the ipython_dir if different from the default.
This is used if the cluster directory is being found by profile.
delay : float
The initial delay between re-connection attempts. Susequent delays
get longer according to ``delay[i] = 1.5*delay[i-1]``.
max_tries : int
The max number of re-connection attempts.
Returns
-------
A deferred to the actual client class.
"""
return self.get_client(
profile, cluster_dir, furl_or_file,
'ipcontroller-mec.furl', ipython_dir,
delay, max_tries
)
def get_client(self, profile='default', cluster_dir=None,
furl_or_file=None, furl_file_name=None, ipython_dir=None,
delay=DELAY, max_tries=MAX_TRIES):
"""Get a remote reference and wrap it in a client by furl.
This method is a simple wrapper around `get_client` that passes in
the default name of the task client FURL file. Usually only
the ``profile`` option will be needed. If a FURL file can't be
found by its profile, use ``cluster_dir`` or ``furl_or_file``.
Parameters
----------
profile : str
The name of a cluster directory profile (default="default"). The
cluster directory "cluster_<profile>" will be searched for
in ``os.getcwd()``, the ipython_dir and then in the directories
listed in the :env:`IPCLUSTER_DIR_PATH` environment variable.
cluster_dir : str
The full path to a cluster directory. This is useful if profiles
are not being used.
furl_or_file : str
A furl or a filename containing a FURL. This is useful if you
simply know the location of the FURL file.
furl_file_name : str
The filename (not the full path) of the FURL. This must be
provided if ``furl_or_file`` is not.
ipython_dir : str
The location of the ipython_dir if different from the default.
This is used if the cluster directory is being found by profile.
delay : float
The initial delay between re-connection attempts. Susequent delays
get longer according to ``delay[i] = 1.5*delay[i-1]``.
max_tries : int
The max number of re-connection attempts.
Returns
-------
A deferred to the actual client class. Or a failure to a
:exc:`FURLError`.
"""
try:
furl_file = self._find_furl(
profile, cluster_dir, furl_or_file,
furl_file_name, ipython_dir
)
# If this succeeds, we know the furl file exists and has a .furl
# extension, but it could still be empty. That is checked each
# connection attempt.
except FURLError:
return defer.fail(failure.Failure())
def _wrap_remote_reference(rr):
d = rr.callRemote('get_client_name')
d.addCallback(lambda name: import_item(name))
def adapt(client_interface):
client = client_interface(rr)
client.tub = self.tub
return client
d.addCallback(adapt)
return d
d = self._try_to_connect(furl_file, delay, max_tries, attempt=0)
d.addCallback(_wrap_remote_reference)
d.addErrback(self._handle_error, furl_file)
return d
def _handle_error(self, f, furl_file):
raise ClientConnectorError('Could not connect to the controller '
'using the FURL file. This usually means that i) the controller '
'was not started or ii) a firewall was blocking the client from '
'connecting to the controller: %s' % furl_file)
@inlineCallbacks
def _try_to_connect(self, furl_or_file, delay, max_tries, attempt):
"""Try to connect to the controller with retry logic."""
if attempt < max_tries:
log.msg("Connecting [%r]" % attempt)
try:
self.furl = find_furl(furl_or_file)
# Uncomment this to see the FURL being tried.
# log.msg("FURL: %s" % self.furl)
rr = yield self.get_reference(self.furl)
log.msg("Connected: %s" % furl_or_file)
except:
if attempt==max_tries-1:
# This will propagate the exception all the way to the top
# where it can be handled.
raise
else:
yield sleep_deferred(delay)
rr = yield self._try_to_connect(
furl_or_file, 1.5*delay, max_tries, attempt+1
)
returnValue(rr)
else:
returnValue(rr)
else:
raise ClientConnectorError(
'Could not connect to controller, max_tries (%r) exceeded. '
'This usually means that i) the controller was not started, '
'or ii) a firewall was blocking the client from connecting '
'to the controller.' % max_tries
)
