def construct(self): # This is the working dir by now. sys.path.insert(0, '') self.start_mpi() self.start_logging() # Create the underlying shell class and EngineService shell_class = import_item(self.master_config.Global.shell_class) self.engine_service = EngineService(shell_class, mpi=mpi) self.exec_lines() # Create the service hierarchy self.main_service = service.MultiService() self.engine_service.setServiceParent(self.main_service) self.tub_service = Tub() self.tub_service.setServiceParent(self.main_service) # This needs to be called before the connection is initiated self.main_service.startService() # This initiates the connection to the controller and calls # register_engine to tell the controller we are ready to do work self.engine_connector = EngineConnector(self.tub_service) log.msg("Using furl file: %s" % self.master_config.Global.furl_file) reactor.callWhenRunning(self.call_connect)
def setUp(self): self.engines = [] self.controller = cs.ControllerService() self.controller.startService() self.imultiengine = me.IMultiEngine(self.controller) self.itc = taskmodule.ITaskController(self.controller) self.itc.failurePenalty = 0 self.mec_referenceable = IFCSynchronousMultiEngine(self.imultiengine) self.tc_referenceable = IFCTaskController(self.itc) self.controller_tub = Tub() self.controller_tub.listenOn('tcp:10105:interface=127.0.0.1') self.controller_tub.setLocation('127.0.0.1:10105') mec_furl = self.controller_tub.registerReference(self.mec_referenceable) tc_furl = self.controller_tub.registerReference(self.tc_referenceable) self.controller_tub.startService() self.client_tub = ClientConnector() d = self.client_tub.get_multiengine_client(mec_furl) d.addCallback(self.handle_mec_client) d.addCallback(lambda _: self.client_tub.get_task_client(tc_furl)) d.addCallback(self.handle_tc_client) return d
def make_tub(ip, port, secure, cert_file): """ Create a listening tub given an ip, port, and cert_file location. :Parameters: ip : str The ip address that the tub should listen on. Empty means all port : int The port that the tub should listen on. A value of 0 means pick a random port secure: boolean Will the connection be secure (in the foolscap sense) cert_file: A filename of a file to be used for theSSL certificate """ if secure: if have_crypto: tub = Tub(certFile=cert_file) else: raise SecurityError("OpenSSL is not available, so we can't run in secure mode, aborting") else: tub = UnauthenticatedTub() # Set the strport based on the ip and port and start listening if ip == '': strport = "tcp:%i" % port else: strport = "tcp:%i:interface=%s" % (port, ip) listener = tub.listenOn(strport) return tub, listener
def setUp(self): self.engines = [] self.controller = ControllerService() self.controller.startService() self.imultiengine = IMultiEngine(self.controller) self.mec_referenceable = IFCSynchronousMultiEngine(self.imultiengine) self.controller_tub = Tub() self.controller_tub.listenOn('tcp:10105:interface=127.0.0.1') self.controller_tub.setLocation('127.0.0.1:10105') furl = self.controller_tub.registerReference(self.mec_referenceable) self.controller_tub.startService() self.client_tub = ClientConnector() d = self.client_tub.get_multiengine_client(furl) d.addCallback(self.handle_got_client) return d
def setUp(self): # Start a server and append to self.servers self.controller_reference = FCRemoteEngineRefFromService(self) self.controller_tub = Tub() self.controller_tub.listenOn('tcp:10111:interface=127.0.0.1') self.controller_tub.setLocation('127.0.0.1:10111') furl = self.controller_tub.registerReference(self.controller_reference) self.controller_tub.startService() # Start an EngineService and append to services/client self.engine_service = es.EngineService() self.engine_service.startService() self.engine_tub = Tub() self.engine_tub.startService() engine_connector = EngineConnector(self.engine_tub) d = engine_connector.connect_to_controller(self.engine_service, furl) # This deferred doesn't fire until after register_engine has returned and # thus, self.engine has been defined and the tets can proceed. return d
def setUp(self): self.engines = [] self.controller = ControllerService() self.controller.startService() self.imultiengine = IMultiEngine(self.controller) self.mec_referenceable = IFCSynchronousMultiEngine(self.imultiengine) self.controller_tub = Tub() self.controller_tub.listenOn('tcp:10111:interface=127.0.0.1') self.controller_tub.setLocation('127.0.0.1:10111') furl = self.controller_tub.registerReference(self.mec_referenceable) self.controller_tub.startService() self.client_tub = AsyncClientConnector() d = self.client_tub.get_multiengine_client(furl_or_file=furl) d.addCallback(self.handle_got_client) return d
class IPEngineApp(ApplicationWithClusterDir): name = u'ipengine' description = _description command_line_loader = IPEngineAppConfigLoader default_config_file_name = default_config_file_name auto_create_cluster_dir = True def create_default_config(self): super(IPEngineApp, self).create_default_config() # The engine should not clean logs as we don't want to remove the # active log files of other running engines. self.default_config.Global.clean_logs = False # Global config attributes self.default_config.Global.exec_lines = [] self.default_config.Global.shell_class = 'IPython.kernel.core.interpreter.Interpreter' # Configuration related to the controller # This must match the filename (path not included) that the controller # used for the FURL file. self.default_config.Global.furl_file_name = u'ipcontroller-engine.furl' # If given, this is the actual location of the controller's FURL file. # If not, this is computed using the profile, app_dir and furl_file_name self.default_config.Global.furl_file = u'' # The max number of connection attemps and the initial delay between # those attemps. self.default_config.Global.connect_delay = 0.1 self.default_config.Global.connect_max_tries = 15 # MPI related config attributes self.default_config.MPI.use = '' self.default_config.MPI.mpi4py = mpi4py_init self.default_config.MPI.pytrilinos = pytrilinos_init def post_load_command_line_config(self): pass def pre_construct(self): super(IPEngineApp, self).pre_construct() self.find_cont_furl_file() def find_cont_furl_file(self): """Set the furl file. Here we don't try to actually see if it exists for is valid as that is hadled by the connection logic. """ config = self.master_config # Find the actual controller FURL file if not config.Global.furl_file: try_this = os.path.join( config.Global.cluster_dir, config.Global.security_dir, config.Global.furl_file_name ) config.Global.furl_file = try_this def construct(self): # This is the working dir by now. sys.path.insert(0, '') self.start_mpi() self.start_logging() # Create the underlying shell class and EngineService shell_class = import_item(self.master_config.Global.shell_class) self.engine_service = EngineService(shell_class, mpi=mpi) self.exec_lines() # Create the service hierarchy self.main_service = service.MultiService() self.engine_service.setServiceParent(self.main_service) self.tub_service = Tub() self.tub_service.setServiceParent(self.main_service) # This needs to be called before the connection is initiated self.main_service.startService() # This initiates the connection to the controller and calls # register_engine to tell the controller we are ready to do work self.engine_connector = EngineConnector(self.tub_service) log.msg("Using furl file: %s" % self.master_config.Global.furl_file) reactor.callWhenRunning(self.call_connect) def call_connect(self): d = self.engine_connector.connect_to_controller( self.engine_service, self.master_config.Global.furl_file, self.master_config.Global.connect_delay, self.master_config.Global.connect_max_tries ) def handle_error(f): log.msg('Error connecting to controller. This usually means that ' 'i) the controller was not started, ii) a firewall was blocking ' 'the engine from connecting to the controller or iii) the engine ' ' was not pointed at the right FURL file:') log.msg(f.getErrorMessage()) reactor.callLater(0.1, reactor.stop) d.addErrback(handle_error) def start_mpi(self): global mpi mpikey = self.master_config.MPI.use mpi_import_statement = self.master_config.MPI.get(mpikey, None) if mpi_import_statement is not None: try: self.log.info("Initializing MPI:") self.log.info(mpi_import_statement) exec mpi_import_statement in globals() except: mpi = None else: mpi = None def exec_lines(self): for line in self.master_config.Global.exec_lines: try: log.msg("Executing statement: '%s'" % line) self.engine_service.execute(line) except: log.msg("Error executing statement: %s" % line) def start_app(self): reactor.run()
class EngineFCTest(DeferredTestCase, IEngineCoreTestCase, IEngineSerializedTestCase, IEngineQueuedTestCase): zi.implements(IControllerBase) def setUp(self): # Start a server and append to self.servers self.controller_reference = FCRemoteEngineRefFromService(self) self.controller_tub = Tub() self.controller_tub.listenOn('tcp:10111:interface=127.0.0.1') self.controller_tub.setLocation('127.0.0.1:10111') furl = self.controller_tub.registerReference(self.controller_reference) self.controller_tub.startService() # Start an EngineService and append to services/client self.engine_service = es.EngineService() self.engine_service.startService() self.engine_tub = Tub() self.engine_tub.startService() engine_connector = EngineConnector(self.engine_tub) d = engine_connector.connect_to_controller(self.engine_service, furl) # This deferred doesn't fire until after register_engine has returned and # thus, self.engine has been defined and the tets can proceed. return d def tearDown(self): dlist = [] # Shut down the engine d = self.engine_tub.stopService() dlist.append(d) # Shut down the controller d = self.controller_tub.stopService() dlist.append(d) return defer.DeferredList(dlist) #--------------------------------------------------------------------------- # Make me look like a basic controller #--------------------------------------------------------------------------- def register_engine(self, engine_ref, id=None, ip=None, port=None, pid=None): self.engine = IEngineQueued(IEngineBase(engine_ref)) return {'id': id} def unregister_engine(self, id): pass
def __init__(self): self._remote_refs = {} self.tub = Tub() self.tub.startService()
class ClientConnector(object): """ This class gets remote references from furls and returns the wrapped clients. This class is also used in `client.py` and `asyncclient.py` to create a single per client-process Tub. """ def __init__(self): self._remote_refs = {} self.tub = Tub() self.tub.startService() def get_reference(self, furl_or_file): """ Get a remote reference using a furl or a file containing a furl. Remote references are cached locally so once a remote reference has been retrieved for a given furl, the cached version is returned. :Parameters: furl_or_file : str A furl or a filename containing a furl :Returns: A deferred to a remote reference """ furl = find_furl(furl_or_file) if furl in self._remote_refs: d = defer.succeed(self._remote_refs[furl]) else: d = self.tub.getReference(furl) d.addCallback(self.save_ref, furl) return d def save_ref(self, ref, furl): """ Cache a remote reference by its furl. """ self._remote_refs[furl] = ref return ref def get_task_client(self, furl_or_file=''): """ Get the task controller client. This method is a simple wrapper around `get_client` that allow `furl_or_file` to be empty, in which case, the furls is taken from the default furl file given in the configuration. :Parameters: furl_or_file : str A furl or a filename containing a furl. If empty, the default furl_file will be used :Returns: A deferred to the actual client class """ task_co = client_co['client_interfaces']['task'] if furl_or_file: ff = furl_or_file else: ff = task_co['furl_file'] return self.get_client(ff) def get_multiengine_client(self, furl_or_file=''): """ Get the multiengine controller client. This method is a simple wrapper around `get_client` that allow `furl_or_file` to be empty, in which case, the furls is taken from the default furl file given in the configuration. :Parameters: furl_or_file : str A furl or a filename containing a furl. If empty, the default furl_file will be used :Returns: A deferred to the actual client class """ task_co = client_co['client_interfaces']['multiengine'] if furl_or_file: ff = furl_or_file else: ff = task_co['furl_file'] return self.get_client(ff) def get_client(self, furl_or_file): """ Get a remote reference and wrap it in a client by furl. This method first gets a remote reference and then calls its `get_client_name` method to find the apprpriate client class that should be used to wrap the remote reference. :Parameters: furl_or_file : str A furl or a filename containing a furl :Returns: A deferred to the actual client class """ furl = find_furl(furl_or_file) d = self.get_reference(furl) def wrap_remote_reference(rr): d = rr.callRemote('get_client_name') d.addCallback(lambda name: import_item(name)) def adapt(client_interface): client = client_interface(rr) client.tub = self.tub return client d.addCallback(adapt) return d d.addCallback(wrap_remote_reference) return d
def start_engine(): """ Start the engine, by creating it and starting the Twisted reactor. This method does: * If it exists, runs the `mpi_import_statement` to call `MPI_Init` * Starts the engine logging * Creates an IPython shell and wraps it in an `EngineService` * Creates a `foolscap.Tub` to use in connecting to a controller. * Uses the tub and the `EngineService` along with a Foolscap URL (or FURL) to connect to the controller and register the engine with the controller """ kernel_config = kernel_config_manager.get_config_obj() core_config = core_config_manager.get_config_obj() # Execute the mpi import statement that needs to call MPI_Init global mpi mpikey = kernel_config['mpi']['default'] mpi_import_statement = kernel_config['mpi'].get(mpikey, None) if mpi_import_statement is not None: try: exec mpi_import_statement in globals() except: mpi = None else: mpi = None # Start logging logfile = kernel_config['engine']['logfile'] if logfile: logfile = logfile + str(os.getpid()) + '.log' try: openLogFile = open(logfile, 'w') except: openLogFile = sys.stdout else: openLogFile = sys.stdout log.startLogging(openLogFile) # Create the underlying shell class and EngineService shell_class = import_item(core_config['shell']['shell_class']) engine_service = EngineService(shell_class, mpi=mpi) shell_import_statement = core_config['shell']['import_statement'] if shell_import_statement: try: engine_service.execute(shell_import_statement) except: log.msg("Error running import_statement: %s" % sis) # Create the service hierarchy main_service = service.MultiService() engine_service.setServiceParent(main_service) tub_service = Tub() tub_service.setServiceParent(main_service) # This needs to be called before the connection is initiated main_service.startService() # This initiates the connection to the controller and calls # register_engine to tell the controller we are ready to do work engine_connector = EngineConnector(tub_service) furl_file = kernel_config['engine']['furl_file'] log.msg("Using furl file: %s" % furl_file) d = engine_connector.connect_to_controller(engine_service, furl_file) d.addErrback(lambda _: reactor.stop()) reactor.run()
class EngineFCTest(DeferredTestCase, IEngineCoreTestCase, IEngineSerializedTestCase, IEngineQueuedTestCase ): zi.implements(IControllerBase) def setUp(self): # Start a server and append to self.servers self.controller_reference = FCRemoteEngineRefFromService(self) self.controller_tub = Tub() self.controller_tub.listenOn('tcp:10111:interface=127.0.0.1') self.controller_tub.setLocation('127.0.0.1:10111') furl = self.controller_tub.registerReference(self.controller_reference) self.controller_tub.startService() # Start an EngineService and append to services/client self.engine_service = es.EngineService() self.engine_service.startService() self.engine_tub = Tub() self.engine_tub.startService() engine_connector = EngineConnector(self.engine_tub) d = engine_connector.connect_to_controller(self.engine_service, furl) # This deferred doesn't fire until after register_engine has returned and # thus, self.engine has been defined and the tets can proceed. return d def tearDown(self): dlist = [] # Shut down the engine d = self.engine_tub.stopService() dlist.append(d) # Shut down the controller d = self.controller_tub.stopService() dlist.append(d) return defer.DeferredList(dlist) #--------------------------------------------------------------------------- # Make me look like a basic controller #--------------------------------------------------------------------------- def register_engine(self, engine_ref, id=None, ip=None, port=None, pid=None): self.engine = IEngineQueued(IEngineBase(engine_ref)) return {'id':id} def unregister_engine(self, id): pass
class TaskTest(DeferredTestCase, ITaskControllerTestCase): def setUp(self): self.engines = [] self.controller = cs.ControllerService() self.controller.startService() self.imultiengine = me.IMultiEngine(self.controller) self.itc = taskmodule.ITaskController(self.controller) self.itc.failurePenalty = 0 self.mec_referenceable = IFCSynchronousMultiEngine(self.imultiengine) self.tc_referenceable = IFCTaskController(self.itc) self.controller_tub = Tub() self.controller_tub.listenOn('tcp:10105:interface=127.0.0.1') self.controller_tub.setLocation('127.0.0.1:10105') mec_furl = self.controller_tub.registerReference(self.mec_referenceable) tc_furl = self.controller_tub.registerReference(self.tc_referenceable) self.controller_tub.startService() self.client_tub = ClientConnector() d = self.client_tub.get_multiengine_client(mec_furl) d.addCallback(self.handle_mec_client) d.addCallback(lambda _: self.client_tub.get_task_client(tc_furl)) d.addCallback(self.handle_tc_client) return d def handle_mec_client(self, client): self.multiengine = client def handle_tc_client(self, client): self.tc = client def tearDown(self): dlist = [] # Shut down the multiengine client d = self.client_tub.tub.stopService() dlist.append(d) # Shut down the engines for e in self.engines: e.stopService() # Shut down the controller d = self.controller_tub.stopService() d.addBoth(lambda _: self.controller.stopService()) dlist.append(d) return defer.DeferredList(dlist) def test_mapper(self): self.addEngine(1) m = self.tc.mapper() self.assertEquals(m.task_controller,self.tc) self.assertEquals(m.clear_before,False) self.assertEquals(m.clear_after,False) self.assertEquals(m.retries,0) self.assertEquals(m.recovery_task,None) self.assertEquals(m.depend,None) self.assertEquals(m.block,True) def test_map_default(self): self.addEngine(1) m = self.tc.mapper() d = m.map(lambda x: 2*x, range(10)) d.addCallback(lambda r: self.assertEquals(r,[2*x for x in range(10)])) d.addCallback(lambda _: self.tc.map(lambda x: 2*x, range(10))) d.addCallback(lambda r: self.assertEquals(r,[2*x for x in range(10)])) return d def test_map_noblock(self): self.addEngine(1) m = self.tc.mapper(block=False) d = m.map(lambda x: 2*x, range(10)) d.addCallback(lambda r: self.assertEquals(r,[x for x in range(10)])) return d def test_mapper_fail(self): self.addEngine(1) m = self.tc.mapper() d = m.map(lambda x: 1/0, range(10)) d.addBoth(lambda f: self.assertRaises(ZeroDivisionError, _raise_it, f)) return d def test_parallel(self): self.addEngine(1) p = self.tc.parallel() self.assert_(isinstance(p, ParallelFunction)) @p def f(x): return 2*x d = f(range(10)) d.addCallback(lambda r: self.assertEquals(r,[2*x for x in range(10)])) return d def test_parallel_noblock(self): self.addEngine(1) p = self.tc.parallel(block=False) self.assert_(isinstance(p, ParallelFunction)) @p def f(x): return 2*x d = f(range(10)) d.addCallback(lambda r: self.assertEquals(r,[x for x in range(10)])) return d def test_parallel_fail(self): self.addEngine(1) p = self.tc.parallel() self.assert_(isinstance(p, ParallelFunction)) @p def f(x): return 1/0 d = f(range(10)) d.addBoth(lambda f: self.assertRaises(ZeroDivisionError, _raise_it, f)) return d
def start_engine(): """ Start the engine, by creating it and starting the Twisted reactor. This method does: * If it exists, runs the `mpi_import_statement` to call `MPI_Init` * Starts the engine logging * Creates an IPython shell and wraps it in an `EngineService` * Creates a `foolscap.Tub` to use in connecting to a controller. * Uses the tub and the `EngineService` along with a Foolscap URL (or FURL) to connect to the controller and register the engine with the controller """ kernel_config = kernel_config_manager.get_config_obj() core_config = core_config_manager.get_config_obj() # Execute the mpi import statement that needs to call MPI_Init global mpi mpikey = kernel_config['mpi']['default'] mpi_import_statement = kernel_config['mpi'].get(mpikey, None) if mpi_import_statement is not None: try: exec mpi_import_statement in globals() except: mpi = None else: mpi = None # Start logging logfile = kernel_config['engine']['logfile'] if logfile: logfile = logfile + str(os.getpid()) + '.log' try: openLogFile = open(logfile, 'w') except: openLogFile = sys.stdout else: openLogFile = sys.stdout log.startLogging(openLogFile) # Create the underlying shell class and EngineService shell_class = import_item(core_config['shell']['shell_class']) engine_service = EngineService(shell_class, mpi=mpi) shell_import_statement = core_config['shell']['import_statement'] if shell_import_statement: try: engine_service.execute(shell_import_statement) except: log.msg("Error running import_statement: %s" % shell_import_statement) # Create the service hierarchy main_service = service.MultiService() engine_service.setServiceParent(main_service) tub_service = Tub() tub_service.setServiceParent(main_service) # This needs to be called before the connection is initiated main_service.startService() # This initiates the connection to the controller and calls # register_engine to tell the controller we are ready to do work engine_connector = EngineConnector(tub_service) furl_file = kernel_config['engine']['furl_file'] log.msg("Using furl file: %s" % furl_file) def call_connect(engine_service, furl_file): d = engine_connector.connect_to_controller(engine_service, furl_file) def handle_error(f): # If this print statement is replaced by a log.err(f) I get # an unhandled error, which makes no sense. I shouldn't have # to use a print statement here. My only thought is that # at the beginning of the process the logging is still starting up print "error connecting to controller:", f.getErrorMessage() reactor.callLater(0.1, reactor.stop) d.addErrback(handle_error) reactor.callWhenRunning(call_connect, engine_service, furl_file) reactor.run()
class FullSynchronousMultiEngineTestCase(DeferredTestCase, IFullSynchronousMultiEngineTestCase): def setUp(self): self.engines = [] self.controller = ControllerService() self.controller.startService() self.imultiengine = IMultiEngine(self.controller) self.mec_referenceable = IFCSynchronousMultiEngine(self.imultiengine) self.controller_tub = Tub() self.controller_tub.listenOn('tcp:10105:interface=127.0.0.1') self.controller_tub.setLocation('127.0.0.1:10105') furl = self.controller_tub.registerReference(self.mec_referenceable) self.controller_tub.startService() self.client_tub = ClientConnector() d = self.client_tub.get_multiengine_client(furl) d.addCallback(self.handle_got_client) return d def handle_got_client(self, client): self.multiengine = client def tearDown(self): dlist = [] # Shut down the multiengine client d = self.client_tub.tub.stopService() dlist.append(d) # Shut down the engines for e in self.engines: e.stopService() # Shut down the controller d = self.controller_tub.stopService() d.addBoth(lambda _: self.controller.stopService()) dlist.append(d) return defer.DeferredList(dlist) def test_mapper(self): self.addEngine(4) m = self.multiengine.mapper() self.assertEquals(m.multiengine, self.multiengine) self.assertEquals(m.dist, 'b') self.assertEquals(m.targets, 'all') self.assertEquals(m.block, True) def test_map_default(self): self.addEngine(4) m = self.multiengine.mapper() d = m.map(lambda x: 2 * x, range(10)) d.addCallback( lambda r: self.assertEquals(r, [2 * x for x in range(10)])) d.addCallback( lambda _: self.multiengine.map(lambda x: 2 * x, range(10))) d.addCallback( lambda r: self.assertEquals(r, [2 * x for x in range(10)])) return d def test_map_noblock(self): self.addEngine(4) m = self.multiengine.mapper(block=False) d = m.map(lambda x: 2 * x, range(10)) d.addCallback( lambda did: self.multiengine.get_pending_deferred(did, True)) d.addCallback( lambda r: self.assertEquals(r, [2 * x for x in range(10)])) return d def test_mapper_fail(self): self.addEngine(4) m = self.multiengine.mapper() d = m.map(lambda x: 1 / 0, range(10)) d.addBoth(lambda f: self.assertRaises(ZeroDivisionError, _raise_it, f)) return d def test_parallel(self): self.addEngine(4) p = self.multiengine.parallel() self.assert_(isinstance(p, ParallelFunction)) @p def f(x): return 2 * x d = f(range(10)) d.addCallback( lambda r: self.assertEquals(r, [2 * x for x in range(10)])) return d def test_parallel_noblock(self): self.addEngine(1) p = self.multiengine.parallel(block=False) self.assert_(isinstance(p, ParallelFunction)) @p def f(x): return 2 * x d = f(range(10)) d.addCallback( lambda did: self.multiengine.get_pending_deferred(did, True)) d.addCallback( lambda r: self.assertEquals(r, [2 * x for x in range(10)])) return d def test_parallel_fail(self): self.addEngine(4) p = self.multiengine.parallel() self.assert_(isinstance(p, ParallelFunction)) @p def f(x): return 1 / 0 d = f(range(10)) d.addBoth(lambda f: self.assertRaises(ZeroDivisionError, _raise_it, f)) return d
class AsyncClientConnector(object): """A class for getting remote references and clients from furls. This start a single :class:`Tub` for all remote reference and caches references. """ def __init__(self): self._remote_refs = {} self.tub = Tub() self.tub.startService() def _find_furl(self, profile='default', cluster_dir=None, furl_or_file=None, furl_file_name=None, ipython_dir=None): """Find a FURL file. If successful, this returns a FURL file that exists on the file system. The contents of the file have not been checked though. This is because we often have to deal with FURL file whose buffers have not been flushed. This raises an :exc:`~IPython.kernel.fcutil.FURLError` exception if a FURL file can't be found. This tries the following: 1. By the name ``furl_or_file``. 2. By ``cluster_dir`` and ``furl_file_name``. 3. By cluster profile with a default of ``default``. This uses ``ipython_dir``. """ # Try by furl_or_file if furl_or_file is not None: if is_valid_furl_or_file(furl_or_file): return furl_or_file if furl_file_name is None: raise FURLError('A furl_file_name must be provided if furl_or_file is not') # Try by cluster_dir if cluster_dir is not None: cluster_dir_obj = ClusterDir.find_cluster_dir(cluster_dir) sdir = cluster_dir_obj.security_dir furl_file = os.path.join(sdir, furl_file_name) validate_furl_or_file(furl_file) return furl_file # Try by profile if ipython_dir is None: ipython_dir = get_ipython_dir() if profile is not None: cluster_dir_obj = ClusterDir.find_cluster_dir_by_profile( ipython_dir, profile) sdir = cluster_dir_obj.security_dir furl_file = os.path.join(sdir, furl_file_name) validate_furl_or_file(furl_file) return furl_file raise FURLError('Could not find a valid FURL file.') def get_reference(self, furl_or_file): """Get a remote reference using a furl or a file containing a furl. Remote references are cached locally so once a remote reference has been retrieved for a given furl, the cached version is returned. Parameters ---------- furl_or_file : str A furl or a filename containing a furl. This should already be validated, but might not yet exist. Returns ------- A deferred to a remote reference """ furl = furl_or_file if furl in self._remote_refs: d = defer.succeed(self._remote_refs[furl]) else: d = self.tub.getReference(furl) d.addCallback(self._save_ref, furl) return d def _save_ref(self, ref, furl): """Cache a remote reference by its furl.""" self._remote_refs[furl] = ref return ref def get_task_client(self, profile='default', cluster_dir=None, furl_or_file=None, ipython_dir=None, delay=DELAY, max_tries=MAX_TRIES): """Get the task controller client. This method is a simple wrapper around `get_client` that passes in the default name of the task client FURL file. Usually only the ``profile`` option will be needed. If a FURL file can't be found by its profile, use ``cluster_dir`` or ``furl_or_file``. Parameters ---------- profile : str The name of a cluster directory profile (default="default"). The cluster directory "cluster_<profile>" will be searched for in ``os.getcwd()``, the ipython_dir and then in the directories listed in the :env:`IPCLUSTER_DIR_PATH` environment variable. cluster_dir : str The full path to a cluster directory. This is useful if profiles are not being used. furl_or_file : str A furl or a filename containing a FURL. This is useful if you simply know the location of the FURL file. ipython_dir : str The location of the ipython_dir if different from the default. This is used if the cluster directory is being found by profile. delay : float The initial delay between re-connection attempts. Susequent delays get longer according to ``delay[i] = 1.5*delay[i-1]``. max_tries : int The max number of re-connection attempts. Returns ------- A deferred to the actual client class. """ return self.get_client( profile, cluster_dir, furl_or_file, 'ipcontroller-tc.furl', ipython_dir, delay, max_tries ) def get_multiengine_client(self, profile='default', cluster_dir=None, furl_or_file=None, ipython_dir=None, delay=DELAY, max_tries=MAX_TRIES): """Get the multiengine controller client. This method is a simple wrapper around `get_client` that passes in the default name of the task client FURL file. Usually only the ``profile`` option will be needed. If a FURL file can't be found by its profile, use ``cluster_dir`` or ``furl_or_file``. Parameters ---------- profile : str The name of a cluster directory profile (default="default"). The cluster directory "cluster_<profile>" will be searched for in ``os.getcwd()``, the ipython_dir and then in the directories listed in the :env:`IPCLUSTER_DIR_PATH` environment variable. cluster_dir : str The full path to a cluster directory. This is useful if profiles are not being used. furl_or_file : str A furl or a filename containing a FURL. This is useful if you simply know the location of the FURL file. ipython_dir : str The location of the ipython_dir if different from the default. This is used if the cluster directory is being found by profile. delay : float The initial delay between re-connection attempts. Susequent delays get longer according to ``delay[i] = 1.5*delay[i-1]``. max_tries : int The max number of re-connection attempts. Returns ------- A deferred to the actual client class. """ return self.get_client( profile, cluster_dir, furl_or_file, 'ipcontroller-mec.furl', ipython_dir, delay, max_tries ) def get_client(self, profile='default', cluster_dir=None, furl_or_file=None, furl_file_name=None, ipython_dir=None, delay=DELAY, max_tries=MAX_TRIES): """Get a remote reference and wrap it in a client by furl. This method is a simple wrapper around `get_client` that passes in the default name of the task client FURL file. Usually only the ``profile`` option will be needed. If a FURL file can't be found by its profile, use ``cluster_dir`` or ``furl_or_file``. Parameters ---------- profile : str The name of a cluster directory profile (default="default"). The cluster directory "cluster_<profile>" will be searched for in ``os.getcwd()``, the ipython_dir and then in the directories listed in the :env:`IPCLUSTER_DIR_PATH` environment variable. cluster_dir : str The full path to a cluster directory. This is useful if profiles are not being used. furl_or_file : str A furl or a filename containing a FURL. This is useful if you simply know the location of the FURL file. furl_file_name : str The filename (not the full path) of the FURL. This must be provided if ``furl_or_file`` is not. ipython_dir : str The location of the ipython_dir if different from the default. This is used if the cluster directory is being found by profile. delay : float The initial delay between re-connection attempts. Susequent delays get longer according to ``delay[i] = 1.5*delay[i-1]``. max_tries : int The max number of re-connection attempts. Returns ------- A deferred to the actual client class. Or a failure to a :exc:`FURLError`. """ try: furl_file = self._find_furl( profile, cluster_dir, furl_or_file, furl_file_name, ipython_dir ) # If this succeeds, we know the furl file exists and has a .furl # extension, but it could still be empty. That is checked each # connection attempt. except FURLError: return defer.fail(failure.Failure()) def _wrap_remote_reference(rr): d = rr.callRemote('get_client_name') d.addCallback(lambda name: import_item(name)) def adapt(client_interface): client = client_interface(rr) client.tub = self.tub return client d.addCallback(adapt) return d d = self._try_to_connect(furl_file, delay, max_tries, attempt=0) d.addCallback(_wrap_remote_reference) d.addErrback(self._handle_error, furl_file) return d def _handle_error(self, f, furl_file): raise ClientConnectorError('Could not connect to the controller ' 'using the FURL file. This usually means that i) the controller ' 'was not started or ii) a firewall was blocking the client from ' 'connecting to the controller: %s' % furl_file) @inlineCallbacks def _try_to_connect(self, furl_or_file, delay, max_tries, attempt): """Try to connect to the controller with retry logic.""" if attempt < max_tries: log.msg("Connecting [%r]" % attempt) try: self.furl = find_furl(furl_or_file) # Uncomment this to see the FURL being tried. # log.msg("FURL: %s" % self.furl) rr = yield self.get_reference(self.furl) log.msg("Connected: %s" % furl_or_file) except: if attempt==max_tries-1: # This will propagate the exception all the way to the top # where it can be handled. raise else: yield sleep_deferred(delay) rr = yield self._try_to_connect( furl_or_file, 1.5*delay, max_tries, attempt+1 ) returnValue(rr) else: returnValue(rr) else: raise ClientConnectorError( 'Could not connect to controller, max_tries (%r) exceeded. ' 'This usually means that i) the controller was not started, ' 'or ii) a firewall was blocking the client from connecting ' 'to the controller.' % max_tries )
class FullSynchronousMultiEngineTestCase( DeferredTestCase, IFullSynchronousMultiEngineTestCase): def setUp(self): self.engines = [] self.controller = ControllerService() self.controller.startService() self.imultiengine = IMultiEngine(self.controller) self.mec_referenceable = IFCSynchronousMultiEngine(self.imultiengine) self.controller_tub = Tub() self.controller_tub.listenOn('tcp:10111:interface=127.0.0.1') self.controller_tub.setLocation('127.0.0.1:10111') furl = self.controller_tub.registerReference(self.mec_referenceable) self.controller_tub.startService() self.client_tub = AsyncClientConnector() d = self.client_tub.get_multiengine_client(furl_or_file=furl) d.addCallback(self.handle_got_client) return d def handle_got_client(self, client): self.multiengine = client def tearDown(self): dlist = [] # Shut down the multiengine client d = self.client_tub.tub.stopService() dlist.append(d) # Shut down the engines for e in self.engines: e.stopService() # Shut down the controller d = self.controller_tub.stopService() d.addBoth(lambda _: self.controller.stopService()) dlist.append(d) return defer.DeferredList(dlist) def test_mapper(self): self.addEngine(4) m = self.multiengine.mapper() self.assertEquals(m.multiengine,self.multiengine) self.assertEquals(m.dist,'b') self.assertEquals(m.targets,'all') self.assertEquals(m.block,True) def test_map_default(self): self.addEngine(4) m = self.multiengine.mapper() d = m.map(lambda x: 2*x, list(range(10))) d.addCallback(lambda r: self.assertEquals(r,[2*x for x in range(10)])) d.addCallback(lambda _: self.multiengine.map(lambda x: 2*x, list(range(10)))) d.addCallback(lambda r: self.assertEquals(r,[2*x for x in range(10)])) return d def test_map_noblock(self): self.addEngine(4) m = self.multiengine.mapper(block=False) d = m.map(lambda x: 2*x, list(range(10))) d.addCallback(lambda did: self.multiengine.get_pending_deferred(did, True)) d.addCallback(lambda r: self.assertEquals(r,[2*x for x in range(10)])) return d def test_mapper_fail(self): self.addEngine(4) m = self.multiengine.mapper() d = m.map(lambda x: 1/0, list(range(10))) d.addBoth(lambda f: self.assertRaises(ZeroDivisionError, _raise_it, f)) return d def test_parallel(self): self.addEngine(4) p = self.multiengine.parallel() self.assert_(isinstance(p, ParallelFunction)) @p def f(x): return 2*x d = f(list(range(10))) d.addCallback(lambda r: self.assertEquals(r,[2*x for x in range(10)])) return d def test_parallel_noblock(self): self.addEngine(1) p = self.multiengine.parallel(block=False) self.assert_(isinstance(p, ParallelFunction)) @p def f(x): return 2*x d = f(list(range(10))) d.addCallback(lambda did: self.multiengine.get_pending_deferred(did, True)) d.addCallback(lambda r: self.assertEquals(r,[2*x for x in range(10)])) return d def test_parallel_fail(self): self.addEngine(4) p = self.multiengine.parallel() self.assert_(isinstance(p, ParallelFunction)) @p def f(x): return 1/0 d = f(list(range(10))) d.addBoth(lambda f: self.assertRaises(ZeroDivisionError, _raise_it, f)) return d
class IPEngineApp(ApplicationWithClusterDir): name = u'ipengine' description = _description command_line_loader = IPEngineAppConfigLoader default_config_file_name = default_config_file_name auto_create_cluster_dir = True def create_default_config(self): super(IPEngineApp, self).create_default_config() # The engine should not clean logs as we don't want to remove the # active log files of other running engines. self.default_config.Global.clean_logs = False # Global config attributes self.default_config.Global.exec_lines = [] self.default_config.Global.shell_class = 'IPython.kernel.core.interpreter.Interpreter' # Configuration related to the controller # This must match the filename (path not included) that the controller # used for the FURL file. self.default_config.Global.furl_file_name = u'ipcontroller-engine.furl' # If given, this is the actual location of the controller's FURL file. # If not, this is computed using the profile, app_dir and furl_file_name self.default_config.Global.furl_file = u'' # The max number of connection attemps and the initial delay between # those attemps. self.default_config.Global.connect_delay = 0.1 self.default_config.Global.connect_max_tries = 15 # MPI related config attributes self.default_config.MPI.use = '' self.default_config.MPI.mpi4py = mpi4py_init self.default_config.MPI.pytrilinos = pytrilinos_init def post_load_command_line_config(self): pass def pre_construct(self): super(IPEngineApp, self).pre_construct() self.find_cont_furl_file() def find_cont_furl_file(self): """Set the furl file. Here we don't try to actually see if it exists for is valid as that is hadled by the connection logic. """ config = self.master_config # Find the actual controller FURL file if not config.Global.furl_file: try_this = os.path.join(config.Global.cluster_dir, config.Global.security_dir, config.Global.furl_file_name) config.Global.furl_file = try_this def construct(self): # This is the working dir by now. sys.path.insert(0, '') self.start_mpi() self.start_logging() # Create the underlying shell class and EngineService shell_class = import_item(self.master_config.Global.shell_class) self.engine_service = EngineService(shell_class, mpi=mpi) self.exec_lines() # Create the service hierarchy self.main_service = service.MultiService() self.engine_service.setServiceParent(self.main_service) self.tub_service = Tub() self.tub_service.setServiceParent(self.main_service) # This needs to be called before the connection is initiated self.main_service.startService() # This initiates the connection to the controller and calls # register_engine to tell the controller we are ready to do work self.engine_connector = EngineConnector(self.tub_service) log.msg("Using furl file: %s" % self.master_config.Global.furl_file) reactor.callWhenRunning(self.call_connect) def call_connect(self): d = self.engine_connector.connect_to_controller( self.engine_service, self.master_config.Global.furl_file, self.master_config.Global.connect_delay, self.master_config.Global.connect_max_tries) def handle_error(f): log.msg( 'Error connecting to controller. This usually means that ' 'i) the controller was not started, ii) a firewall was blocking ' 'the engine from connecting to the controller or iii) the engine ' ' was not pointed at the right FURL file:') log.msg(f.getErrorMessage()) reactor.callLater(0.1, reactor.stop) d.addErrback(handle_error) def start_mpi(self): global mpi mpikey = self.master_config.MPI.use mpi_import_statement = self.master_config.MPI.get(mpikey, None) if mpi_import_statement is not None: try: self.log.info("Initializing MPI:") self.log.info(mpi_import_statement) exec mpi_import_statement in globals() except: mpi = None else: mpi = None def exec_lines(self): for line in self.master_config.Global.exec_lines: try: log.msg("Executing statement: '%s'" % line) self.engine_service.execute(line) except: log.msg("Error executing statement: %s" % line) def start_app(self): reactor.run()