def setUp(self):
self.engines = []
self.controller = cs.ControllerService()
self.controller.startService()
self.imultiengine = me.IMultiEngine(self.controller)
self.itc = taskmodule.ITaskController(self.controller)
self.itc.failurePenalty = 0
self.mec_referenceable = IFCSynchronousMultiEngine(self.imultiengine)
self.tc_referenceable = IFCTaskController(self.itc)
self.controller_tub = Tub()
self.controller_tub.listenOn('tcp:10111:interface=127.0.0.1')
self.controller_tub.setLocation('127.0.0.1:10111')
mec_furl = self.controller_tub.registerReference(
self.mec_referenceable)
tc_furl = self.controller_tub.registerReference(self.tc_referenceable)
self.controller_tub.startService()
self.client_tub = AsyncClientConnector()
d = self.client_tub.get_multiengine_client(furl_or_file=mec_furl)
d.addCallback(self.handle_mec_client)
d.addCallback(
lambda _: self.client_tub.get_task_client(furl_or_file=tc_furl))
d.addCallback(self.handle_tc_client)
return d
def setUp(self):
self.engines = []
self.controller = cs.ControllerService()
self.controller.startService()
self.imultiengine = me.IMultiEngine(self.controller)
self.itc = taskmodule.ITaskController(self.controller)
self.itc.failurePenalty = 0
self.mec_referenceable = IFCSynchronousMultiEngine(self.imultiengine)
self.tc_referenceable = IFCTaskController(self.itc)
self.controller_tub = Tub()
self.controller_tub.listenOn('tcp:10111:interface=127.0.0.1')
self.controller_tub.setLocation('127.0.0.1:10111')
mec_furl = self.controller_tub.registerReference(self.mec_referenceable)
tc_furl = self.controller_tub.registerReference(self.tc_referenceable)
self.controller_tub.startService()
self.client_tub = AsyncClientConnector()
d = self.client_tub.get_multiengine_client(furl_or_file=mec_furl)
d.addCallback(self.handle_mec_client)
d.addCallback(lambda _: self.client_tub.get_task_client(furl_or_file=tc_furl))
d.addCallback(self.handle_tc_client)
return d
def setUp(self):
self.engines = []
self.controller = ControllerService()
self.controller.startService()
self.imultiengine = IMultiEngine(self.controller)
self.mec_referenceable = IFCSynchronousMultiEngine(self.imultiengine)
self.controller_tub = Tub()
self.controller_tub.listenOn('tcp:10111:interface=127.0.0.1')
self.controller_tub.setLocation('127.0.0.1:10111')
furl = self.controller_tub.registerReference(self.mec_referenceable)
self.controller_tub.startService()
self.client_tub = AsyncClientConnector()
d = self.client_tub.get_multiengine_client(furl_or_file=furl)
d.addCallback(self.handle_got_client)
return d
def setUp(self):
self.engines = []
self.controller = ControllerService()
self.controller.startService()
self.imultiengine = IMultiEngine(self.controller)
self.mec_referenceable = IFCSynchronousMultiEngine(self.imultiengine)
self.controller_tub = Tub()
self.controller_tub.listenOn('tcp:10111:interface=127.0.0.1')
self.controller_tub.setLocation('127.0.0.1:10111')
furl = self.controller_tub.registerReference(self.mec_referenceable)
self.controller_tub.startService()
self.client_tub = AsyncClientConnector()
d = self.client_tub.get_multiengine_client(furl_or_file=furl)
d.addCallback(self.handle_got_client)
return d
deferreds to the result. The main methods are are `get_*_client` and `get_client`. """ #----------------------------------------------------------------------------- # Copyright (C) 2008-2009 The IPython Development Team # # Distributed under the terms of the BSD License. The full license is in # the file COPYING, distributed as part of this software. #----------------------------------------------------------------------------- #----------------------------------------------------------------------------- # Imports #----------------------------------------------------------------------------- from IPython.kernel import codeutil from IPython.kernel.clientconnector import (AsyncClientConnector, AsyncCluster) # Other things that the user will need from IPython.kernel.task import MapTask, StringTask from IPython.kernel.error import CompositeError #----------------------------------------------------------------------------- # Code #----------------------------------------------------------------------------- _client_tub = AsyncClientConnector() get_multiengine_client = _client_tub.get_multiengine_client get_task_client = _client_tub.get_task_client get_client = _client_tub.get_client
class FullSynchronousMultiEngineTestCase(
DeferredTestCase, IFullSynchronousMultiEngineTestCase):
def setUp(self):
self.engines = []
self.controller = ControllerService()
self.controller.startService()
self.imultiengine = IMultiEngine(self.controller)
self.mec_referenceable = IFCSynchronousMultiEngine(self.imultiengine)
self.controller_tub = Tub()
self.controller_tub.listenOn('tcp:10111:interface=127.0.0.1')
self.controller_tub.setLocation('127.0.0.1:10111')
furl = self.controller_tub.registerReference(self.mec_referenceable)
self.controller_tub.startService()
self.client_tub = AsyncClientConnector()
d = self.client_tub.get_multiengine_client(furl_or_file=furl)
d.addCallback(self.handle_got_client)
return d
def handle_got_client(self, client):
self.multiengine = client
def tearDown(self):
dlist = []
# Shut down the multiengine client
d = self.client_tub.tub.stopService()
dlist.append(d)
# Shut down the engines
for e in self.engines:
e.stopService()
# Shut down the controller
d = self.controller_tub.stopService()
d.addBoth(lambda _: self.controller.stopService())
dlist.append(d)
return defer.DeferredList(dlist)
def test_mapper(self):
self.addEngine(4)
m = self.multiengine.mapper()
self.assertEquals(m.multiengine,self.multiengine)
self.assertEquals(m.dist,'b')
self.assertEquals(m.targets,'all')
self.assertEquals(m.block,True)
def test_map_default(self):
self.addEngine(4)
m = self.multiengine.mapper()
d = m.map(lambda x: 2*x, list(range(10)))
d.addCallback(lambda r: self.assertEquals(r,[2*x for x in range(10)]))
d.addCallback(lambda _: self.multiengine.map(lambda x: 2*x, list(range(10))))
d.addCallback(lambda r: self.assertEquals(r,[2*x for x in range(10)]))
return d
def test_map_noblock(self):
self.addEngine(4)
m = self.multiengine.mapper(block=False)
d = m.map(lambda x: 2*x, list(range(10)))
d.addCallback(lambda did: self.multiengine.get_pending_deferred(did, True))
d.addCallback(lambda r: self.assertEquals(r,[2*x for x in range(10)]))
return d
def test_mapper_fail(self):
self.addEngine(4)
m = self.multiengine.mapper()
d = m.map(lambda x: 1/0, list(range(10)))
d.addBoth(lambda f: self.assertRaises(ZeroDivisionError, _raise_it, f))
return d
def test_parallel(self):
self.addEngine(4)
p = self.multiengine.parallel()
self.assert_(isinstance(p, ParallelFunction))
@p
def f(x): return 2*x
d = f(list(range(10)))
d.addCallback(lambda r: self.assertEquals(r,[2*x for x in range(10)]))
return d
def test_parallel_noblock(self):
self.addEngine(1)
p = self.multiengine.parallel(block=False)
self.assert_(isinstance(p, ParallelFunction))
@p
def f(x): return 2*x
d = f(list(range(10)))
d.addCallback(lambda did: self.multiengine.get_pending_deferred(did, True))
d.addCallback(lambda r: self.assertEquals(r,[2*x for x in range(10)]))
return d
def test_parallel_fail(self):
self.addEngine(4)
p = self.multiengine.parallel()
self.assert_(isinstance(p, ParallelFunction))
@p
def f(x): return 1/0
d = f(list(range(10)))
d.addBoth(lambda f: self.assertRaises(ZeroDivisionError, _raise_it, f))
return d
class TaskTest(DeferredTestCase, ITaskControllerTestCase):
def setUp(self):
self.engines = []
self.controller = cs.ControllerService()
self.controller.startService()
self.imultiengine = me.IMultiEngine(self.controller)
self.itc = taskmodule.ITaskController(self.controller)
self.itc.failurePenalty = 0
self.mec_referenceable = IFCSynchronousMultiEngine(self.imultiengine)
self.tc_referenceable = IFCTaskController(self.itc)
self.controller_tub = Tub()
self.controller_tub.listenOn('tcp:10111:interface=127.0.0.1')
self.controller_tub.setLocation('127.0.0.1:10111')
mec_furl = self.controller_tub.registerReference(self.mec_referenceable)
tc_furl = self.controller_tub.registerReference(self.tc_referenceable)
self.controller_tub.startService()
self.client_tub = AsyncClientConnector()
d = self.client_tub.get_multiengine_client(furl_or_file=mec_furl)
d.addCallback(self.handle_mec_client)
d.addCallback(lambda _: self.client_tub.get_task_client(furl_or_file=tc_furl))
d.addCallback(self.handle_tc_client)
return d
def handle_mec_client(self, client):
self.multiengine = client
def handle_tc_client(self, client):
self.tc = client
def tearDown(self):
dlist = []
# Shut down the multiengine client
d = self.client_tub.tub.stopService()
dlist.append(d)
# Shut down the engines
for e in self.engines:
e.stopService()
# Shut down the controller
d = self.controller_tub.stopService()
d.addBoth(lambda _: self.controller.stopService())
dlist.append(d)
return defer.DeferredList(dlist)
def test_mapper(self):
self.addEngine(1)
m = self.tc.mapper()
self.assertEquals(m.task_controller,self.tc)
self.assertEquals(m.clear_before,False)
self.assertEquals(m.clear_after,False)
self.assertEquals(m.retries,0)
self.assertEquals(m.recovery_task,None)
self.assertEquals(m.depend,None)
self.assertEquals(m.block,True)
def test_map_default(self):
self.addEngine(1)
m = self.tc.mapper()
d = m.map(lambda x: 2*x, list(range(10)))
d.addCallback(lambda r: self.assertEquals(r,[2*x for x in range(10)]))
d.addCallback(lambda _: self.tc.map(lambda x: 2*x, list(range(10))))
d.addCallback(lambda r: self.assertEquals(r,[2*x for x in range(10)]))
return d
def test_map_noblock(self):
self.addEngine(1)
m = self.tc.mapper(block=False)
d = m.map(lambda x: 2*x, list(range(10)))
d.addCallback(lambda r: self.assertEquals(r,[x for x in range(10)]))
return d
def test_mapper_fail(self):
self.addEngine(1)
m = self.tc.mapper()
d = m.map(lambda x: 1/0, list(range(10)))
d.addBoth(lambda f: self.assertRaises(ZeroDivisionError, _raise_it, f))
return d
def test_parallel(self):
self.addEngine(1)
p = self.tc.parallel()
self.assert_(isinstance(p, ParallelFunction))
@p
def f(x): return 2*x
d = f(list(range(10)))
d.addCallback(lambda r: self.assertEquals(r,[2*x for x in range(10)]))
return d
def test_parallel_noblock(self):
self.addEngine(1)
p = self.tc.parallel(block=False)
self.assert_(isinstance(p, ParallelFunction))
@p
def f(x): return 2*x
d = f(list(range(10)))
d.addCallback(lambda r: self.assertEquals(r,[x for x in range(10)]))
return d
def test_parallel_fail(self):
self.addEngine(1)
p = self.tc.parallel()
self.assert_(isinstance(p, ParallelFunction))
@p
def f(x): return 1/0
d = f(list(range(10)))
d.addBoth(lambda f: self.assertRaises(ZeroDivisionError, _raise_it, f))
return d
class TaskTest(DeferredTestCase, ITaskControllerTestCase):
def setUp(self):
self.engines = []
self.controller = cs.ControllerService()
self.controller.startService()
self.imultiengine = me.IMultiEngine(self.controller)
self.itc = taskmodule.ITaskController(self.controller)
self.itc.failurePenalty = 0
self.mec_referenceable = IFCSynchronousMultiEngine(self.imultiengine)
self.tc_referenceable = IFCTaskController(self.itc)
self.controller_tub = Tub()
self.controller_tub.listenOn('tcp:10111:interface=127.0.0.1')
self.controller_tub.setLocation('127.0.0.1:10111')
mec_furl = self.controller_tub.registerReference(
self.mec_referenceable)
tc_furl = self.controller_tub.registerReference(self.tc_referenceable)
self.controller_tub.startService()
self.client_tub = AsyncClientConnector()
d = self.client_tub.get_multiengine_client(furl_or_file=mec_furl)
d.addCallback(self.handle_mec_client)
d.addCallback(
lambda _: self.client_tub.get_task_client(furl_or_file=tc_furl))
d.addCallback(self.handle_tc_client)
return d
def handle_mec_client(self, client):
self.multiengine = client
def handle_tc_client(self, client):
self.tc = client
def tearDown(self):
dlist = []
# Shut down the multiengine client
d = self.client_tub.tub.stopService()
dlist.append(d)
# Shut down the engines
for e in self.engines:
e.stopService()
# Shut down the controller
d = self.controller_tub.stopService()
d.addBoth(lambda _: self.controller.stopService())
dlist.append(d)
return defer.DeferredList(dlist)
def test_mapper(self):
self.addEngine(1)
m = self.tc.mapper()
self.assertEquals(m.task_controller, self.tc)
self.assertEquals(m.clear_before, False)
self.assertEquals(m.clear_after, False)
self.assertEquals(m.retries, 0)
self.assertEquals(m.recovery_task, None)
self.assertEquals(m.depend, None)
self.assertEquals(m.block, True)
def test_map_default(self):
self.addEngine(1)
m = self.tc.mapper()
d = m.map(lambda x: 2 * x, range(10))
d.addCallback(
lambda r: self.assertEquals(r, [2 * x for x in range(10)]))
d.addCallback(lambda _: self.tc.map(lambda x: 2 * x, range(10)))
d.addCallback(
lambda r: self.assertEquals(r, [2 * x for x in range(10)]))
return d
def test_map_noblock(self):
self.addEngine(1)
m = self.tc.mapper(block=False)
d = m.map(lambda x: 2 * x, range(10))
d.addCallback(lambda r: self.assertEquals(r, [x for x in range(10)]))
return d
def test_mapper_fail(self):
self.addEngine(1)
m = self.tc.mapper()
d = m.map(lambda x: 1 / 0, range(10))
d.addBoth(lambda f: self.assertRaises(ZeroDivisionError, _raise_it, f))
return d
def test_parallel(self):
self.addEngine(1)
p = self.tc.parallel()
self.assert_(isinstance(p, ParallelFunction))
@p
def f(x):
return 2 * x
d = f(range(10))
d.addCallback(
lambda r: self.assertEquals(r, [2 * x for x in range(10)]))
return d
def test_parallel_noblock(self):
self.addEngine(1)
p = self.tc.parallel(block=False)
self.assert_(isinstance(p, ParallelFunction))
@p
def f(x):
return 2 * x
d = f(range(10))
d.addCallback(lambda r: self.assertEquals(r, [x for x in range(10)]))
return d
def test_parallel_fail(self):
self.addEngine(1)
p = self.tc.parallel()
self.assert_(isinstance(p, ParallelFunction))
@p
def f(x):
return 1 / 0
d = f(range(10))
d.addBoth(lambda f: self.assertRaises(ZeroDivisionError, _raise_it, f))
return d
class FullSynchronousMultiEngineTestCase(DeferredTestCase,
IFullSynchronousMultiEngineTestCase):
def setUp(self):
self.engines = []
self.controller = ControllerService()
self.controller.startService()
self.imultiengine = IMultiEngine(self.controller)
self.mec_referenceable = IFCSynchronousMultiEngine(self.imultiengine)
self.controller_tub = Tub()
self.controller_tub.listenOn('tcp:10111:interface=127.0.0.1')
self.controller_tub.setLocation('127.0.0.1:10111')
furl = self.controller_tub.registerReference(self.mec_referenceable)
self.controller_tub.startService()
self.client_tub = AsyncClientConnector()
d = self.client_tub.get_multiengine_client(furl_or_file=furl)
d.addCallback(self.handle_got_client)
return d
def handle_got_client(self, client):
self.multiengine = client
def tearDown(self):
dlist = []
# Shut down the multiengine client
d = self.client_tub.tub.stopService()
dlist.append(d)
# Shut down the engines
for e in self.engines:
e.stopService()
# Shut down the controller
d = self.controller_tub.stopService()
d.addBoth(lambda _: self.controller.stopService())
dlist.append(d)
return defer.DeferredList(dlist)
def test_mapper(self):
self.addEngine(4)
m = self.multiengine.mapper()
self.assertEquals(m.multiengine, self.multiengine)
self.assertEquals(m.dist, 'b')
self.assertEquals(m.targets, 'all')
self.assertEquals(m.block, True)
def test_map_default(self):
self.addEngine(4)
m = self.multiengine.mapper()
d = m.map(lambda x: 2 * x, range(10))
d.addCallback(
lambda r: self.assertEquals(r, [2 * x for x in range(10)]))
d.addCallback(
lambda _: self.multiengine.map(lambda x: 2 * x, range(10)))
d.addCallback(
lambda r: self.assertEquals(r, [2 * x for x in range(10)]))
return d
def test_map_noblock(self):
self.addEngine(4)
m = self.multiengine.mapper(block=False)
d = m.map(lambda x: 2 * x, range(10))
d.addCallback(
lambda did: self.multiengine.get_pending_deferred(did, True))
d.addCallback(
lambda r: self.assertEquals(r, [2 * x for x in range(10)]))
return d
def test_mapper_fail(self):
self.addEngine(4)
m = self.multiengine.mapper()
d = m.map(lambda x: 1 / 0, range(10))
d.addBoth(lambda f: self.assertRaises(ZeroDivisionError, _raise_it, f))
return d
def test_parallel(self):
self.addEngine(4)
p = self.multiengine.parallel()
self.assert_(isinstance(p, ParallelFunction))
@p
def f(x):
return 2 * x
d = f(range(10))
d.addCallback(
lambda r: self.assertEquals(r, [2 * x for x in range(10)]))
return d
def test_parallel_noblock(self):
self.addEngine(1)
p = self.multiengine.parallel(block=False)
self.assert_(isinstance(p, ParallelFunction))
@p
def f(x):
return 2 * x
d = f(range(10))
d.addCallback(
lambda did: self.multiengine.get_pending_deferred(did, True))
d.addCallback(
lambda r: self.assertEquals(r, [2 * x for x in range(10)]))
return d
def test_parallel_fail(self):
self.addEngine(4)
p = self.multiengine.parallel()
self.assert_(isinstance(p, ParallelFunction))
@p
def f(x):
return 1 / 0
d = f(range(10))
d.addBoth(lambda f: self.assertRaises(ZeroDivisionError, _raise_it, f))
return d