class EngineFCTest(DeferredTestCase, IEngineCoreTestCase,
IEngineSerializedTestCase, IEngineQueuedTestCase):
zi.implements(IControllerBase)
def setUp(self):
# Start a server and append to self.servers
self.controller_reference = FCRemoteEngineRefFromService(self)
self.controller_tub = Tub()
self.controller_tub.listenOn('tcp:10111:interface=127.0.0.1')
self.controller_tub.setLocation('127.0.0.1:10111')
furl = self.controller_tub.registerReference(self.controller_reference)
self.controller_tub.startService()
# Start an EngineService and append to services/client
self.engine_service = es.EngineService()
self.engine_service.startService()
self.engine_tub = Tub()
self.engine_tub.startService()
engine_connector = EngineConnector(self.engine_tub)
d = engine_connector.connect_to_controller(self.engine_service, furl)
# This deferred doesn't fire until after register_engine has returned and
# thus, self.engine has been defined and the tets can proceed.
return d
def tearDown(self):
dlist = []
# Shut down the engine
d = self.engine_tub.stopService()
dlist.append(d)
# Shut down the controller
d = self.controller_tub.stopService()
dlist.append(d)
return defer.DeferredList(dlist)
#---------------------------------------------------------------------------
# Make me look like a basic controller
#---------------------------------------------------------------------------
def register_engine(self,
engine_ref,
id=None,
ip=None,
port=None,
pid=None):
self.engine = IEngineQueued(IEngineBase(engine_ref))
return {'id': id}
def unregister_engine(self, id):
pass
class EngineFCTest(DeferredTestCase,
IEngineCoreTestCase,
IEngineSerializedTestCase,
IEngineQueuedTestCase
):
zi.implements(IControllerBase)
def setUp(self):
# Start a server and append to self.servers
self.controller_reference = FCRemoteEngineRefFromService(self)
self.controller_tub = Tub()
self.controller_tub.listenOn('tcp:10111:interface=127.0.0.1')
self.controller_tub.setLocation('127.0.0.1:10111')
furl = self.controller_tub.registerReference(self.controller_reference)
self.controller_tub.startService()
# Start an EngineService and append to services/client
self.engine_service = es.EngineService()
self.engine_service.startService()
self.engine_tub = Tub()
self.engine_tub.startService()
engine_connector = EngineConnector(self.engine_tub)
d = engine_connector.connect_to_controller(self.engine_service, furl)
# This deferred doesn't fire until after register_engine has returned and
# thus, self.engine has been defined and the tets can proceed.
return d
def tearDown(self):
dlist = []
# Shut down the engine
d = self.engine_tub.stopService()
dlist.append(d)
# Shut down the controller
d = self.controller_tub.stopService()
dlist.append(d)
return defer.DeferredList(dlist)
#---------------------------------------------------------------------------
# Make me look like a basic controller
#---------------------------------------------------------------------------
def register_engine(self, engine_ref, id=None, ip=None, port=None, pid=None):
self.engine = IEngineQueued(IEngineBase(engine_ref))
return {'id':id}
def unregister_engine(self, id):
pass
class FullSynchronousMultiEngineTestCase(DeferredTestCase,
IFullSynchronousMultiEngineTestCase):
def setUp(self):
self.engines = []
self.controller = ControllerService()
self.controller.startService()
self.imultiengine = IMultiEngine(self.controller)
self.mec_referenceable = IFCSynchronousMultiEngine(self.imultiengine)
self.controller_tub = Tub()
self.controller_tub.listenOn('tcp:10105:interface=127.0.0.1')
self.controller_tub.setLocation('127.0.0.1:10105')
furl = self.controller_tub.registerReference(self.mec_referenceable)
self.controller_tub.startService()
self.client_tub = ClientConnector()
d = self.client_tub.get_multiengine_client(furl)
d.addCallback(self.handle_got_client)
return d
def handle_got_client(self, client):
self.multiengine = client
def tearDown(self):
dlist = []
# Shut down the multiengine client
d = self.client_tub.tub.stopService()
dlist.append(d)
# Shut down the engines
for e in self.engines:
e.stopService()
# Shut down the controller
d = self.controller_tub.stopService()
d.addBoth(lambda _: self.controller.stopService())
dlist.append(d)
return defer.DeferredList(dlist)
def test_mapper(self):
self.addEngine(4)
m = self.multiengine.mapper()
self.assertEquals(m.multiengine, self.multiengine)
self.assertEquals(m.dist, 'b')
self.assertEquals(m.targets, 'all')
self.assertEquals(m.block, True)
def test_map_default(self):
self.addEngine(4)
m = self.multiengine.mapper()
d = m.map(lambda x: 2 * x, range(10))
d.addCallback(
lambda r: self.assertEquals(r, [2 * x for x in range(10)]))
d.addCallback(
lambda _: self.multiengine.map(lambda x: 2 * x, range(10)))
d.addCallback(
lambda r: self.assertEquals(r, [2 * x for x in range(10)]))
return d
def test_map_noblock(self):
self.addEngine(4)
m = self.multiengine.mapper(block=False)
d = m.map(lambda x: 2 * x, range(10))
d.addCallback(
lambda did: self.multiengine.get_pending_deferred(did, True))
d.addCallback(
lambda r: self.assertEquals(r, [2 * x for x in range(10)]))
return d
def test_mapper_fail(self):
self.addEngine(4)
m = self.multiengine.mapper()
d = m.map(lambda x: 1 / 0, range(10))
d.addBoth(lambda f: self.assertRaises(ZeroDivisionError, _raise_it, f))
return d
def test_parallel(self):
self.addEngine(4)
p = self.multiengine.parallel()
self.assert_(isinstance(p, ParallelFunction))
@p
def f(x):
return 2 * x
d = f(range(10))
d.addCallback(
lambda r: self.assertEquals(r, [2 * x for x in range(10)]))
return d
def test_parallel_noblock(self):
self.addEngine(1)
p = self.multiengine.parallel(block=False)
self.assert_(isinstance(p, ParallelFunction))
@p
def f(x):
return 2 * x
d = f(range(10))
d.addCallback(
lambda did: self.multiengine.get_pending_deferred(did, True))
d.addCallback(
lambda r: self.assertEquals(r, [2 * x for x in range(10)]))
return d
def test_parallel_fail(self):
self.addEngine(4)
p = self.multiengine.parallel()
self.assert_(isinstance(p, ParallelFunction))
@p
def f(x):
return 1 / 0
d = f(range(10))
d.addBoth(lambda f: self.assertRaises(ZeroDivisionError, _raise_it, f))
return d
class TaskTest(DeferredTestCase, ITaskControllerTestCase):
def setUp(self):
self.engines = []
self.controller = cs.ControllerService()
self.controller.startService()
self.imultiengine = me.IMultiEngine(self.controller)
self.itc = taskmodule.ITaskController(self.controller)
self.itc.failurePenalty = 0
self.mec_referenceable = IFCSynchronousMultiEngine(self.imultiengine)
self.tc_referenceable = IFCTaskController(self.itc)
self.controller_tub = Tub()
self.controller_tub.listenOn('tcp:10105:interface=127.0.0.1')
self.controller_tub.setLocation('127.0.0.1:10105')
mec_furl = self.controller_tub.registerReference(self.mec_referenceable)
tc_furl = self.controller_tub.registerReference(self.tc_referenceable)
self.controller_tub.startService()
self.client_tub = ClientConnector()
d = self.client_tub.get_multiengine_client(mec_furl)
d.addCallback(self.handle_mec_client)
d.addCallback(lambda _: self.client_tub.get_task_client(tc_furl))
d.addCallback(self.handle_tc_client)
return d
def handle_mec_client(self, client):
self.multiengine = client
def handle_tc_client(self, client):
self.tc = client
def tearDown(self):
dlist = []
# Shut down the multiengine client
d = self.client_tub.tub.stopService()
dlist.append(d)
# Shut down the engines
for e in self.engines:
e.stopService()
# Shut down the controller
d = self.controller_tub.stopService()
d.addBoth(lambda _: self.controller.stopService())
dlist.append(d)
return defer.DeferredList(dlist)
def test_mapper(self):
self.addEngine(1)
m = self.tc.mapper()
self.assertEquals(m.task_controller,self.tc)
self.assertEquals(m.clear_before,False)
self.assertEquals(m.clear_after,False)
self.assertEquals(m.retries,0)
self.assertEquals(m.recovery_task,None)
self.assertEquals(m.depend,None)
self.assertEquals(m.block,True)
def test_map_default(self):
self.addEngine(1)
m = self.tc.mapper()
d = m.map(lambda x: 2*x, range(10))
d.addCallback(lambda r: self.assertEquals(r,[2*x for x in range(10)]))
d.addCallback(lambda _: self.tc.map(lambda x: 2*x, range(10)))
d.addCallback(lambda r: self.assertEquals(r,[2*x for x in range(10)]))
return d
def test_map_noblock(self):
self.addEngine(1)
m = self.tc.mapper(block=False)
d = m.map(lambda x: 2*x, range(10))
d.addCallback(lambda r: self.assertEquals(r,[x for x in range(10)]))
return d
def test_mapper_fail(self):
self.addEngine(1)
m = self.tc.mapper()
d = m.map(lambda x: 1/0, range(10))
d.addBoth(lambda f: self.assertRaises(ZeroDivisionError, _raise_it, f))
return d
def test_parallel(self):
self.addEngine(1)
p = self.tc.parallel()
self.assert_(isinstance(p, ParallelFunction))
@p
def f(x): return 2*x
d = f(range(10))
d.addCallback(lambda r: self.assertEquals(r,[2*x for x in range(10)]))
return d
def test_parallel_noblock(self):
self.addEngine(1)
p = self.tc.parallel(block=False)
self.assert_(isinstance(p, ParallelFunction))
@p
def f(x): return 2*x
d = f(range(10))
d.addCallback(lambda r: self.assertEquals(r,[x for x in range(10)]))
return d
def test_parallel_fail(self):
self.addEngine(1)
p = self.tc.parallel()
self.assert_(isinstance(p, ParallelFunction))
@p
def f(x): return 1/0
d = f(range(10))
d.addBoth(lambda f: self.assertRaises(ZeroDivisionError, _raise_it, f))
return d
class FullSynchronousMultiEngineTestCase(
DeferredTestCase, IFullSynchronousMultiEngineTestCase):
def setUp(self):
self.engines = []
self.controller = ControllerService()
self.controller.startService()
self.imultiengine = IMultiEngine(self.controller)
self.mec_referenceable = IFCSynchronousMultiEngine(self.imultiengine)
self.controller_tub = Tub()
self.controller_tub.listenOn('tcp:10111:interface=127.0.0.1')
self.controller_tub.setLocation('127.0.0.1:10111')
furl = self.controller_tub.registerReference(self.mec_referenceable)
self.controller_tub.startService()
self.client_tub = AsyncClientConnector()
d = self.client_tub.get_multiengine_client(furl_or_file=furl)
d.addCallback(self.handle_got_client)
return d
def handle_got_client(self, client):
self.multiengine = client
def tearDown(self):
dlist = []
# Shut down the multiengine client
d = self.client_tub.tub.stopService()
dlist.append(d)
# Shut down the engines
for e in self.engines:
e.stopService()
# Shut down the controller
d = self.controller_tub.stopService()
d.addBoth(lambda _: self.controller.stopService())
dlist.append(d)
return defer.DeferredList(dlist)
def test_mapper(self):
self.addEngine(4)
m = self.multiengine.mapper()
self.assertEquals(m.multiengine,self.multiengine)
self.assertEquals(m.dist,'b')
self.assertEquals(m.targets,'all')
self.assertEquals(m.block,True)
def test_map_default(self):
self.addEngine(4)
m = self.multiengine.mapper()
d = m.map(lambda x: 2*x, list(range(10)))
d.addCallback(lambda r: self.assertEquals(r,[2*x for x in range(10)]))
d.addCallback(lambda _: self.multiengine.map(lambda x: 2*x, list(range(10))))
d.addCallback(lambda r: self.assertEquals(r,[2*x for x in range(10)]))
return d
def test_map_noblock(self):
self.addEngine(4)
m = self.multiengine.mapper(block=False)
d = m.map(lambda x: 2*x, list(range(10)))
d.addCallback(lambda did: self.multiengine.get_pending_deferred(did, True))
d.addCallback(lambda r: self.assertEquals(r,[2*x for x in range(10)]))
return d
def test_mapper_fail(self):
self.addEngine(4)
m = self.multiengine.mapper()
d = m.map(lambda x: 1/0, list(range(10)))
d.addBoth(lambda f: self.assertRaises(ZeroDivisionError, _raise_it, f))
return d
def test_parallel(self):
self.addEngine(4)
p = self.multiengine.parallel()
self.assert_(isinstance(p, ParallelFunction))
@p
def f(x): return 2*x
d = f(list(range(10)))
d.addCallback(lambda r: self.assertEquals(r,[2*x for x in range(10)]))
return d
def test_parallel_noblock(self):
self.addEngine(1)
p = self.multiengine.parallel(block=False)
self.assert_(isinstance(p, ParallelFunction))
@p
def f(x): return 2*x
d = f(list(range(10)))
d.addCallback(lambda did: self.multiengine.get_pending_deferred(did, True))
d.addCallback(lambda r: self.assertEquals(r,[2*x for x in range(10)]))
return d
def test_parallel_fail(self):
self.addEngine(4)
p = self.multiengine.parallel()
self.assert_(isinstance(p, ParallelFunction))
@p
def f(x): return 1/0
d = f(list(range(10)))
d.addBoth(lambda f: self.assertRaises(ZeroDivisionError, _raise_it, f))
return d
class TaskTest(DeferredTestCase, ITaskControllerTestCase):
def setUp(self):
self.engines = []
self.controller = cs.ControllerService()
self.controller.startService()
self.imultiengine = me.IMultiEngine(self.controller)
self.itc = taskmodule.ITaskController(self.controller)
self.itc.failurePenalty = 0
self.mec_referenceable = IFCSynchronousMultiEngine(self.imultiengine)
self.tc_referenceable = IFCTaskController(self.itc)
self.controller_tub = Tub()
self.controller_tub.listenOn('tcp:10105:interface=127.0.0.1')
self.controller_tub.setLocation('127.0.0.1:10105')
mec_furl = self.controller_tub.registerReference(self.mec_referenceable)
tc_furl = self.controller_tub.registerReference(self.tc_referenceable)
self.controller_tub.startService()
self.client_tub = ClientConnector()
d = self.client_tub.get_multiengine_client(mec_furl)
d.addCallback(self.handle_mec_client)
d.addCallback(lambda _: self.client_tub.get_task_client(tc_furl))
d.addCallback(self.handle_tc_client)
return d
def handle_mec_client(self, client):
self.multiengine = client
def handle_tc_client(self, client):
self.tc = client
def tearDown(self):
dlist = []
# Shut down the multiengine client
d = self.client_tub.tub.stopService()
dlist.append(d)
# Shut down the engines
for e in self.engines:
e.stopService()
# Shut down the controller
d = self.controller_tub.stopService()
d.addBoth(lambda _: self.controller.stopService())
dlist.append(d)
return defer.DeferredList(dlist)
def test_mapper(self):
self.addEngine(1)
m = self.tc.mapper()
self.assertEquals(m.task_controller,self.tc)
self.assertEquals(m.clear_before,False)
self.assertEquals(m.clear_after,False)
self.assertEquals(m.retries,0)
self.assertEquals(m.recovery_task,None)
self.assertEquals(m.depend,None)
self.assertEquals(m.block,True)
def test_map_default(self):
self.addEngine(1)
m = self.tc.mapper()
d = m.map(lambda x: 2*x, range(10))
d.addCallback(lambda r: self.assertEquals(r,[2*x for x in range(10)]))
d.addCallback(lambda _: self.tc.map(lambda x: 2*x, range(10)))
d.addCallback(lambda r: self.assertEquals(r,[2*x for x in range(10)]))
return d
def test_map_noblock(self):
self.addEngine(1)
m = self.tc.mapper(block=False)
d = m.map(lambda x: 2*x, range(10))
d.addCallback(lambda r: self.assertEquals(r,[x for x in range(10)]))
return d
def test_mapper_fail(self):
self.addEngine(1)
m = self.tc.mapper()
d = m.map(lambda x: 1/0, range(10))
d.addBoth(lambda f: self.assertRaises(ZeroDivisionError, _raise_it, f))
return d
def test_parallel(self):
self.addEngine(1)
p = self.tc.parallel()
self.assert_(isinstance(p, ParallelFunction))
@p
def f(x): return 2*x
d = f(range(10))
d.addCallback(lambda r: self.assertEquals(r,[2*x for x in range(10)]))
return d
def test_parallel_noblock(self):
self.addEngine(1)
p = self.tc.parallel(block=False)
self.assert_(isinstance(p, ParallelFunction))
@p
def f(x): return 2*x
d = f(range(10))
d.addCallback(lambda r: self.assertEquals(r,[x for x in range(10)]))
return d
def test_parallel_fail(self):
self.addEngine(1)
p = self.tc.parallel()
self.assert_(isinstance(p, ParallelFunction))
@p
def f(x): return 1/0
d = f(range(10))
d.addBoth(lambda f: self.assertRaises(ZeroDivisionError, _raise_it, f))
return d
