def testNoEnginesRegistered(self):
badID = 'all'
d = self.multiengine.execute(badID, 'a=5')
d.addErrback(
lambda f: self.assertRaises(NoEnginesRegistered, f.raiseException))
d.addCallback(lambda _: self.multiengine.push(badID, a=5))
d.addErrback(
lambda f: self.assertRaises(NoEnginesRegistered, f.raiseException))
d.addCallback(lambda _: self.multiengine.pull(badID, 'a'))
d.addErrback(
lambda f: self.assertRaises(NoEnginesRegistered, f.raiseException))
d.addCallback(lambda _: self.multiengine.getResult(badID))
d.addErrback(
lambda f: self.assertRaises(NoEnginesRegistered, f.raiseException))
d.addCallback(lambda _: self.multiengine.reset(badID))
d.addErrback(
lambda f: self.assertRaises(NoEnginesRegistered, f.raiseException))
d.addCallback(lambda _: self.multiengine.keys(badID))
d.addErrback(
lambda f: self.assertRaises(NoEnginesRegistered, f.raiseException))
d.addCallback(lambda _: self.multiengine.pushSerialized(
badID, a=newserialized.serialize(10)))
d.addErrback(
lambda f: self.assertRaises(NoEnginesRegistered, f.raiseException))
d.addCallback(lambda _: self.multiengine.pullSerialized(badID, 'a'))
d.addErrback(
lambda f: self.assertRaises(NoEnginesRegistered, f.raiseException))
d.addCallback(lambda _: self.multiengine.queueStatus(badID))
d.addErrback(
lambda f: self.assertRaises(NoEnginesRegistered, f.raiseException))
return d
def pushPaging(self, key, obj):
try:
serial = newserialized.serialize(obj)
except:
return defer.fail(failure.Failure())
else:
return self.pushPagingSerialized(key, serial)
def testInvalidEngineID(self):
self.addEngine(1)
badID = 100
d = self.multiengine.execute(badID, 'a=5')
d.addErrback(
lambda f: self.assertRaises(InvalidEngineID, f.raiseException))
d.addCallback(lambda _: self.multiengine.push(badID, a=5))
d.addErrback(
lambda f: self.assertRaises(InvalidEngineID, f.raiseException))
d.addCallback(lambda _: self.multiengine.pull(badID, 'a'))
d.addErrback(
lambda f: self.assertRaises(InvalidEngineID, f.raiseException))
d.addCallback(lambda _: self.multiengine.getResult(badID))
d.addErrback(
lambda f: self.assertRaises(InvalidEngineID, f.raiseException))
d.addCallback(lambda _: self.multiengine.reset(badID))
d.addErrback(
lambda f: self.assertRaises(InvalidEngineID, f.raiseException))
d.addCallback(lambda _: self.multiengine.keys(badID))
d.addErrback(
lambda f: self.assertRaises(InvalidEngineID, f.raiseException))
d.addCallback(lambda _: self.multiengine.pushSerialized(
badID, a=newserialized.serialize(10)))
d.addErrback(
lambda f: self.assertRaises(InvalidEngineID, f.raiseException))
d.addCallback(lambda _: self.multiengine.pullSerialized(badID, 'a'))
d.addErrback(
lambda f: self.assertRaises(InvalidEngineID, f.raiseException))
d.addCallback(lambda _: self.multiengine.queueStatus(badID))
d.addErrback(
lambda f: self.assertRaises(InvalidEngineID, f.raiseException))
return d
def handleGetResultSuccess(self, result):
try:
serial = newserialized.serialize(result)
except:
self.handleGetResultFailure(Failure())
else:
self.sendSerialized(serial, 'RESULT')
self.getResultOK()
def handleExecuteSuccess(self, result):
try:
serial = newserialized.serialize(result)
except:
self.handleExecuteFailure(Failure())
else:
self.sendSerialized(serial, 'RESULT')
self.executeOK()
def handleExecuteFailure(self, reason):
try:
serial = newserialized.serialize(reason)
except:
self.executeFail()
else:
self.sendSerialized(serial, 'FAILURE')
self.executeOK()
def packThemUp(values):
serials = []
for v in values:
try:
serials.append(newserialized.serialize(v))
except:
return defer.fail(failure.Failure())
return dict(zip(keys, serials))
def handleGetResultFailure(self, reason):
# I am not sure we need to catch this PickleError
try:
serial = newserialized.serialize(reason)
except:
self.getResultFail()
else:
self.sendSerialized(serial, 'FAILURE')
self.getResultOK()
def testPushPullSerialized(self):
objs = [10,"hi there",1.2342354,{"p":(1,2)}]
d = defer.succeed(None)
for o in objs:
self.engine.pushSerialized(key=newserialized.serialize(o))
value = self.engine.pullSerialized('key')
value.addCallback(lambda serial: newserialized.IUnSerialized(serial).getObject())
d = self.assertDeferredEquals(value,o,d)
return d
def testIEngineSerializedDeferreds(self):
dList = []
d = self.engine.pushSerialized(key=newserialized.serialize(12345))
self.assert_(isinstance(d, defer.Deferred))
dList.append(d)
d = self.engine.pullSerialized('key')
self.assert_(isinstance(d, defer.Deferred))
dList.append(d)
D = defer.DeferredList(dList)
return D
def serialize(obj):
serial = newserialized.serialize(obj)
data = serial.getData()
typeDescriptor = serial.getTypeDescriptor()
md = serial.getMetadata()
headers = {'serial-type': typeDescriptor}
if typeDescriptor == 'ndarray':
# pass
headers['dtype'] = md['dtype']
headers['shape'] = str(md['shape'])
headers = prefixHeaders(headers)
# print headers
return headers, data
def isPushReady(self, msg):
if msg == 'READY':
for k, v in self.workVars['namespace'].iteritems():
try:
s = newserialized.serialize(v)
except Exception, e:
log.msg(
'You tried to push an unserializable type, ignoring: '
+ k)
self.pushFail(Failure())
else:
self.sendSerialized(s, k)
self.finishPush()
def testIEngineMultiplexerDeferreds(self):
self.addEngine(1)
d = self.multiengine.execute('a=5', targets=0)
d.addCallback(lambda _: self.multiengine.push(dict(a=5),targets=0))
d.addCallback(lambda _: self.multiengine.push(dict(a=5, b='asdf', c=[1,2,3]),targets=0))
d.addCallback(lambda _: self.multiengine.pull(('a','b','c'),targets=0))
d.addCallback(lambda _: self.multiengine.get_result(targets=0))
d.addCallback(lambda _: self.multiengine.reset(targets=0))
d.addCallback(lambda _: self.multiengine.keys(targets=0))
d.addCallback(lambda _: self.multiengine.push_serialized(dict(a=newserialized.serialize(10)),targets=0))
d.addCallback(lambda _: self.multiengine.pull_serialized('a',targets=0))
d.addCallback(lambda _: self.multiengine.clear_queue(targets=0))
d.addCallback(lambda _: self.multiengine.queue_status(targets=0))
return d
def testPushPullSerialized(self):
self.addEngine(1)
objs = [10,"hi there",1.2342354,{"p":(1,2)}]
d = self.multiengine.push_serialized(dict(key=newserialized.serialize(objs[0])), targets=0)
d.addCallback(lambda _: self.multiengine.pull_serialized('key', targets=0))
d.addCallback(lambda serial: newserialized.IUnSerialized(serial[0]).getObject())
d.addCallback(lambda r: self.assertEquals(r, objs[0]))
d.addCallback(lambda _: self.multiengine.push_serialized(dict(key=newserialized.serialize(objs[1])), targets=0))
d.addCallback(lambda _: self.multiengine.pull_serialized('key', targets=0))
d.addCallback(lambda serial: newserialized.IUnSerialized(serial[0]).getObject())
d.addCallback(lambda r: self.assertEquals(r, objs[1]))
d.addCallback(lambda _: self.multiengine.push_serialized(dict(key=newserialized.serialize(objs[2])), targets=0))
d.addCallback(lambda _: self.multiengine.pull_serialized('key', targets=0))
d.addCallback(lambda serial: newserialized.IUnSerialized(serial[0]).getObject())
d.addCallback(lambda r: self.assertEquals(r, objs[2]))
d.addCallback(lambda _: self.multiengine.push_serialized(dict(key=newserialized.serialize(objs[3])), targets=0))
d.addCallback(lambda _: self.multiengine.pull_serialized('key', targets=0))
d.addCallback(lambda serial: newserialized.IUnSerialized(serial[0]).getObject())
d.addCallback(lambda r: self.assertEquals(r, objs[3]))
d.addCallback(lambda _: self.multiengine.push(dict(a=10,b=range(5)), targets=0))
d.addCallback(lambda _: self.multiengine.pull_serialized(('a','b'), targets=0))
d.addCallback(lambda serial: [newserialized.IUnSerialized(s).getObject() for s in serial[0]])
d.addCallback(lambda r: self.assertEquals(r, [10, range(5)]))
d.addCallback(lambda _: self.multiengine.reset(targets=0))
d.addCallback(lambda _: self.multiengine.pull_serialized('a', targets=0))
d.addErrback(lambda f: self.assertRaises(NameError, _raise_it, f))
return d
objs = [10,"hi there",1.2342354,{"p":(1,2)}]
d = defer.succeed(None)
for o in objs:
self.multiengine.push_serialized(0, key=newserialized.serialize(o))
value = self.multiengine.pull_serialized(0, 'key')
value.addCallback(lambda serial: newserialized.IUnSerialized(serial[0]).getObject())
d = self.assertDeferredEquals(value,o,d)
return d
def testIEngineMultiplexerDeferreds(self):
self.addEngine(1)
l = [
self.multiengine.execute(0, 'a=5'),
self.multiengine.push(0, a=5),
self.multiengine.push(0, a=5, b='asdf', c=[1, 2, 3]),
self.multiengine.pull(0, 'a', 'b', 'c'),
self.multiengine.getResult(0),
self.multiengine.reset(0),
self.multiengine.keys(0),
self.multiengine.pushSerialized(0, a=newserialized.serialize(10)),
self.multiengine.pullSerialized(0, 'a'),
self.multiengine.clearQueue(0),
self.multiengine.queueStatus(0),
]
for d in l:
self.assert_(isinstance(d, defer.Deferred))
return defer.DeferredList(l, consumeErrors=1)
def testInvalidEngineID(self):
self.addEngine(1)
badID = 100
d = self.multiengine.execute('a=5', targets=badID)
d.addErrback(lambda f: self.assertRaises(InvalidEngineID, f.raiseException))
d.addCallback(lambda _: self.multiengine.push(dict(a=5), targets=badID))
d.addErrback(lambda f: self.assertRaises(InvalidEngineID, f.raiseException))
d.addCallback(lambda _: self.multiengine.pull('a', targets=badID))
d.addErrback(lambda f: self.assertRaises(InvalidEngineID, f.raiseException))
d.addCallback(lambda _: self.multiengine.reset(targets=badID))
d.addErrback(lambda f: self.assertRaises(InvalidEngineID, f.raiseException))
d.addCallback(lambda _: self.multiengine.keys(targets=badID))
d.addErrback(lambda f: self.assertRaises(InvalidEngineID, f.raiseException))
d.addCallback(lambda _: self.multiengine.push_serialized(dict(a=newserialized.serialize(10)), targets=badID))
d.addErrback(lambda f: self.assertRaises(InvalidEngineID, f.raiseException))
d.addCallback(lambda _: self.multiengine.pull_serialized('a', targets=badID))
d.addErrback(lambda f: self.assertRaises(InvalidEngineID, f.raiseException))
d.addCallback(lambda _: self.multiengine.queue_status(targets=badID))
d.addErrback(lambda f: self.assertRaises(InvalidEngineID, f.raiseException))
return d
def testNoEnginesRegistered(self):
badID = 'all'
d = self.multiengine.execute('a=5', targets=badID)
d.addErrback(lambda f: self.assertRaises(NoEnginesRegistered, f.raiseException))
d.addCallback(lambda _: self.multiengine.push(dict(a=5), targets=badID))
d.addErrback(lambda f: self.assertRaises(NoEnginesRegistered, f.raiseException))
d.addCallback(lambda _: self.multiengine.pull('a', targets=badID))
d.addErrback(lambda f: self.assertRaises(NoEnginesRegistered, f.raiseException))
d.addCallback(lambda _: self.multiengine.get_result(targets=badID))
d.addErrback(lambda f: self.assertRaises(NoEnginesRegistered, f.raiseException))
d.addCallback(lambda _: self.multiengine.reset(targets=badID))
d.addErrback(lambda f: self.assertRaises(NoEnginesRegistered, f.raiseException))
d.addCallback(lambda _: self.multiengine.keys(targets=badID))
d.addErrback(lambda f: self.assertRaises(NoEnginesRegistered, f.raiseException))
d.addCallback(lambda _: self.multiengine.push_serialized(dict(a=newserialized.serialize(10)), targets=badID))
d.addErrback(lambda f: self.assertRaises(NoEnginesRegistered, f.raiseException))
d.addCallback(lambda _: self.multiengine.pull_serialized('a', targets=badID))
d.addErrback(lambda f: self.assertRaises(NoEnginesRegistered, f.raiseException))
d.addCallback(lambda _: self.multiengine.queue_status(targets=badID))
d.addErrback(lambda f: self.assertRaises(NoEnginesRegistered, f.raiseException))
return d
def testPushPullSerialized(self):
self.addEngine(1)
dikt = dict(a=10,b='hi there',c=1.2345,d={'p':(1,2)})
sdikt = {}
for k,v in dikt.iteritems():
sdikt[k] = newserialized.serialize(v)
d = self.multiengine.push_serialized(dict(a=sdikt['a']), targets=0)
d.addCallback(lambda _: self.multiengine.pull('a',targets=0))
d.addCallback(lambda r: self.assertEquals(r[0], dikt['a']))
d.addCallback(lambda _: self.multiengine.pull_serialized('a', targets=0))
d.addCallback(lambda serial: newserialized.IUnSerialized(serial[0]).getObject())
d.addCallback(lambda r: self.assertEquals(r, dikt['a']))
d.addCallback(lambda _: self.multiengine.push_serialized(sdikt, targets=0))
d.addCallback(lambda _: self.multiengine.pull_serialized(sdikt.keys(), targets=0))
d.addCallback(lambda serial: [newserialized.IUnSerialized(s).getObject() for s in serial[0]])
d.addCallback(lambda r: self.assertEquals(r, dikt.values()))
d.addCallback(lambda _: self.multiengine.reset(targets=0))
d.addCallback(lambda _: self.multiengine.pull_serialized('a', targets=0))
d.addErrback(lambda f: self.assertRaises(NameError, _raise_it, f))
# Non-blocking mode
d.addCallback(lambda r: self.multiengine.push_serialized(dict(a=sdikt['a']), targets=0, block=False))
d.addCallback(lambda did: self.multiengine.get_pending_deferred(did, True))
d.addCallback(lambda _: self.multiengine.pull('a',targets=0))
d.addCallback(lambda r: self.assertEquals(r[0], dikt['a']))
d.addCallback(lambda _: self.multiengine.pull_serialized('a', targets=0, block=False))
d.addCallback(lambda did: self.multiengine.get_pending_deferred(did, True))
d.addCallback(lambda serial: newserialized.IUnSerialized(serial[0]).getObject())
d.addCallback(lambda r: self.assertEquals(r, dikt['a']))
d.addCallback(lambda _: self.multiengine.push_serialized(sdikt, targets=0, block=False))
d.addCallback(lambda did: self.multiengine.get_pending_deferred(did, True))
d.addCallback(lambda _: self.multiengine.pull_serialized(sdikt.keys(), targets=0, block=False))
d.addCallback(lambda did: self.multiengine.get_pending_deferred(did, True))
d.addCallback(lambda serial: [newserialized.IUnSerialized(s).getObject() for s in serial[0]])
d.addCallback(lambda r: self.assertEquals(r, dikt.values()))
d.addCallback(lambda _: self.multiengine.reset(targets=0))
d.addCallback(lambda _: self.multiengine.pull_serialized('a', targets=0, block=False))
d.addCallback(lambda did: self.multiengine.get_pending_deferred(did, True))
d.addErrback(lambda f: self.assertRaises(NameError, _raise_it, f))
return d
def statusOK(self, args):
serial = newserialized.serialize(args)
self.sendSerialized(serial, 'STATUS')
self._reset()
self.sendString('STATUS OK')
