def testNoEnginesRegistered(self): badID = 'all' d = self.multiengine.execute(badID, 'a=5') d.addErrback( lambda f: self.assertRaises(NoEnginesRegistered, f.raiseException)) d.addCallback(lambda _: self.multiengine.push(badID, a=5)) d.addErrback( lambda f: self.assertRaises(NoEnginesRegistered, f.raiseException)) d.addCallback(lambda _: self.multiengine.pull(badID, 'a')) d.addErrback( lambda f: self.assertRaises(NoEnginesRegistered, f.raiseException)) d.addCallback(lambda _: self.multiengine.getResult(badID)) d.addErrback( lambda f: self.assertRaises(NoEnginesRegistered, f.raiseException)) d.addCallback(lambda _: self.multiengine.reset(badID)) d.addErrback( lambda f: self.assertRaises(NoEnginesRegistered, f.raiseException)) d.addCallback(lambda _: self.multiengine.keys(badID)) d.addErrback( lambda f: self.assertRaises(NoEnginesRegistered, f.raiseException)) d.addCallback(lambda _: self.multiengine.pushSerialized( badID, a=newserialized.serialize(10))) d.addErrback( lambda f: self.assertRaises(NoEnginesRegistered, f.raiseException)) d.addCallback(lambda _: self.multiengine.pullSerialized(badID, 'a')) d.addErrback( lambda f: self.assertRaises(NoEnginesRegistered, f.raiseException)) d.addCallback(lambda _: self.multiengine.queueStatus(badID)) d.addErrback( lambda f: self.assertRaises(NoEnginesRegistered, f.raiseException)) return d
def pushPaging(self, key, obj): try: serial = newserialized.serialize(obj) except: return defer.fail(failure.Failure()) else: return self.pushPagingSerialized(key, serial)
def testInvalidEngineID(self): self.addEngine(1) badID = 100 d = self.multiengine.execute(badID, 'a=5') d.addErrback( lambda f: self.assertRaises(InvalidEngineID, f.raiseException)) d.addCallback(lambda _: self.multiengine.push(badID, a=5)) d.addErrback( lambda f: self.assertRaises(InvalidEngineID, f.raiseException)) d.addCallback(lambda _: self.multiengine.pull(badID, 'a')) d.addErrback( lambda f: self.assertRaises(InvalidEngineID, f.raiseException)) d.addCallback(lambda _: self.multiengine.getResult(badID)) d.addErrback( lambda f: self.assertRaises(InvalidEngineID, f.raiseException)) d.addCallback(lambda _: self.multiengine.reset(badID)) d.addErrback( lambda f: self.assertRaises(InvalidEngineID, f.raiseException)) d.addCallback(lambda _: self.multiengine.keys(badID)) d.addErrback( lambda f: self.assertRaises(InvalidEngineID, f.raiseException)) d.addCallback(lambda _: self.multiengine.pushSerialized( badID, a=newserialized.serialize(10))) d.addErrback( lambda f: self.assertRaises(InvalidEngineID, f.raiseException)) d.addCallback(lambda _: self.multiengine.pullSerialized(badID, 'a')) d.addErrback( lambda f: self.assertRaises(InvalidEngineID, f.raiseException)) d.addCallback(lambda _: self.multiengine.queueStatus(badID)) d.addErrback( lambda f: self.assertRaises(InvalidEngineID, f.raiseException)) return d
def handleGetResultSuccess(self, result): try: serial = newserialized.serialize(result) except: self.handleGetResultFailure(Failure()) else: self.sendSerialized(serial, 'RESULT') self.getResultOK()
def handleExecuteSuccess(self, result): try: serial = newserialized.serialize(result) except: self.handleExecuteFailure(Failure()) else: self.sendSerialized(serial, 'RESULT') self.executeOK()
def handleExecuteFailure(self, reason): try: serial = newserialized.serialize(reason) except: self.executeFail() else: self.sendSerialized(serial, 'FAILURE') self.executeOK()
def packThemUp(values): serials = [] for v in values: try: serials.append(newserialized.serialize(v)) except: return defer.fail(failure.Failure()) return dict(zip(keys, serials))
def handleGetResultFailure(self, reason): # I am not sure we need to catch this PickleError try: serial = newserialized.serialize(reason) except: self.getResultFail() else: self.sendSerialized(serial, 'FAILURE') self.getResultOK()
def testPushPullSerialized(self): objs = [10,"hi there",1.2342354,{"p":(1,2)}] d = defer.succeed(None) for o in objs: self.engine.pushSerialized(key=newserialized.serialize(o)) value = self.engine.pullSerialized('key') value.addCallback(lambda serial: newserialized.IUnSerialized(serial).getObject()) d = self.assertDeferredEquals(value,o,d) return d
def testIEngineSerializedDeferreds(self): dList = [] d = self.engine.pushSerialized(key=newserialized.serialize(12345)) self.assert_(isinstance(d, defer.Deferred)) dList.append(d) d = self.engine.pullSerialized('key') self.assert_(isinstance(d, defer.Deferred)) dList.append(d) D = defer.DeferredList(dList) return D
def serialize(obj): serial = newserialized.serialize(obj) data = serial.getData() typeDescriptor = serial.getTypeDescriptor() md = serial.getMetadata() headers = {'serial-type': typeDescriptor} if typeDescriptor == 'ndarray': # pass headers['dtype'] = md['dtype'] headers['shape'] = str(md['shape']) headers = prefixHeaders(headers) # print headers return headers, data
def isPushReady(self, msg): if msg == 'READY': for k, v in self.workVars['namespace'].iteritems(): try: s = newserialized.serialize(v) except Exception, e: log.msg( 'You tried to push an unserializable type, ignoring: ' + k) self.pushFail(Failure()) else: self.sendSerialized(s, k) self.finishPush()
def testIEngineMultiplexerDeferreds(self): self.addEngine(1) d = self.multiengine.execute('a=5', targets=0) d.addCallback(lambda _: self.multiengine.push(dict(a=5),targets=0)) d.addCallback(lambda _: self.multiengine.push(dict(a=5, b='asdf', c=[1,2,3]),targets=0)) d.addCallback(lambda _: self.multiengine.pull(('a','b','c'),targets=0)) d.addCallback(lambda _: self.multiengine.get_result(targets=0)) d.addCallback(lambda _: self.multiengine.reset(targets=0)) d.addCallback(lambda _: self.multiengine.keys(targets=0)) d.addCallback(lambda _: self.multiengine.push_serialized(dict(a=newserialized.serialize(10)),targets=0)) d.addCallback(lambda _: self.multiengine.pull_serialized('a',targets=0)) d.addCallback(lambda _: self.multiengine.clear_queue(targets=0)) d.addCallback(lambda _: self.multiengine.queue_status(targets=0)) return d
def testPushPullSerialized(self): self.addEngine(1) objs = [10,"hi there",1.2342354,{"p":(1,2)}] d = self.multiengine.push_serialized(dict(key=newserialized.serialize(objs[0])), targets=0) d.addCallback(lambda _: self.multiengine.pull_serialized('key', targets=0)) d.addCallback(lambda serial: newserialized.IUnSerialized(serial[0]).getObject()) d.addCallback(lambda r: self.assertEquals(r, objs[0])) d.addCallback(lambda _: self.multiengine.push_serialized(dict(key=newserialized.serialize(objs[1])), targets=0)) d.addCallback(lambda _: self.multiengine.pull_serialized('key', targets=0)) d.addCallback(lambda serial: newserialized.IUnSerialized(serial[0]).getObject()) d.addCallback(lambda r: self.assertEquals(r, objs[1])) d.addCallback(lambda _: self.multiengine.push_serialized(dict(key=newserialized.serialize(objs[2])), targets=0)) d.addCallback(lambda _: self.multiengine.pull_serialized('key', targets=0)) d.addCallback(lambda serial: newserialized.IUnSerialized(serial[0]).getObject()) d.addCallback(lambda r: self.assertEquals(r, objs[2])) d.addCallback(lambda _: self.multiengine.push_serialized(dict(key=newserialized.serialize(objs[3])), targets=0)) d.addCallback(lambda _: self.multiengine.pull_serialized('key', targets=0)) d.addCallback(lambda serial: newserialized.IUnSerialized(serial[0]).getObject()) d.addCallback(lambda r: self.assertEquals(r, objs[3])) d.addCallback(lambda _: self.multiengine.push(dict(a=10,b=range(5)), targets=0)) d.addCallback(lambda _: self.multiengine.pull_serialized(('a','b'), targets=0)) d.addCallback(lambda serial: [newserialized.IUnSerialized(s).getObject() for s in serial[0]]) d.addCallback(lambda r: self.assertEquals(r, [10, range(5)])) d.addCallback(lambda _: self.multiengine.reset(targets=0)) d.addCallback(lambda _: self.multiengine.pull_serialized('a', targets=0)) d.addErrback(lambda f: self.assertRaises(NameError, _raise_it, f)) return d objs = [10,"hi there",1.2342354,{"p":(1,2)}] d = defer.succeed(None) for o in objs: self.multiengine.push_serialized(0, key=newserialized.serialize(o)) value = self.multiengine.pull_serialized(0, 'key') value.addCallback(lambda serial: newserialized.IUnSerialized(serial[0]).getObject()) d = self.assertDeferredEquals(value,o,d) return d
def testIEngineMultiplexerDeferreds(self): self.addEngine(1) l = [ self.multiengine.execute(0, 'a=5'), self.multiengine.push(0, a=5), self.multiengine.push(0, a=5, b='asdf', c=[1, 2, 3]), self.multiengine.pull(0, 'a', 'b', 'c'), self.multiengine.getResult(0), self.multiengine.reset(0), self.multiengine.keys(0), self.multiengine.pushSerialized(0, a=newserialized.serialize(10)), self.multiengine.pullSerialized(0, 'a'), self.multiengine.clearQueue(0), self.multiengine.queueStatus(0), ] for d in l: self.assert_(isinstance(d, defer.Deferred)) return defer.DeferredList(l, consumeErrors=1)
def testInvalidEngineID(self): self.addEngine(1) badID = 100 d = self.multiengine.execute('a=5', targets=badID) d.addErrback(lambda f: self.assertRaises(InvalidEngineID, f.raiseException)) d.addCallback(lambda _: self.multiengine.push(dict(a=5), targets=badID)) d.addErrback(lambda f: self.assertRaises(InvalidEngineID, f.raiseException)) d.addCallback(lambda _: self.multiengine.pull('a', targets=badID)) d.addErrback(lambda f: self.assertRaises(InvalidEngineID, f.raiseException)) d.addCallback(lambda _: self.multiengine.reset(targets=badID)) d.addErrback(lambda f: self.assertRaises(InvalidEngineID, f.raiseException)) d.addCallback(lambda _: self.multiengine.keys(targets=badID)) d.addErrback(lambda f: self.assertRaises(InvalidEngineID, f.raiseException)) d.addCallback(lambda _: self.multiengine.push_serialized(dict(a=newserialized.serialize(10)), targets=badID)) d.addErrback(lambda f: self.assertRaises(InvalidEngineID, f.raiseException)) d.addCallback(lambda _: self.multiengine.pull_serialized('a', targets=badID)) d.addErrback(lambda f: self.assertRaises(InvalidEngineID, f.raiseException)) d.addCallback(lambda _: self.multiengine.queue_status(targets=badID)) d.addErrback(lambda f: self.assertRaises(InvalidEngineID, f.raiseException)) return d
def testNoEnginesRegistered(self): badID = 'all' d = self.multiengine.execute('a=5', targets=badID) d.addErrback(lambda f: self.assertRaises(NoEnginesRegistered, f.raiseException)) d.addCallback(lambda _: self.multiengine.push(dict(a=5), targets=badID)) d.addErrback(lambda f: self.assertRaises(NoEnginesRegistered, f.raiseException)) d.addCallback(lambda _: self.multiengine.pull('a', targets=badID)) d.addErrback(lambda f: self.assertRaises(NoEnginesRegistered, f.raiseException)) d.addCallback(lambda _: self.multiengine.get_result(targets=badID)) d.addErrback(lambda f: self.assertRaises(NoEnginesRegistered, f.raiseException)) d.addCallback(lambda _: self.multiengine.reset(targets=badID)) d.addErrback(lambda f: self.assertRaises(NoEnginesRegistered, f.raiseException)) d.addCallback(lambda _: self.multiengine.keys(targets=badID)) d.addErrback(lambda f: self.assertRaises(NoEnginesRegistered, f.raiseException)) d.addCallback(lambda _: self.multiengine.push_serialized(dict(a=newserialized.serialize(10)), targets=badID)) d.addErrback(lambda f: self.assertRaises(NoEnginesRegistered, f.raiseException)) d.addCallback(lambda _: self.multiengine.pull_serialized('a', targets=badID)) d.addErrback(lambda f: self.assertRaises(NoEnginesRegistered, f.raiseException)) d.addCallback(lambda _: self.multiengine.queue_status(targets=badID)) d.addErrback(lambda f: self.assertRaises(NoEnginesRegistered, f.raiseException)) return d
def testPushPullSerialized(self): self.addEngine(1) dikt = dict(a=10,b='hi there',c=1.2345,d={'p':(1,2)}) sdikt = {} for k,v in dikt.iteritems(): sdikt[k] = newserialized.serialize(v) d = self.multiengine.push_serialized(dict(a=sdikt['a']), targets=0) d.addCallback(lambda _: self.multiengine.pull('a',targets=0)) d.addCallback(lambda r: self.assertEquals(r[0], dikt['a'])) d.addCallback(lambda _: self.multiengine.pull_serialized('a', targets=0)) d.addCallback(lambda serial: newserialized.IUnSerialized(serial[0]).getObject()) d.addCallback(lambda r: self.assertEquals(r, dikt['a'])) d.addCallback(lambda _: self.multiengine.push_serialized(sdikt, targets=0)) d.addCallback(lambda _: self.multiengine.pull_serialized(sdikt.keys(), targets=0)) d.addCallback(lambda serial: [newserialized.IUnSerialized(s).getObject() for s in serial[0]]) d.addCallback(lambda r: self.assertEquals(r, dikt.values())) d.addCallback(lambda _: self.multiengine.reset(targets=0)) d.addCallback(lambda _: self.multiengine.pull_serialized('a', targets=0)) d.addErrback(lambda f: self.assertRaises(NameError, _raise_it, f)) # Non-blocking mode d.addCallback(lambda r: self.multiengine.push_serialized(dict(a=sdikt['a']), targets=0, block=False)) d.addCallback(lambda did: self.multiengine.get_pending_deferred(did, True)) d.addCallback(lambda _: self.multiengine.pull('a',targets=0)) d.addCallback(lambda r: self.assertEquals(r[0], dikt['a'])) d.addCallback(lambda _: self.multiengine.pull_serialized('a', targets=0, block=False)) d.addCallback(lambda did: self.multiengine.get_pending_deferred(did, True)) d.addCallback(lambda serial: newserialized.IUnSerialized(serial[0]).getObject()) d.addCallback(lambda r: self.assertEquals(r, dikt['a'])) d.addCallback(lambda _: self.multiengine.push_serialized(sdikt, targets=0, block=False)) d.addCallback(lambda did: self.multiengine.get_pending_deferred(did, True)) d.addCallback(lambda _: self.multiengine.pull_serialized(sdikt.keys(), targets=0, block=False)) d.addCallback(lambda did: self.multiengine.get_pending_deferred(did, True)) d.addCallback(lambda serial: [newserialized.IUnSerialized(s).getObject() for s in serial[0]]) d.addCallback(lambda r: self.assertEquals(r, dikt.values())) d.addCallback(lambda _: self.multiengine.reset(targets=0)) d.addCallback(lambda _: self.multiengine.pull_serialized('a', targets=0, block=False)) d.addCallback(lambda did: self.multiengine.get_pending_deferred(did, True)) d.addErrback(lambda f: self.assertRaises(NameError, _raise_it, f)) return d
def statusOK(self, args): serial = newserialized.serialize(args) self.sendSerialized(serial, 'STATUS') self._reset() self.sendString('STATUS OK')