def testErrorIntervalAndLoggedMessage(self): def raiser(): raise Exception('exception') yield self.observer.methods['handle'] = raiser addTimer, = self.reactor.calledMethods self.reactor.calledMethods.reset() addTimer.args[1]() addProcess, = self.reactor.calledMethods self.reactor.calledMethods.reset() with stderr_replaced() as err: addProcess.args[0]() errValue = err.getvalue() self.assertTrue(errValue.startswith(repr(self.pc))) self.assertTrue('Traceback' in errValue, errValue) self.assertTrue('Exception: exception' in errValue, errValue) self.assertEquals('exception', self.pc.getState().errorState) self.assertEquals(['handle'], self.observer.calledMethodNames()) self.assertEquals(['removeProcess', 'addTimer'], self.reactor.calledMethodNames()) _, addTimer = self.reactor.calledMethods self.assertEquals(((15, self.pc._periodicCall), {}), (addTimer.args, addTimer.kwargs))
def test(): sp = SocketPool(reactor=CallTrace(), limits={'totalSize': 2}) # Limits enforced on put, not async. def stillPooled(): wasStillPooled = [] for destHost, destPort in [('h', 1), ('i', 2), ('j', 3)]: while True: # do ... while (fromPool is not None) fromPool = yield sp.getPooledSocket(host=destHost, port=destPort) if fromPool: wasStillPooled.append(fromPool) if fromPool is None: break raise StopIteration(wasStillPooled) yield sp.putSocketInPool(host='h', port=1, sock=MockSok('sH')) yield sp.putSocketInPool(host='i', port=2, sock=MockSok('sI')) with stderr_replaced() as err: yield sp.putSocketInPool(host='j', port=3, sock=MockSok('sJ')) self.assertEquals('', err.getvalue(), err.getvalue()) wasStillPooled = yield stillPooled() self.assertEquals(2, len(wasStillPooled)) self.assertTrue( set(wasStillPooled).issubset(set(['sH', 'sI', 'sJ'])))
def test(): sp = SocketPool(reactor=CallTrace(), limits={'destinationSize': 2}) # Limits enforced on put, not async. def stillPooled(): wasStillPooled = [] for destHost, destPort in [('h', 1), ('i', 2), ('j', 3)]: while True: # do ... while (fromPool is not None) fromPool = yield sp.getPooledSocket(host=destHost, port=destPort) if fromPool: wasStillPooled.append(fromPool) if fromPool is None: break raise StopIteration(wasStillPooled) sJ = MockSok('sJ') sJ2 = MockSok('sJ2') yield sp.putSocketInPool(host='h', port=1, sock=MockSok('sH')) yield sp.putSocketInPool(host='i', port=2, sock=MockSok('sI')) yield sp.putSocketInPool(host='j', port=3, sock=sJ) yield sp.putSocketInPool(host='j', port=3, sock=sJ2) with stderr_replaced() as err: yield sp.putSocketInPool(host='j', port=3, sock=MockSok('sJ3')) self.assertEquals('', err.getvalue(), err.getvalue()) wasStillPooled = yield stillPooled() self.assertEquals(4, len(wasStillPooled)) self.assertEquals(['sH', 'sI', 'sJ3', 'sJ2'], wasStillPooled) self.assertEquals(['shutdown', 'close'], sJ.log.calledMethodNames())
def test(): sp = SocketPool(reactor=CallTrace(), limits={'totalSize': 2}) # Limits enforced on put, not async. def stillPooled(): wasStillPooled = [] while True: # do ... while (fromPool is not None) fromPool = yield sp.getPooledSocket(host='h', port=1) if fromPool: wasStillPooled.append(fromPool) if fromPool is None: break raise StopIteration(wasStillPooled) s0 = MockSok('s0') s1 = MockSok('s1') yield sp.putSocketInPool(host='h', port=1, sock=s0) yield sp.putSocketInPool(host='h', port=1, sock=s1) with stderr_replaced() as err: yield sp.putSocketInPool(host='h', port=1, sock=MockSok('s2')) self.assertEquals('', err.getvalue(), err.getvalue()) #@@ wasStillPooled = yield stillPooled() self.assertEquals(['s2', 's0'], wasStillPooled) self.assertEquals(['shutdown', 'close'], s1.log.calledMethodNames()) shutCall, closeCall = s1.log.calledMethods self.assertEquals(((SHUT_RDWR,), {}), (shutCall.args, shutCall.kwargs)) self.assertEquals(((), {}), (closeCall.args, closeCall.kwargs)) self.assertEquals([], s0.log.calledMethodNames()) yield sp.putSocketInPool(host='h', port=1, sock=MockSok('s0')) yield sp.putSocketInPool(host='h', port=1, sock=MockSok('s1')) yield sp.putSocketInPool(host='h', port=1, sock=MockSok('s2')) yield sp.putSocketInPool(host='h', port=1, sock=MockSok('s3')) wasStillPooled = yield stillPooled() self.assertEquals(['s3', 's0'], wasStillPooled)
def setUpClass(cls): with stderr_replaced() as err: TagFactory().compose(lambda: None) err_val = err.getvalue() assert 'FutureWarning' in err_val, 'Missing expected warning.' SeecrTestCase.setUpClass()
def testSuspendTimeoutOnTimeoutCallbackGivesException(self): def prepare(_onTimeout): # Use indirect tracing of onTimeout call (otherwise CallTrace internals leak into the stacktrace). trace = CallTrace(returnValues={'addTimer': 'timerToken'}) onTimeoutM = trace.onTimeout def onTimeoutTraced(): onTimeoutM() # We have been called Watson! return _onTimeout() trace.onTimeout = onTimeoutTraced suspend = Suspend(doNext=trace.doNext, timeout=3.14, onTimeout=trace.onTimeout) suspend(reactor=trace, whenDone=trace.whenDone) self.assertEquals(['doNext', 'addTimer', 'suspend'], trace.calledMethodNames()) trace.calledMethods.reset() return trace, suspend # basic invariant trace, suspend = prepare(lambda: None) suspend._timedOut() self.assertEquals(['onTimeout', 'whenDone'], trace.calledMethodNames()) self.assertRaises(TimeoutException, lambda: suspend.getResult()) # normal exceptions def onTimeout(): raise Exception("This Should Never Happen But Don't Expose Exception If It Does Anyway !") trace, suspend = prepare(onTimeout) with stderr_replaced() as err: suspend._timedOut() expectedTraceback = ignoreLineNumbers('''Unexpected exception raised on Suspend's onTimeout callback (ignored): Traceback (most recent call last): File "%(suspend.py)s", line 101, in _timedOut self._onTimeout() File "%(__file__)s", line 382, in onTimeoutTraced return _onTimeout() File "%(__file__)s", line 398, in onTimeout raise Exception("This Should Never Happen But Don't Expose Exception If It Does Anyway !") Exception: This Should Never Happen But Don't Expose Exception If It Does Anyway ! ''' % fileDict) self.assertEqualsWS(expectedTraceback, ignoreLineNumbers(err.getvalue())) self.assertEquals(['onTimeout', 'whenDone'], trace.calledMethodNames()) self.assertRaises(TimeoutException, lambda: suspend.getResult()) # fatal exceptions def suspendCallWithOnTimeoutRaising(exception): def onTimeout(): raise exception trace, suspend = prepare(onTimeout) try: suspend._timedOut() except: c, v, t = exc_info() self.assertEquals(['onTimeout'], trace.calledMethodNames()) raise c, v, t.tb_next self.assertRaises(KeyboardInterrupt, lambda: suspendCallWithOnTimeoutRaising(KeyboardInterrupt())) self.assertRaises(SystemExit, lambda: suspendCallWithOnTimeoutRaising(SystemExit())) self.assertRaises(AssertionError, lambda: suspendCallWithOnTimeoutRaising(AssertionError()))
def testPauseRemovesInitialPendingTimer(self): self.assertEquals(['addTimer'], self.reactor.calledMethodNames()) self.reactor.calledMethods.reset() with stderr_replaced() as err: self.pc.pause() self.assertEquals('%s: paused\n' % repr(self.pc), err.getvalue()) self.assertEquals(['removeTimer'], self.reactor.calledMethodNames()) removeTimer, = self.reactor.calledMethods self.assertEquals((('TOKEN',), {}), (removeTimer.args, removeTimer.kwargs))
def testError(self): parser = ParseArguments() parser.addOption('', '--option') with stderr_replaced() as err: try: parser.error('error msg.') self.fail() except SystemExit, e: self.assertEquals(2, e.code) lines = err.getvalue().split('\n')
def test(suspendMethod): with stderr_replaced() as s: s1 = compose(suspendMethod(clientIdentifier="a-client-id", prefix='prefix', sets=[], continueAfter='9876')).next() s1(CallTrace('reactor'), lambda: None) compose(suspendMethod(clientIdentifier="another-client-id", prefix='prefix', sets=[], continueAfter='9876')).next() try: s1.getResult() self.fail() except ForcedResumeException: self.assertEquals("Too many suspended connections in SuspendRegister. One random connection has been resumed.\n", s.getvalue())
def setUp(self): SeecrTestCase.setUp(self) self.observer = CallTrace() with stderr_replaced() as err: self.dna = be( (Observable(), (OaiSetSelect(['set1', 'set2']), (self.observer, )))) self.assertTrue(not err.getvalue() or \ 'warn("OaiSetSelect is deprecated;' in err.getvalue(), err.getvalue())
def setUp(self): SeecrTestCase.setUp(self) self.observer = CallTrace() with stderr_replaced() as err: self.dna = be( (Observable(), (OaiSetSelect(['set1', 'set2']), (self.observer,) ) ) ) self.assertTrue(not err.getvalue() or \ 'warn("OaiSetSelect is deprecated;' in err.getvalue(), err.getvalue())
def testWriteLogMustNotFail(self): logwriter = CallTrace('logwriter', emptyGeneratorMethods=['someMessage']) logwriter.exceptions['writeLog'] = ValueError top = be((Observable(), (LogCollector('default'), (logwriter, ), (FilterMessages(allowed=['someMessage']), (LogKeyValue(dict(name='A')), ))))) with stderr_replaced() as err: try: consume(top.all.someMessage()) except ValueError: self.fail("Should not raise an error; Only print it") self.assertTrue('ValueError' in err.getvalue(), err.getvalue())
def testDiagnosticOnExecuteCql(self): with stderr_replaced(): class RaisesException(object): def executeQuery(self, *args, **kwargs): raise Exception("Test Exception") component = SruHandler() component.addObserver(RaisesException()) arguments = dict(startRecord=11, maximumRecords=15, query='query', recordPacking='string', recordSchema='schema') result = parse(StringIO("".join(compose(component.searchRetrieve(sruArguments=arguments, **arguments))))) diagnostic = result.xpath("/srw:searchRetrieveResponse/srw:diagnostics/diag:diagnostic", namespaces=namespaces) self.assertEquals(1, len(diagnostic)) self.assertEquals(["info://srw/diagnostics/1/48"], diagnostic[0].xpath("diag:uri/text()", namespaces=namespaces)) self.assertEquals(["Query Feature Unsupported"], diagnostic[0].xpath("diag:message/text()", namespaces=namespaces)) self.assertEquals(["Test Exception"], diagnostic[0].xpath("diag:details/text()", namespaces=namespaces))
def testShouldRaiseExceptionOnSameRequestTwice(self): self.run = True portNumber = randint(50000, 60000) oaiJazz = OaiJazz(join(self.tempdir, 'oai')) suspendRegister = SuspendRegister() oaiJazz.addObserver(suspendRegister) storageComponent = MultiSequentialStorage(join(self.tempdir, 'storage')) clientId = str(uuid4()) requests = [] def doOaiListRecord(port): header, body = getRequest(port=portNumber, path="/", arguments={"verb": "ListRecords", "metadataPrefix": "prefix", "x-wait": "True"}, additionalHeaders={'X-Meresco-Oai-Client-Identifier': clientId}, parse=False) requests.append((header, body)) oaiPmhThread = Thread(None, lambda: self.startOaiPmh(portNumber, oaiJazz, storageComponent, suspendRegister)) harvestThread1 = Thread(None, lambda: doOaiListRecord(portNumber)) harvestThread2 = Thread(None, lambda: doOaiListRecord(portNumber)) with stderr_replaced(): oaiPmhThread.start() harvestThread1.start() try: while len(suspendRegister) == 0: sleep(0.01) harvest1Suspend = suspendRegister._suspendObject(clientId) self.assertTrue(harvest1Suspend is not None) harvestThread2.start() while harvest1Suspend == suspendRegister._suspendObject(clientId): sleep(0.01) sleep(0.01) self.assertTrue(clientId in suspendRegister) self.assertTrue(harvest1Suspend != suspendRegister._suspendObject(clientId)) self.assertEquals(1, len(requests)) header, body = requests[0] self.assertTrue('500' in header, header) self.assertTrue(body.startswith('Aborting suspended request'), body) storageComponent.addData(identifier="id1", name="prefix", data="<a>a1</a>") oaiJazz.addOaiRecord(identifier="id1", sets=[], metadataFormats=[("prefix", "", "")]) sleep(0.1) finally: self.run = False oaiPmhThread.join() harvestThread1.join() harvestThread2.join() oaiJazz.close()
def testError(self): parser = ParseArguments() parser.addOption('', '--option') with stderr_replaced() as err: try: parser.error('error msg.') self.fail() except SystemExit as e: self.assertEqual(2, e.code) lines = err.getvalue().split('\n') self.assertEqual(4, len(lines)) self.assertTrue(lines[0].startswith('Usage: ')) self.assertEqual('', lines[1]) self.assertTrue(lines[2].endswith(': error: error msg.')) self.assertEqual('', lines[3])
def testListRecordsWithoutClientIdentifierGeneratesOne(self): self.oaiList = OaiList(batchSize=2, supportXWait=True, repository=OaiRepository()) self.oaiList.addObserver(self.observer) self.httpkwargs = { 'path': '/path/to/oai', 'Headers':{'Host':'server'}, 'port':9000, 'Client': ('127.0.0.1', 1234) } with stderr_replaced() as s: result = compose(self.oaiList.listRecords(arguments={'verb':['ListRecords'], 'metadataPrefix': ['oai_dc'], 'x-wait': ['True']}, **self.httpkwargs)) result.next() self.assertEquals(['suspendBeforeSelect', 'getAllPrefixes', 'suspendAfterNoResult'], [m.name for m in self.observer.calledMethods]) self.assertTrue('clientIdentifier' in self.observer.calledMethods[-1].kwargs) self.assertEquals(len(str(uuid4())), len(self.observer.calledMethods[-1].kwargs['clientIdentifier'])) self.assertEquals("X-Meresco-Oai-Client-Identifier not found in HTTP Headers. Generated a uuid for OAI client from 127.0.0.1\n", s.getvalue())
def testShouldNotStartToLoopLikeAMadMan(self): self.run = True portNumber = randint(50000, 60000) oaiJazz = OaiJazz(join(self.tempdir, 'oai')) suspendRegister = SuspendRegister(maximumSuspendedConnections=5) oaiJazz.addObserver(suspendRegister) storageComponent = MultiSequentialStorage(join(self.tempdir, 'storage')) def doUrlOpenWithTimeout(port, basket): try: response = urlopen( "http://localhost:%s/?verb=ListRecords&metadataPrefix=prefix&x-wait=True" % port, timeout=0.5) basket.append(response.getcode()) except timeout as e: self.assertTrue('timed out' in str(e), str(e)) oaiPmhThread = Thread( None, lambda: self.startOaiPmh(portNumber, oaiJazz, storageComponent, suspendRegister)) threads = [] todo = [doUrlOpenWithTimeout] * 7 statusCodes = [] oaiPmhThread.start() with stderr_replaced(): while todo: func = todo.pop() harvestThread = Thread(None, lambda: func(portNumber, statusCodes)) threads.append(harvestThread) harvestThread.start() try: while len(suspendRegister) == 0: sleep(0.01) finally: for t in threads: t.join() self.run = False oaiPmhThread.join() oaiJazz.close() self.assertEqual([204] * 2, statusCodes)
def testTracebackPreservedAcrossSuspend(self): backofficeport = PortNumberGenerator.next() expectedrequest = '' testserver(backofficeport, [], expectedrequest) target = ('localhost', backofficeport, '/') exceptions = [] def failingserver(*args, **kwarg): response = yield httpget(*target) self.handler = failingserver def requestLine(self, *args, **kwargs): raise RuntimeError("Boom!") try: originalRequestLine = httpRequestModule._requestLine httpRequestModule._requestLine = requestLine clientget('localhost', self.port, '/') with stderr_replaced(): self._loopReactorUntilDone() expectedTraceback = ignoreLineNumbers( """Traceback (most recent call last): File "%(__file__)s", line 0, in handle yield self.handler(*args, **kwargs) File "%(__file__)s", line 192, in failingserver response = yield httpget(*target) File "%(httprequest.py)s", line 129, in httprequest result = s.getResult() File "%(httprequest.py)s", line 83, in _do yield _sendHttpHeaders(sok, method, request, headers) File "%(httprequest.py)s", line 121, in _sendHttpHeaders data = _requestLine(method, request) File "%(__file__)s", line 198, in requestLine raise RuntimeError("Boom!") RuntimeError: Boom!""" % fileDict) resultingTraceback = ''.join(format_exception(*self.error)) self.assertEqualsWS(expectedTraceback, ignoreLineNumbers(resultingTraceback)) finally: httpRequestModule._requestLine = originalRequestLine
def testWriteLogMustNotFail(self): logwriter = CallTrace('logwriter', emptyGeneratorMethods=['someMessage']) logwriter.exceptions['writeLog'] = ValueError top = be((Observable(), (LogCollector('default'), (logwriter,), (FilterMessages(allowed=['someMessage']), (LogKeyValue(dict(name='A')),) ) ) )) with stderr_replaced() as err: try: consume(top.all.someMessage()) except ValueError: self.fail("Should not raise an error; Only print it") self.assertTrue('ValueError' in err.getvalue(), err.getvalue())
def testPossibleShutdownAtWrongTime(self): # We suspect a bad shutdown could have cause a difference between keyvaluestore and the data. uri1 = "uri:someuri1" rdfFillTitle = """<rdf:RDF xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"><rdf:Description rdf:about="%s" xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"> <dc:title xmlns:dc="http://purl.org/dc/elements/1.1/" xml:lang="en">%%s</dc:title> </rdf:Description></rdf:RDF>""" % uri1 consume(self.dna.all.add(identifier="identifier", partname="ignored", lxmlNode=parse(StringIO(rdfFillTitle % 'title')))) record1 = self.storage.getData(identifier=uri1, name='rdf') self.assertEquals('title', xpathFirst(XML(record1), '/rdf:RDF/rdf:Description/dc:title/text()')) # HACK the data in storage, which could have happened if shutdown while adding. self.storage.addData(identifier=uri1, name='rdf', data=rdfFillTitle % 'other title') # Service is shutdown after adding the uri to the storage, but just before registring the fragmentHashes in the key value store # The next call caused a KeyError while removing old fragmentHashes. with stderr_replaced(): consume(self.dna.all.add(identifier="identifier", partname="ignored", lxmlNode=parse(StringIO(rdfFillTitle % 'other title')))) record1 = self.storage.getData(identifier=uri1, name='rdf') self.assertEquals('other title', xpathFirst(XML(record1), '/rdf:RDF/rdf:Description/dc:title/text()'))
def testStdErrReplaced(self): idStderr = id(sys.stderr) with stderr_replaced() as s: self.assertNotEqual(idStderr, id(sys.stderr)) try: raise Exception('xcptn') except Exception: print_exc() result = s.getvalue() self.assertTrue('Traceback' in result, result) self.assertTrue('Exception: xcptn' in result, result) self.assertEquals(idStderr, id(sys.stderr)) @stderr_replaced def f(): self.assertNotEqual(idStderr, id(sys.stderr)) raise Exception() self.assertEquals(idStderr, id(sys.stderr))
def testPauseRemovesEndOfProcessPendingTimer(self): self.assertEquals(['addTimer'], self.reactor.calledMethodNames()) addTimer, = self.reactor.calledMethods self.reactor.calledMethods.reset() # Process until timer added again addTimer.args[1]() addProcess, = self.reactor.calledMethods self.reactor.calledMethods.reset() addProcess.args[0]() self.assertEquals(['removeProcess', 'addTimer'], self.reactor.calledMethodNames()) self.reactor.calledMethods.reset() # pause at pending timer with stderr_replaced() as err: self.pc.pause() self.assertEquals('%s: paused\n' % repr(self.pc), err.getvalue()) self.assertEquals(['removeTimer'], self.reactor.calledMethodNames()) removeTimer, = self.reactor.calledMethods self.assertEquals((('TOKEN',), {}), (removeTimer.args, removeTimer.kwargs))
def testTracebackPreservedAcrossSuspend(self): backofficeport = PortNumberGenerator.next() expectedrequest = '' testserver(backofficeport, [], expectedrequest) target = ('localhost', backofficeport, '/') exceptions = [] def failingserver(*args, **kwarg): response = yield httpget(*target) self.handler = failingserver def requestLine(self, *args, **kwargs): raise RuntimeError("Boom!") try: originalRequestLine = httpRequestModule._requestLine httpRequestModule._requestLine = requestLine clientget('localhost', self.port, '/') with stderr_replaced(): self._loopReactorUntilDone() expectedTraceback = ignoreLineNumbers("""Traceback (most recent call last): File "%(__file__)s", line 0, in handle yield self.handler(*args, **kwargs) File "%(__file__)s", line 192, in failingserver response = yield httpget(*target) File "%(httprequest.py)s", line 129, in httprequest result = s.getResult() File "%(httprequest.py)s", line 83, in _do yield _sendHttpHeaders(sok, method, request, headers) File "%(httprequest.py)s", line 121, in _sendHttpHeaders data = _requestLine(method, request) File "%(__file__)s", line 198, in requestLine raise RuntimeError("Boom!") RuntimeError: Boom!""" % fileDict) resultingTraceback = ''.join(format_exception(*self.error)) self.assertEqualsWS(expectedTraceback, ignoreLineNumbers(resultingTraceback)) finally: httpRequestModule._requestLine = originalRequestLine
def testErrorStateReset(self): handleCalledBefore = [] def handle(): if not handleCalledBefore: handleCalledBefore.append(True) raise Exception('exception') yield Yield self.observer.methods['handle'] = handle addTimer, = self.reactor.calledMethods timerCallback = addTimer.args[1] self.reactor.calledMethods.reset() timerCallback() addProcess, = self.reactor.calledMethods addProcessCallback = addProcess.args[0] self.reactor.calledMethods.reset() with stderr_replaced(): addProcessCallback() self.assertEqual('exception', self.pc.getState().errorState) self.assertEqual(['handle'], self.observer.calledMethodNames()) self.assertEqual(['removeProcess', 'addTimer'], self.reactor.calledMethodNames()) _, addTimer = self.reactor.calledMethods timerCallback = addTimer.args[1] self.reactor.calledMethods.reset() timerCallback() addProcess, = self.reactor.calledMethods addProcessCallback = addProcess.args[0] self.reactor.calledMethods.reset() addProcessCallback() self.assertEqual(['handle', 'handle'], self.observer.calledMethodNames()) addProcessCallback() self.assertEqual(['removeProcess', 'addTimer'], self.reactor.calledMethodNames()) self.assertEqual(None, self.pc.getState().errorState)
def testResumeStartsWhenPaused(self): self.newDNA(schedule=Schedule(period=2), autoStart=False) list(compose(self.dna.once.observer_init())) self.assertEquals([], self.reactor.calledMethodNames()) with stderr_replaced() as err: self.pc.resume() self.assertEquals('%s: resumed\n' % repr(self.pc), err.getvalue()) self.assertEquals(['addTimer'], self.reactor.calledMethodNames()) addTimer, = self.reactor.calledMethods self.assertEquals(self.pc._periodicCall, addTimer.args[1]) # finish one task (1/2) self.reactor.calledMethods.reset() addTimer.args[1]() addProcess, = self.reactor.calledMethods # finish one task (2/2) self.reactor.calledMethods.reset() addProcess.args[0]() self.assertEquals(['removeProcess', 'addTimer'], self.reactor.calledMethodNames())
def testStdErrReplaced(self): idStderr = id(sys.stderr) with stderr_replaced() as s: self.assertNotEqual(idStderr, id(sys.stderr)) try: raise Exception('xcptn') except Exception: print_exc() result = s.getvalue() self.assertTrue('Traceback' in result, result) self.assertTrue('Exception: xcptn' in result, result) self.assertEqual(idStderr, id(sys.stderr)) @stderr_replaced def f(): self.assertNotEqual(idStderr, id(sys.stderr)) raise Exception() self.assertEqual(idStderr, id(sys.stderr))
def test(): sp = SocketPool(reactor=CallTrace(), limits={'totalSize': 2}) # Limits enforced on put, not async. def stillPooled(): wasStillPooled = [] for destHost, destPort in [('h', 1), ('i', 2), ('j', 3)]: while True: # do ... while (fromPool is not None) fromPool = yield sp.getPooledSocket(host=destHost, port=destPort) if fromPool: wasStillPooled.append(fromPool) if fromPool is None: break raise StopIteration(wasStillPooled) yield sp.putSocketInPool(host='h', port=1, sock=MockSok('sH')) yield sp.putSocketInPool(host='i', port=2, sock=MockSok('sI')) with stderr_replaced() as err: yield sp.putSocketInPool(host='j', port=3, sock=MockSok('sJ')) self.assertEquals('', err.getvalue(), err.getvalue()) wasStillPooled = yield stillPooled() self.assertEquals(2, len(wasStillPooled)) self.assertTrue(set(wasStillPooled).issubset(set(['sH', 'sI', 'sJ'])))
def test(suspendMethod): with stderr_replaced() as s: s1 = next( compose( suspendMethod(clientIdentifier="a-client-id", prefix='prefix', sets=[], continueAfter='9876'))) s1(CallTrace('reactor'), lambda: None) next( compose( suspendMethod(clientIdentifier="another-client-id", prefix='prefix', sets=[], continueAfter='9876'))) try: s1.getResult() self.fail() except ForcedResumeException: self.assertEqual( "Too many suspended connections in SuspendRegister. One random connection has been resumed.\n", s.getvalue())
def testHandlerExitsWithException(self): exceptions = [] exc_message = "this is not an I/O exception, but an application exception" yielded_data = "a bit of data" def handler(**kwargs): yield yielded_data raise Exception(exc_message) yield server = HttpServer(self.reactor, self.port, handler, maxConnections=5) server.listen() sock = socket() sock.connect(('localhost', self.port)) self.reactor.step() sock.send("GET / HTTP/1.0\r\n\r\n") self.reactor.step() with stderr_replaced() as f: self.reactor.step().step() self.assertFalse(yielded_data in f.getvalue()) self.assertTrue("Exception: %s" % exc_message in f.getvalue()) server.shutdown() sock.close()
def test(): sp = SocketPool(reactor=CallTrace(), limits={'totalSize': 2}) # Limits enforced on put, not async. def stillPooled(): wasStillPooled = [] while True: # do ... while (fromPool is not None) fromPool = yield sp.getPooledSocket(host='h', port=1) if fromPool: wasStillPooled.append(fromPool) if fromPool is None: break raise StopIteration(wasStillPooled) s0 = MockSok('s0') s1 = MockSok('s1') yield sp.putSocketInPool(host='h', port=1, sock=s0) yield sp.putSocketInPool(host='h', port=1, sock=s1) with stderr_replaced() as err: yield sp.putSocketInPool(host='h', port=1, sock=MockSok('s2')) self.assertEquals('', err.getvalue(), err.getvalue()) #@@ wasStillPooled = yield stillPooled() self.assertEquals(['s2', 's0'], wasStillPooled) self.assertEquals(['shutdown', 'close'], s1.log.calledMethodNames()) shutCall, closeCall = s1.log.calledMethods self.assertEquals(((SHUT_RDWR, ), {}), (shutCall.args, shutCall.kwargs)) self.assertEquals(((), {}), (closeCall.args, closeCall.kwargs)) self.assertEquals([], s0.log.calledMethodNames()) yield sp.putSocketInPool(host='h', port=1, sock=MockSok('s0')) yield sp.putSocketInPool(host='h', port=1, sock=MockSok('s1')) yield sp.putSocketInPool(host='h', port=1, sock=MockSok('s2')) yield sp.putSocketInPool(host='h', port=1, sock=MockSok('s3')) wasStillPooled = yield stillPooled() self.assertEquals(['s3', 's0'], wasStillPooled)
def testErrorStateReset(self): handleCalledBefore = [] def handle(): if not handleCalledBefore: handleCalledBefore.append(True) raise Exception('exception') yield Yield self.observer.methods['handle'] = handle addTimer, = self.reactor.calledMethods timerCallback = addTimer.args[1] self.reactor.calledMethods.reset() timerCallback() addProcess, = self.reactor.calledMethods addProcessCallback = addProcess.args[0] self.reactor.calledMethods.reset() with stderr_replaced(): addProcessCallback() self.assertEquals('exception', self.pc.getState().errorState) self.assertEquals(['handle'], self.observer.calledMethodNames()) self.assertEquals(['removeProcess', 'addTimer'], self.reactor.calledMethodNames()) _, addTimer = self.reactor.calledMethods timerCallback = addTimer.args[1] self.reactor.calledMethods.reset() timerCallback() addProcess, = self.reactor.calledMethods addProcessCallback = addProcess.args[0] self.reactor.calledMethods.reset() addProcessCallback() self.assertEquals(['handle', 'handle'], self.observer.calledMethodNames()) addProcessCallback() self.assertEquals(['removeProcess', 'addTimer'], self.reactor.calledMethodNames()) self.assertEquals(None, self.pc.getState().errorState)
def testDiagnosticGetHandledByObserver(self): def mockAdditionalDiagnosticDetails(**kwargs): yield "additional details" def mockExecuteQuery(*args, **kwargs): raise Exception("Zo maar iets") yield observer = CallTrace(methods={ 'additionalDiagnosticDetails': mockAdditionalDiagnosticDetails, 'executeQuery': mockExecuteQuery}) dna = be( (Observable(), (SruHandler(), (observer, ) ) ) ) with stderr_replaced(): response = ''.join(compose(dna.all.searchRetrieve(query="word", sruArguments={}))) self.assertEquals(['executeQuery', 'additionalDiagnosticDetails'], observer.calledMethodNames()) self.assertTrue("<details>Zo maar iets - additional details</details>" in response, response)
def testHttpGetWithMaxSize(self): get_requests = [] port = PortNumberGenerator.next() self.referenceHttpServer(port, get_requests, streamingData="response"*1024) responses = [] def gethandler(*args, **kwargs): try: response = yield httpget('localhost', port, '/path', maxResponseSize=1024) responses.append(response) except Exception, e: responses.append(e) self.handler = gethandler clientget('localhost', self.port, '/') with stderr_replaced(): with stdout_replaced(): self._loopReactorUntilDone() self.assertEquals([TooBigResponseException], [type(r) for r in responses]) self.assertEquals(1024, responses[0].args[0]) self.assertEquals(1, len(get_requests)) self.assertEquals('GET', get_requests[0]['command']) def testHttpAndHttpsGetStreaming(self): for useSsl in [False, True]: get_request = [] port = PortNumberGenerator.next() streamingData = StreamingData(data=[c for c in "STREAMING GET RESPONSE"]) self.referenceHttpServer(port, get_request, ssl=useSsl, streamingData=streamingData)
def testDeprecatedPartsSpecification(self): with warnings.catch_warnings(): warnings.simplefilter("default") with stderr_replaced() as s: venturi = Venturi(should=[('partname', '/x/path')]) self.assertTrue("Please use {'partname'" in s.getvalue())
def test_full_cicle(self): # Parse html -> to py_str -> exec -> call main with tag -> extract generated html -> parse to data -> compare with expected. def f(html_text, remove_blank_text=None): kw = {} if remove_blank_text is not None: kw['remove_blank_text'] = remove_blank_text return etree_to_data(html_to_etree(processTemplate(self, html_to_tag(html_text, **kw)), **kw)) # simple self.assertEquals( {'tag': 'h1', 'text': 'Hi!'}, f('<h1>Hi!</h1>')) # nested with "irrelevant" whitespace removed. chicken_soup = '''\ <html lang="en"> <head> <title>Hello</title> </head> <body> <div id="container"> <h1 class="red hero">Welcome !</h1> <p> According to Dr. Jason chicken soup is one of the <strong>best</strong> soups invented. </p> </div> </body> </html>''' self.assertEquals( {'tag': 'html', 'attribs': {'lang': 'en'}, 'children': [ {'tag': 'head', 'children': [ {'tag': 'title', 'text': 'Hello'}]}, {'tag': 'body', 'children': [ {'tag': 'div', 'attribs': {'id': 'container'}, 'children': [ {'tag': 'h1', 'attribs': {'class': 'red hero'}, 'text': 'Welcome !'}, {'tag': 'p', 'text': '\n According to Dr. Jason chicken soup is one of the ', 'children': [ {'tag': 'strong', 'text': 'best', 'tail': ' soups invented.\n ', }]}]}]}]}, f(chicken_soup)) # nested without "irrelevant" whitespace removed self.assertEquals( {'tag': 'html', 'attribs': {'lang': 'en'}, 'children': [ {'tag': 'head', 'text': '\n ', 'tail': '\n ', 'children': [ {'tag': 'title', 'text': 'Hello', 'tail': '\n ', }], }, {'tag': 'body', 'text': '\n ', 'children': [ {'tag': 'div', 'text': '\n ', 'tail': '\n \n', 'attribs': {'id': 'container'}, 'children': [ {'tag': 'h1', 'text': 'Welcome !', 'tail': '\n ', 'attribs': {'class': 'red hero'}, }, {'tag': 'p', 'text': '\n According to Dr. Jason chicken soup is one of the ', 'tail': '\n ', 'children': [ {'tag': 'strong', 'text': 'best', 'tail': ' soups invented.\n ', }], }], }], }], }, f(chicken_soup, remove_blank_text=False)) # highlights namespace-bugs in html5lib (iff this test starts failing - party!). weird_namespaceness = '''\ <a>some text<svg xmlns="http://www.w3.org/2000/svg" width="32" height="32" viewbox="0 0 32 32" xmlns:xlink="http://www.w3.org/1999/xlink"><path fill="#fff" d="M15.6097079,14.5927191"/></svg> </a>''' expected_bad = { 'tag': 'a', 'text': 'some text', 'children': [ {'tag': 'svg', 'tail': '\n', 'attribs': { 'height': '32', 'ns1u0003axlink': 'http://www.w3.org/1999/xlink', 'ns1u0003axmlns': 'http://www.w3.org/2000/svg', 'viewBox': '0 0 32 32', 'width': '32', 'xmlnsU0003Ans0': 'http://www.w3.org/2000/svg', 'xmlnsu0003ans0': 'http://www.w3.org/2000/svg'}, 'children': [ {'attribs': { 'd': 'M15.6097079,14.5927191', 'fill': '#fff'}, 'tag': 'path'}], }]} with stderr_replaced() as err: result = f(weird_namespaceness) self.assertTrue('DataLossWarning' in err.getvalue(), err.getvalue()) # This is weird in its own right... self.assertEquals( expected_bad, result) result = f(weird_namespaceness, remove_blank_text=False) self.assertEquals('pre', result['tag']) self.assertEquals(None, result.get('children')) self.assertTrue('Traceback' in result['text'])#, result['text']) self.assertTrue('SyntaxError:' in result['text'])#, result['text'])
def testSuspendTimeoutOnTimeoutCallbackGivesException(self): def prepare(_onTimeout): # Use indirect tracing of onTimeout call (otherwise CallTrace internals leak into the stacktrace). trace = CallTrace(returnValues={'addTimer': 'timerToken'}) onTimeoutM = trace.onTimeout def onTimeoutTraced(): onTimeoutM() # We have been called Watson! return _onTimeout() trace.onTimeout = onTimeoutTraced suspend = Suspend(doNext=trace.doNext, timeout=3.14, onTimeout=trace.onTimeout) suspend(reactor=trace, whenDone=trace.whenDone) self.assertEquals(['doNext', 'addTimer', 'suspend'], trace.calledMethodNames()) trace.calledMethods.reset() return trace, suspend # basic invariant trace, suspend = prepare(lambda: None) suspend._timedOut() self.assertEquals(['onTimeout', 'whenDone'], trace.calledMethodNames()) self.assertRaises(TimeoutException, lambda: suspend.getResult()) # normal exceptions def onTimeout(): raise Exception( "This Should Never Happen But Don't Expose Exception If It Does Anyway !" ) trace, suspend = prepare(onTimeout) with stderr_replaced() as err: suspend._timedOut() expectedTraceback = ignoreLineNumbers( '''Unexpected exception raised on Suspend's onTimeout callback (ignored): Traceback (most recent call last): File "%(suspend.py)s", line 101, in _timedOut self._onTimeout() File "%(__file__)s", line 382, in onTimeoutTraced return _onTimeout() File "%(__file__)s", line 398, in onTimeout raise Exception("This Should Never Happen But Don't Expose Exception If It Does Anyway !") Exception: This Should Never Happen But Don't Expose Exception If It Does Anyway ! ''' % fileDict) self.assertEqualsWS(expectedTraceback, ignoreLineNumbers(err.getvalue())) self.assertEquals(['onTimeout', 'whenDone'], trace.calledMethodNames()) self.assertRaises(TimeoutException, lambda: suspend.getResult()) # fatal exceptions def suspendCallWithOnTimeoutRaising(exception): def onTimeout(): raise exception trace, suspend = prepare(onTimeout) try: suspend._timedOut() except: c, v, t = exc_info() self.assertEquals(['onTimeout'], trace.calledMethodNames()) raise c, v, t.tb_next self.assertRaises( KeyboardInterrupt, lambda: suspendCallWithOnTimeoutRaising(KeyboardInterrupt())) self.assertRaises( SystemExit, lambda: suspendCallWithOnTimeoutRaising(SystemExit())) self.assertRaises( AssertionError, lambda: suspendCallWithOnTimeoutRaising(AssertionError()))
def testGetTime(self): timedDict = TimedDictionary(TWO_HOURS) timedDict[1] = SomeObject('id1') with stderr_replaced(): self.assertTrue(time() - timedDict.getTime(1) < 2.0)
def testGetTime(self): self.timedDict[1] = SomeObject('id1') with stderr_replaced(): self.assertTrue(time() - self.timedDict.getTime(1) < 2.0)
def testShouldRaiseExceptionOnSameRequestTwice(self): self.run = True portNumber = randint(50000, 60000) oaiJazz = OaiJazz(join(self.tempdir, 'oai')) oaiJazz.updateMetadataFormat(prefix="prefix", schema="", namespace="") suspendRegister = SuspendRegister() oaiJazz.addObserver(suspendRegister) storageComponent = MultiSequentialStorage(join(self.tempdir, 'storage')) clientId = str(uuid4()) responses = [] def doOaiListRecord(port): header, body = getRequest(port=portNumber, path="/", arguments={ "verb": "ListRecords", "metadataPrefix": "prefix", "x-wait": "True" }, additionalHeaders={ 'X-Meresco-Oai-Client-Identifier': clientId }, parse=False) responses.append((header, body)) oaiPmhThread = Thread( None, lambda: self.startOaiPmh(portNumber, oaiJazz, storageComponent, suspendRegister)) harvestThread1 = Thread(None, lambda: doOaiListRecord(portNumber)) harvestThread2 = Thread(None, lambda: doOaiListRecord(portNumber)) with stderr_replaced(): oaiPmhThread.start() harvestThread1.start() try: while len(suspendRegister) == 0: sleep(0.01) harvest1Suspend = suspendRegister._suspendObject(clientId) self.assertTrue(harvest1Suspend is not None) harvestThread2.start() while harvest1Suspend == suspendRegister._suspendObject( clientId): sleep(0.01) sleep(0.01) self.assertTrue(clientId in suspendRegister) self.assertTrue( harvest1Suspend != suspendRegister._suspendObject(clientId) ) self.assertEqual(1, len(responses)) statusAndHeader, body = responses[0] self.assertEqual("204", statusAndHeader['StatusCode']) self.assertTrue(body.startswith(b'Aborting suspended request'), body) storageComponent.addData(identifier="id1", name="prefix", data=b"<a>a1</a>") oaiJazz.addOaiRecord(identifier="id1", metadataPrefixes=["prefix"]) sleep(0.1) finally: self.run = False oaiPmhThread.join() harvestThread1.join() harvestThread2.join() oaiJazz.close()
def test_full_cicle(self): # Parse html -> to py_str -> exec -> call main with tag -> extract generated html -> parse to data -> compare with expected. def f(html_text, remove_blank_text=None): kw = {} if remove_blank_text is not None: kw['remove_blank_text'] = remove_blank_text return etree_to_data( html_to_etree( processTemplate(self, html_to_tag(html_text, **kw)), **kw)) # simple self.assertEquals({'tag': 'h1', 'text': 'Hi!'}, f('<h1>Hi!</h1>')) # nested with "irrelevant" whitespace removed. chicken_soup = '''\ <html lang="en"> <head> <title>Hello</title> </head> <body> <div id="container"> <h1 class="red hero">Welcome !</h1> <p> According to Dr. Jason chicken soup is one of the <strong>best</strong> soups invented. </p> </div> </body> </html>''' self.assertEquals( { 'tag': 'html', 'attribs': { 'lang': 'en' }, 'children': [{ 'tag': 'head', 'children': [{ 'tag': 'title', 'text': 'Hello' }] }, { 'tag': 'body', 'children': [{ 'tag': 'div', 'attribs': { 'id': 'container' }, 'children': [{ 'tag': 'h1', 'attribs': { 'class': 'red hero' }, 'text': 'Welcome !' }, { 'tag': 'p', 'text': '\n According to Dr. Jason chicken soup is one of the ', 'children': [{ 'tag': 'strong', 'text': 'best', 'tail': ' soups invented.\n ', }] }] }] }] }, f(chicken_soup)) # nested without "irrelevant" whitespace removed self.assertEquals( { 'tag': 'html', 'attribs': { 'lang': 'en' }, 'children': [{ 'tag': 'head', 'text': '\n ', 'tail': '\n ', 'children': [{ 'tag': 'title', 'text': 'Hello', 'tail': '\n ', }], }, { 'tag': 'body', 'text': '\n ', 'children': [{ 'tag': 'div', 'text': '\n ', 'tail': '\n \n', 'attribs': { 'id': 'container' }, 'children': [{ 'tag': 'h1', 'text': 'Welcome !', 'tail': '\n ', 'attribs': { 'class': 'red hero' }, }, { 'tag': 'p', 'text': '\n According to Dr. Jason chicken soup is one of the ', 'tail': '\n ', 'children': [{ 'tag': 'strong', 'text': 'best', 'tail': ' soups invented.\n ', }], }], }], }], }, f(chicken_soup, remove_blank_text=False)) # highlights namespace-bugs in html5lib (iff this test starts failing - party!). weird_namespaceness = '''\ <a>some text<svg xmlns="http://www.w3.org/2000/svg" width="32" height="32" viewbox="0 0 32 32" xmlns:xlink="http://www.w3.org/1999/xlink"><path fill="#fff" d="M15.6097079,14.5927191"/></svg> </a>''' expected_bad = { 'tag': 'a', 'text': 'some text', 'children': [{ 'tag': 'svg', 'tail': '\n', 'attribs': { 'height': '32', 'ns1u0003axlink': 'http://www.w3.org/1999/xlink', 'ns1u0003axmlns': 'http://www.w3.org/2000/svg', 'viewBox': '0 0 32 32', 'width': '32', 'xmlnsU0003Ans0': 'http://www.w3.org/2000/svg', 'xmlnsu0003ans0': 'http://www.w3.org/2000/svg' }, 'children': [{ 'attribs': { 'd': 'M15.6097079,14.5927191', 'fill': '#fff' }, 'tag': 'path' }], }] } with stderr_replaced() as err: result = f(weird_namespaceness) self.assertTrue( 'DataLossWarning' in err.getvalue(), err.getvalue()) # This is weird in its own right... self.assertEquals(expected_bad, result) result = f(weird_namespaceness, remove_blank_text=False) self.assertEquals('pre', result['tag']) self.assertEquals(None, result.get('children')) self.assertTrue('Traceback' in result['text']) #, result['text']) self.assertTrue('SyntaxError:' in result['text']) #, result['text'])
exceptions = [] yielded_data = "OK" * 1000 def handler(**kwargs): try: while True: yield yielded_data except Exception, e: exceptions.append(e) server = HttpServer(self.reactor, self.port, handler, maxConnections=5) server.listen() sock = socket() sock.connect(('localhost', self.port)) self.reactor.step() sock.send("GET / HTTP/1.0\r\n\r\n") sock.shutdown(SHUT_RDWR) with stderr_replaced() as f: while not exceptions: self.reactor.step() self.assertTrue("Error while sending data \"%s" % yielded_data[:10] in f.getvalue()) self.assertTrue("Traceback" in f.getvalue()) self.assertTrue("[Errno 32] Broken pipe" in f.getvalue()) self.assertEquals("[Errno 32] Broken pipe", str(exceptions[0])) server.shutdown() sock.close() self.reactor.removeTimer(token=token) def testHandlerExitsWithException(self): exceptions = []
responses = [] def gethandler(*args, **kwargs): try: response = yield httpget('localhost', port, '/path', maxResponseSize=1024) responses.append(response) except Exception, e: responses.append(e) self.handler = gethandler clientget('localhost', self.port, '/') with stderr_replaced(): with stdout_replaced(): self._loopReactorUntilDone() self.assertEquals([TooBigResponseException], [type(r) for r in responses]) self.assertEquals(1024, responses[0].args[0]) self.assertEquals(1, len(get_requests)) self.assertEquals('GET', get_requests[0]['command']) def testHttpAndHttpsGetStreaming(self): for useSsl in [False, True]: get_request = [] port = PortNumberGenerator.next() streamingData = StreamingData( data=[c for c in "STREAMING GET RESPONSE"])