def testSuspendProtocolWithThrow(self): with Reactor() as reactor: suspend = Suspend() def handler(**httpvars): yield 'before suspend' yield suspend try: suspend.getResult() self.fail() except ValueError, e: tbstring = format_exc() yield "result = %s" % tbstring yield 'after suspend' listener = MyMockSocket() port = PortNumberGenerator.next() httpserver = HttpServer(reactor, port, handler, sok=listener) httpserver.listen() reactor.removeReader(listener) # avoid new connections httpserver._acceptor._accept() reactor.step() reactor.step() reactor.step() self.assertEquals(reactor, suspend._reactor) self.assertEquals(0, len(reactor._fds)) def raiser(): raise ValueError("BAD VALUE") try: raiser() except ValueError, e: exc_type, exc_value, exc_traceback = exc_info() suspend.throw(exc_type, exc_value, exc_traceback)
def testDoNextErrorReRaisedOnGetResult(self): def razor(ignored): 1 / 0 # Division by zero exception suspend = Suspend(doNext=razor) olderr = sys.stderr sys.stderr = StringIO() try: suspend(reactor=CallTrace(), whenDone="not called") finally: sys.stderr = olderr try: suspend.getResult() except: exc_type, exc_value, exc_traceback = exc_info() expectedTraceback = ignoreLineNumbers( """Traceback (most recent call last): File "%(__file__)s", line 200, in testDoNextErrorReRaisedOnGetResult suspend.getResult() File "%(suspend.py)s", line 40, in __call__ self._doNext(self) File "%(__file__)s", line 196, in razor 1/0 # Division by zero exception ZeroDivisionError: integer division or modulo by zero """ % fileDict) self.assertEquals(ZeroDivisionError, exc_type) self.assertEqualsWS(expectedTraceback, ignoreLineNumbers(format_exc(exc_traceback)))
def testSuspendTimingOut(self): # with calltrace; happy path trace = CallTrace(returnValues={'addTimer': 'timerToken'}) suspend = Suspend(doNext=trace.doNext, timeout=3.14, onTimeout=trace.onTimeout) self.assertEquals([], trace.calledMethodNames()) self.assertEquals(None, suspend._timer) suspend(reactor=trace, whenDone=trace.whenDone) self.assertEquals(['doNext', 'addTimer', 'suspend'], trace.calledMethodNames()) doNextM, addTimerM, suspendM = trace.calledMethods self.assertEquals(((suspend, ), {}), (doNextM.args, doNextM.kwargs)) self.assertEquals(((), { 'seconds': 3.14, 'callback': suspend._timedOut }), (addTimerM.args, addTimerM.kwargs)) self.assertEquals(((), {}), (suspendM.args, suspendM.kwargs)) self.assertEquals('timerToken', suspend._timer) trace.calledMethods.reset() suspend._timedOut() self.assertEquals(['onTimeout', 'whenDone'], trace.calledMethodNames()) onTimeoutM, whenDoneM = trace.calledMethods self.assertEquals(((), {}), (onTimeoutM.args, onTimeoutM.kwargs)) self.assertEquals(((), {}), (whenDoneM.args, whenDoneM.kwargs)) self.assertEquals(None, suspend._timer) self.assertRaises(TimeoutException, lambda: suspend.getResult())
def testDoNextErrorReRaisedOnGetResult(self): def razor(ignored): 1/0 # Division by zero exception suspend = Suspend(doNext=razor) olderr = sys.stderr sys.stderr = StringIO() try: suspend(reactor=CallTrace(), whenDone="not called") finally: sys.stderr = olderr try: suspend.getResult() except: exc_type, exc_value, exc_traceback = exc_info() expectedTraceback = ignoreLineNumbers("""Traceback (most recent call last): File "%(__file__)s", line 200, in testDoNextErrorReRaisedOnGetResult suspend.getResult() File "%(suspend.py)s", line 40, in __call__ self._doNext(self) File "%(__file__)s", line 196, in razor 1/0 # Division by zero exception ZeroDivisionError: integer division or modulo by zero """ % fileDict) self.assertEquals(ZeroDivisionError, exc_type) self.assertEqualsWS(expectedTraceback, ignoreLineNumbers(format_exc(exc_traceback)))
def testSuspendProtocol(self): with Reactor() as reactor: suspend = Suspend() def handler(**httpvars): yield 'before suspend' yield suspend yield "result = %s" % suspend.getResult() yield 'after suspend' listener = MyMockSocket() port = PortNumberGenerator.next() httpserver = HttpServer(reactor, port, handler, sok=listener) httpserver.listen() reactor.removeReader(listener) # avoid new connections httpserver._acceptor._accept() self.assertEquals(1, len(reactor._fds)) self.assertEquals([READ_INTENT], [v.intent for v in reactor._fds.values()]) reactor.step() self.assertEquals(1, len(reactor._fds)) self.assertEquals([WRITE_INTENT], [v.intent for v in reactor._fds.values()]) reactor.step() reactor.step() self.assertEquals(reactor, suspend._reactor) self.assertEquals(0, len(reactor._fds)) suspend.resume('RESPONSE') self.assertEquals(1, len(reactor._fds)) self.assertEquals([WRITE_INTENT], [v.intent for v in reactor._fds.values()]) reactor.step() reactor.step() reactor.step() self.assertEquals(['before suspend', 'result = RESPONSE', 'after suspend'], listener.data) self.assertEquals(0, len(reactor._fds)) # cleanup (most) fd's listener.close()
def testSuspendProtocol(self): reactor = Reactor(select_func=mockselect) suspend = Suspend() def handler(**httpvars): yield 'before suspend' yield suspend yield "result = %s" % suspend.getResult() yield 'after suspend' listener = MyMockSocket() port = 9 httpserver = HttpServer(reactor, port, handler, sok=listener) httpserver.listen() reactor.removeReader(listener) # avoid new connections httpserver._acceptor._accept() reactor.step() reactor.step() self.assertEquals(1, len(reactor._writers)) reactor.step() self.assertEquals(reactor, suspend._reactor) self.assertEquals(0, len(reactor._writers)) suspend.resume('RESPONSE') reactor.step() reactor.step() reactor.step() self.assertEquals(['before suspend', 'result = RESPONSE', 'after suspend'], listener.data)
def suspendWrap(): doNext, onTimeout = workInSuspend() s = Suspend(doNext=doNext, timeout=0.01, onTimeout=onTimeout) yield s result = s.getResult() if expectTimeout: self.fail('Should not have come here') raise StopIteration(result)
def testSuspendThrowBackwardsCompatibleWithInstanceOnlyThrow_YouWillMissTracebackHistory( self): with Reactor() as reactor: suspend = Suspend() def handler(**httpvars): yield 'before suspend' yield suspend try: suspend.getResult() self.fail() except ValueError, e: tbstring = format_exc() yield "result = %s" % tbstring yield 'after suspend' listener = MyMockSocket() port = PortNumberGenerator.next() httpserver = HttpServer(reactor, port, handler, sok=listener) httpserver.listen() reactor.removeReader(listener) # avoid new connections httpserver._acceptor._accept() self.assertEquals(1, len(reactor._fds)) reactor.step() reactor.step() reactor.step() self.assertEquals(reactor, suspend._reactor) self.assertEquals(0, len(reactor._fds)) def raiser(): raise ValueError("BAD VALUE") try: raiser() except: exc_value = exc_info()[1] suspend.throw(exc_value) self.assertEquals(1, len(reactor._fds)) reactor.step() reactor.step() reactor.step() expectedTraceback = ignoreLineNumbers( """Traceback (most recent call last): File "%(__file__)s", line 201, in handler suspend.getResult() File "%(suspend.py)s", line 62, in getResult raise self._exception[0], self._exception[1], self._exception[2] ValueError: BAD VALUE """ % fileDict) self.assertEquals(3, len(listener.data)) self.assertEquals('before suspend', listener.data[0]) self.assertEqualsWS("result = %s" % expectedTraceback, ignoreLineNumbers(listener.data[1])) self.assertEquals('after suspend', listener.data[2]) self.assertEquals(0, len(reactor._fds)) # cleanup (most) fd's listener.close()
def prepare(): trace = CallTrace(returnValues={'addTimer': 'timerToken'}) suspend = Suspend(doNext=trace.doNext, timeout=3.14, onTimeout=trace.onTimeout) suspend(reactor=trace, whenDone=trace.whenDone) self.assertEquals(['doNext', 'addTimer', 'suspend'], trace.calledMethodNames()) trace.calledMethods.reset() suspend._timedOut() self.assertEquals(['onTimeout', 'whenDone'], trace.calledMethodNames()) self.assertEquals(True, suspend._settled) trace.calledMethods.reset() return trace, suspend
def testSuspendThrowBackwardsCompatibleWithInstanceOnlyThrow_YouWillMissTracebackHistory(self): with Reactor() as reactor: suspend = Suspend() def handler(**httpvars): yield 'before suspend' yield suspend try: suspend.getResult() self.fail() except ValueError, e: tbstring = format_exc() yield "result = %s" % tbstring yield 'after suspend' listener = MyMockSocket() port = PortNumberGenerator.next() httpserver = HttpServer(reactor, port, handler, sok=listener) httpserver.listen() reactor.removeReader(listener) # avoid new connections httpserver._acceptor._accept() self.assertEquals(1, len(reactor._fds)) reactor.step() reactor.step() reactor.step() self.assertEquals(reactor, suspend._reactor) self.assertEquals(0, len(reactor._fds)) def raiser(): raise ValueError("BAD VALUE") try: raiser() except: exc_value = exc_info()[1] suspend.throw(exc_value) self.assertEquals(1, len(reactor._fds)) reactor.step() reactor.step() reactor.step() expectedTraceback = ignoreLineNumbers("""Traceback (most recent call last): File "%(__file__)s", line 201, in handler suspend.getResult() File "%(suspend.py)s", line 62, in getResult raise self._exception[0], self._exception[1], self._exception[2] ValueError: BAD VALUE """ % fileDict) self.assertEquals(3, len(listener.data)) self.assertEquals('before suspend', listener.data[0]) self.assertEqualsWS("result = %s" % expectedTraceback, ignoreLineNumbers(listener.data[1])) self.assertEquals('after suspend', listener.data[2]) self.assertEquals(0, len(reactor._fds)) # cleanup (most) fd's listener.close()
def _suspend(self, clientIdentifier, prefix, sets, **ignored): suspend = Suspend() suspend.oaiListResumeMask = dict( prefix=prefix, sets=set() if sets is None else set(sets) ) if clientIdentifier in self._register: self._register.pop(clientIdentifier).throw(exc_type=ForcedResumeException, exc_value=ForcedResumeException("Aborting suspended request because of new request for the same OaiClient with identifier: %s." % clientIdentifier), exc_traceback=None) elif len(self._register) == self._maximumSuspendedConnections: self._register.pop(choice(self._register.keys())).throw(exc_type=ForcedResumeException, exc_value=ForcedResumeException("OAI x-wait connection has been forcefully resumed."), exc_traceback=None) sys.stderr.write("Too many suspended connections in SuspendRegister. One random connection has been resumed.\n") self._register[clientIdentifier] = suspend yield suspend suspend.getResult()
def sleep(seconds): def doNext(): suspend = yield # from Suspend.__call__ yield # Wait for timeout yield # wait for GC g = doNext(); g.next() # + autostart s = Suspend(doNext=g.send, timeout=seconds, onTimeout=g.next) yield s try: s.getResult() except TimeoutException: pass raise StopIteration(None)
def httprequest(host, port, request, body=None, headers=None, proxyServer=None, ssl=False, prio=None, handlePartialResponse=None, method='GET', timeout=None, maxResponseSize=None): g = _do(method, host=host, port=port, request=request, headers=headers, proxyServer=proxyServer, body=body, ssl=ssl, prio=prio, handlePartialResponse=handlePartialResponse, maxResponseSize=maxResponseSize) kw = {} if timeout is not None: def onTimeout(): g.throw(TimeoutException, TimeoutException(), None) kw = { 'timeout': timeout, 'onTimeout': onTimeout, } s = Suspend(doNext=g.send, **kw) yield s result = s.getResult() raise StopIteration(result)
def httprequest1_1(self, host, port, request, body=None, headers=None, secure=False, prio=None, method='GET', timeout=None): g = _do(method=method, host=host, port=port, request=request, headers=headers, body=body, secure=secure, prio=prio, observable=self) kw = {} if timeout is not None: def onTimeout(): g.throw(TimeoutException, TimeoutException(), None) kw = { 'timeout': timeout, 'onTimeout': onTimeout, } s = Suspend(doNext=g.send, **kw) yield s result = s.getResult() raise StopIteration(result)
def httprequest1_1(self, host, port, request, body=None, headers=None, secure=False, prio=None, method='GET', timeout=None, bodyMaxSize=None): g = _do(method=method, host=host, port=port, request=request, headers=headers, body=body, bodyMaxSize=bodyMaxSize, secure=secure, prio=prio, observable=self) kw = {} if timeout is not None: def onTimeout(): g.throw(TimeoutException, TimeoutException(), None) kw = { 'timeout': timeout, 'onTimeout': onTimeout, } s = Suspend(doNext=g.send, **kw) yield s result = s.getResult() raise StopIteration(result)
def testSuspendProtocol(self): with Reactor() as reactor: suspend = Suspend() def handler(**httpvars): yield 'before suspend' yield suspend yield "result = %s" % suspend.getResult() yield 'after suspend' listener = MyMockSocket() port = PortNumberGenerator.next() httpserver = HttpServer(reactor, port, handler, sok=listener) httpserver.listen() reactor.removeReader(listener) # avoid new connections httpserver._acceptor._accept() self.assertEquals(1, len(reactor._fds)) self.assertEquals([READ_INTENT], [v.intent for v in reactor._fds.values()]) reactor.step() self.assertEquals(1, len(reactor._fds)) self.assertEquals([WRITE_INTENT], [v.intent for v in reactor._fds.values()]) reactor.step() reactor.step() self.assertEquals(reactor, suspend._reactor) self.assertEquals(0, len(reactor._fds)) suspend.resume('RESPONSE') self.assertEquals(1, len(reactor._fds)) self.assertEquals([WRITE_INTENT], [v.intent for v in reactor._fds.values()]) reactor.step() reactor.step() reactor.step() self.assertEquals( ['before suspend', 'result = RESPONSE', 'after suspend'], listener.data) self.assertEquals(0, len(reactor._fds)) # cleanup (most) fd's listener.close()
def getSuspend(): trace.calledMethods.reset() suspend = Suspend(doNext=trace.doNext) self.assertEquals([], trace.calledMethodNames()) suspend(reactor=trace, whenDone=trace.whenDone) self.assertEquals(['doNext', 'suspend'], trace.calledMethodNames()) doNextM, suspendM = trace.calledMethods self.assertEquals(((suspend, ), {}), (doNextM.args, doNextM.kwargs)) self.assertEquals(((), {}), (suspendM.args, suspendM.kwargs)) trace.calledMethods.reset() return suspend
def _suspend(self, clientIdentifier, prefix, sets, **ignored): suspend = Suspend() suspend.oaiListResumeMask = dict( prefix=prefix, sets=set() if sets is None else set(sets)) if clientIdentifier in self._register: self._register.pop(clientIdentifier).throw( exc_type=ForcedResumeException, exc_value=ForcedResumeException( "Aborting suspended request because of new request for the same OaiClient with identifier: %s." % clientIdentifier), exc_traceback=None) elif len(self._register) == self._maximumSuspendedConnections: self._register.pop(choice(list(self._register.keys()))).throw( exc_type=ForcedResumeException, exc_value=ForcedResumeException( "OAI x-wait connection has been forcefully resumed."), exc_traceback=None) sys.stderr.write( "Too many suspended connections in SuspendRegister. One random connection has been resumed.\n" ) self._register[clientIdentifier] = suspend yield suspend suspend.getResult()
def testYieldSuspendFromAdd(self): observer = CallTrace() oaiDownloadProcessor = OaiDownloadProcessor( path="/oai", metadataPrefix="oai_dc", workingDirectory=self.tempdir, xWait=False) oaiDownloadProcessor.addObserver(observer) suspend = Suspend() observer.returnValues['add'] = (x for x in [suspend]) yields = list( compose( oaiDownloadProcessor.handle( parse(StringIO(LISTRECORDS_RESPONSE % ''))))) self.assertEqual([suspend, None], yields)
def testSuspendTimingOut(self): # with calltrace; happy path trace = CallTrace(returnValues={'addTimer': 'timerToken'}) suspend = Suspend(doNext=trace.doNext, timeout=3.14, onTimeout=trace.onTimeout) self.assertEquals([], trace.calledMethodNames()) self.assertEquals(None, suspend._timer) suspend(reactor=trace, whenDone=trace.whenDone) self.assertEquals(['doNext', 'addTimer', 'suspend'], trace.calledMethodNames()) doNextM, addTimerM, suspendM = trace.calledMethods self.assertEquals(((suspend,), {}), (doNextM.args, doNextM.kwargs)) self.assertEquals(((), {'seconds': 3.14, 'callback': suspend._timedOut}), (addTimerM.args, addTimerM.kwargs)) self.assertEquals(((), {}), (suspendM.args, suspendM.kwargs)) self.assertEquals('timerToken', suspend._timer) trace.calledMethods.reset() suspend._timedOut() self.assertEquals(['onTimeout', 'whenDone'], trace.calledMethodNames()) onTimeoutM, whenDoneM = trace.calledMethods self.assertEquals(((), {}), (onTimeoutM.args, onTimeoutM.kwargs)) self.assertEquals(((), {}), (whenDoneM.args, whenDoneM.kwargs)) self.assertEquals(None, suspend._timer) self.assertRaises(TimeoutException, lambda: suspend.getResult())
def prepare(_onTimeout): # Use indirect tracing of onTimeout call (otherwise CallTrace internals leak into the stacktrace). trace = CallTrace(returnValues={'addTimer': 'timerToken'}) onTimeoutM = trace.onTimeout def onTimeoutTraced(): onTimeoutM() # We have been called Watson! return _onTimeout() trace.onTimeout = onTimeoutTraced suspend = Suspend(doNext=trace.doNext, timeout=3.14, onTimeout=trace.onTimeout) suspend(reactor=trace, whenDone=trace.whenDone) self.assertEquals(['doNext', 'addTimer', 'suspend'], trace.calledMethodNames()) trace.calledMethods.reset() return trace, suspend
def prepare(): trace = CallTrace(returnValues={'addTimer': 'timerToken'}) suspend = Suspend(doNext=trace.doNext, timeout=3.14, onTimeout=trace.onTimeout) self.assertEquals([], trace.calledMethodNames()) suspend(reactor=trace, whenDone=trace.whenDone) self.assertEquals(['doNext', 'addTimer', 'suspend'], trace.calledMethodNames()) doNextM, addTimerM, suspendM = trace.calledMethods self.assertEquals(((suspend, ), {}), (doNextM.args, doNextM.kwargs)) self.assertEquals(((), { 'seconds': 3.14, 'callback': suspend._timedOut }), (addTimerM.args, addTimerM.kwargs)) self.assertEquals(((), {}), (suspendM.args, suspendM.kwargs)) trace.calledMethods.reset() return trace, suspend
def httppost(**_kwargs): kwargs.append(_kwargs) s = Suspend() yield s result = s.getResult() raise StopIteration(result)
def testGetNoneResult(self): reactor = CallTrace('reactor') s = Suspend() s(reactor, whenDone=lambda:None) s.resume() self.assertEquals(None, s.getResult())
def testGetResultRaisesException(self): reactor = CallTrace('reactor') s = Suspend() s(reactor, whenDone=lambda:None) s.throw(ValueError('bad value')) self.assertRaises(ValueError, s.getResult)
def suspending(): s = Suspend(whileSuspended().send) yield s raise StopIteration(s.getResult())
def httpget(host, port, request, headers=None): s = Suspend(_do('GET', host, port, request, headers=headers).send) yield s result = s.getResult() raise StopIteration(result)
def testSuspendTimeoutArguments(self): self.assertRaises(ValueError, lambda: Suspend(timeout=3)) self.assertRaises(ValueError, lambda: Suspend(onTimeout=lambda: None)) Suspend() Suspend(timeout=3, onTimeout=lambda: None)
def suspendWithDoNextException(exception): def doNextRaising(suspend): raise exception suspend = Suspend(doNext=doNextRaising) suspend(reactor=CallTrace(), whenDone="not called")
def testGetResultRaisesException(self): reactor = CallTrace('reactor') s = Suspend() s(reactor, whenDone=lambda: None) s.throw(ValueError('bad value')) self.assertRaises(ValueError, s.getResult)
def testGetNoneResult(self): reactor = CallTrace('reactor') s = Suspend() s(reactor, whenDone=lambda: None) s.resume() self.assertEquals(None, s.getResult())