def testSuspendProtocolWithThrow(self):
with Reactor() as reactor:
suspend = Suspend()
def handler(**httpvars):
yield 'before suspend'
yield suspend
try:
suspend.getResult()
self.fail()
except ValueError, e:
tbstring = format_exc()
yield "result = %s" % tbstring
yield 'after suspend'
listener = MyMockSocket()
port = PortNumberGenerator.next()
httpserver = HttpServer(reactor, port, handler, sok=listener)
httpserver.listen()
reactor.removeReader(listener) # avoid new connections
httpserver._acceptor._accept()
reactor.step()
reactor.step()
reactor.step()
self.assertEquals(reactor, suspend._reactor)
self.assertEquals(0, len(reactor._fds))
def raiser():
raise ValueError("BAD VALUE")
try:
raiser()
except ValueError, e:
exc_type, exc_value, exc_traceback = exc_info()
suspend.throw(exc_type, exc_value, exc_traceback)
def testDoNextErrorReRaisedOnGetResult(self):
def razor(ignored):
1 / 0 # Division by zero exception
suspend = Suspend(doNext=razor)
olderr = sys.stderr
sys.stderr = StringIO()
try:
suspend(reactor=CallTrace(), whenDone="not called")
finally:
sys.stderr = olderr
try:
suspend.getResult()
except:
exc_type, exc_value, exc_traceback = exc_info()
expectedTraceback = ignoreLineNumbers(
"""Traceback (most recent call last):
File "%(__file__)s", line 200, in testDoNextErrorReRaisedOnGetResult
suspend.getResult()
File "%(suspend.py)s", line 40, in __call__
self._doNext(self)
File "%(__file__)s", line 196, in razor
1/0 # Division by zero exception
ZeroDivisionError: integer division or modulo by zero
""" % fileDict)
self.assertEquals(ZeroDivisionError, exc_type)
self.assertEqualsWS(expectedTraceback,
ignoreLineNumbers(format_exc(exc_traceback)))
def testSuspendTimingOut(self):
# with calltrace; happy path
trace = CallTrace(returnValues={'addTimer': 'timerToken'})
suspend = Suspend(doNext=trace.doNext,
timeout=3.14,
onTimeout=trace.onTimeout)
self.assertEquals([], trace.calledMethodNames())
self.assertEquals(None, suspend._timer)
suspend(reactor=trace, whenDone=trace.whenDone)
self.assertEquals(['doNext', 'addTimer', 'suspend'],
trace.calledMethodNames())
doNextM, addTimerM, suspendM = trace.calledMethods
self.assertEquals(((suspend, ), {}), (doNextM.args, doNextM.kwargs))
self.assertEquals(((), {
'seconds': 3.14,
'callback': suspend._timedOut
}), (addTimerM.args, addTimerM.kwargs))
self.assertEquals(((), {}), (suspendM.args, suspendM.kwargs))
self.assertEquals('timerToken', suspend._timer)
trace.calledMethods.reset()
suspend._timedOut()
self.assertEquals(['onTimeout', 'whenDone'], trace.calledMethodNames())
onTimeoutM, whenDoneM = trace.calledMethods
self.assertEquals(((), {}), (onTimeoutM.args, onTimeoutM.kwargs))
self.assertEquals(((), {}), (whenDoneM.args, whenDoneM.kwargs))
self.assertEquals(None, suspend._timer)
self.assertRaises(TimeoutException, lambda: suspend.getResult())
def testDoNextErrorReRaisedOnGetResult(self):
def razor(ignored):
1/0 # Division by zero exception
suspend = Suspend(doNext=razor)
olderr = sys.stderr
sys.stderr = StringIO()
try:
suspend(reactor=CallTrace(), whenDone="not called")
finally:
sys.stderr = olderr
try:
suspend.getResult()
except:
exc_type, exc_value, exc_traceback = exc_info()
expectedTraceback = ignoreLineNumbers("""Traceback (most recent call last):
File "%(__file__)s", line 200, in testDoNextErrorReRaisedOnGetResult
suspend.getResult()
File "%(suspend.py)s", line 40, in __call__
self._doNext(self)
File "%(__file__)s", line 196, in razor
1/0 # Division by zero exception
ZeroDivisionError: integer division or modulo by zero
""" % fileDict)
self.assertEquals(ZeroDivisionError, exc_type)
self.assertEqualsWS(expectedTraceback, ignoreLineNumbers(format_exc(exc_traceback)))
def testSuspendProtocolWithThrow(self):
with Reactor() as reactor:
suspend = Suspend()
def handler(**httpvars):
yield 'before suspend'
yield suspend
try:
suspend.getResult()
self.fail()
except ValueError, e:
tbstring = format_exc()
yield "result = %s" % tbstring
yield 'after suspend'
listener = MyMockSocket()
port = PortNumberGenerator.next()
httpserver = HttpServer(reactor, port, handler, sok=listener)
httpserver.listen()
reactor.removeReader(listener) # avoid new connections
httpserver._acceptor._accept()
reactor.step()
reactor.step()
reactor.step()
self.assertEquals(reactor, suspend._reactor)
self.assertEquals(0, len(reactor._fds))
def raiser():
raise ValueError("BAD VALUE")
try:
raiser()
except ValueError, e:
exc_type, exc_value, exc_traceback = exc_info()
suspend.throw(exc_type, exc_value, exc_traceback)
def testSuspendProtocol(self):
with Reactor() as reactor:
suspend = Suspend()
def handler(**httpvars):
yield 'before suspend'
yield suspend
yield "result = %s" % suspend.getResult()
yield 'after suspend'
listener = MyMockSocket()
port = PortNumberGenerator.next()
httpserver = HttpServer(reactor, port, handler, sok=listener)
httpserver.listen()
reactor.removeReader(listener) # avoid new connections
httpserver._acceptor._accept()
self.assertEquals(1, len(reactor._fds))
self.assertEquals([READ_INTENT], [v.intent for v in reactor._fds.values()])
reactor.step()
self.assertEquals(1, len(reactor._fds))
self.assertEquals([WRITE_INTENT], [v.intent for v in reactor._fds.values()])
reactor.step()
reactor.step()
self.assertEquals(reactor, suspend._reactor)
self.assertEquals(0, len(reactor._fds))
suspend.resume('RESPONSE')
self.assertEquals(1, len(reactor._fds))
self.assertEquals([WRITE_INTENT], [v.intent for v in reactor._fds.values()])
reactor.step()
reactor.step()
reactor.step()
self.assertEquals(['before suspend', 'result = RESPONSE', 'after suspend'], listener.data)
self.assertEquals(0, len(reactor._fds))
# cleanup (most) fd's
listener.close()
def testSuspendProtocol(self):
reactor = Reactor(select_func=mockselect)
suspend = Suspend()
def handler(**httpvars):
yield 'before suspend'
yield suspend
yield "result = %s" % suspend.getResult()
yield 'after suspend'
listener = MyMockSocket()
port = 9
httpserver = HttpServer(reactor, port, handler, sok=listener)
httpserver.listen()
reactor.removeReader(listener) # avoid new connections
httpserver._acceptor._accept()
reactor.step()
reactor.step()
self.assertEquals(1, len(reactor._writers))
reactor.step()
self.assertEquals(reactor, suspend._reactor)
self.assertEquals(0, len(reactor._writers))
suspend.resume('RESPONSE')
reactor.step()
reactor.step()
reactor.step()
self.assertEquals(['before suspend', 'result = RESPONSE', 'after suspend'], listener.data)
def suspendWrap():
doNext, onTimeout = workInSuspend()
s = Suspend(doNext=doNext, timeout=0.01, onTimeout=onTimeout)
yield s
result = s.getResult()
if expectTimeout:
self.fail('Should not have come here')
raise StopIteration(result)
def suspendWrap():
doNext, onTimeout = workInSuspend()
s = Suspend(doNext=doNext, timeout=0.01, onTimeout=onTimeout)
yield s
result = s.getResult()
if expectTimeout:
self.fail('Should not have come here')
raise StopIteration(result)
def testSuspendThrowBackwardsCompatibleWithInstanceOnlyThrow_YouWillMissTracebackHistory(
self):
with Reactor() as reactor:
suspend = Suspend()
def handler(**httpvars):
yield 'before suspend'
yield suspend
try:
suspend.getResult()
self.fail()
except ValueError, e:
tbstring = format_exc()
yield "result = %s" % tbstring
yield 'after suspend'
listener = MyMockSocket()
port = PortNumberGenerator.next()
httpserver = HttpServer(reactor, port, handler, sok=listener)
httpserver.listen()
reactor.removeReader(listener) # avoid new connections
httpserver._acceptor._accept()
self.assertEquals(1, len(reactor._fds))
reactor.step()
reactor.step()
reactor.step()
self.assertEquals(reactor, suspend._reactor)
self.assertEquals(0, len(reactor._fds))
def raiser():
raise ValueError("BAD VALUE")
try:
raiser()
except:
exc_value = exc_info()[1]
suspend.throw(exc_value)
self.assertEquals(1, len(reactor._fds))
reactor.step()
reactor.step()
reactor.step()
expectedTraceback = ignoreLineNumbers(
"""Traceback (most recent call last):
File "%(__file__)s", line 201, in handler
suspend.getResult()
File "%(suspend.py)s", line 62, in getResult
raise self._exception[0], self._exception[1], self._exception[2]
ValueError: BAD VALUE
""" % fileDict)
self.assertEquals(3, len(listener.data))
self.assertEquals('before suspend', listener.data[0])
self.assertEqualsWS("result = %s" % expectedTraceback,
ignoreLineNumbers(listener.data[1]))
self.assertEquals('after suspend', listener.data[2])
self.assertEquals(0, len(reactor._fds))
# cleanup (most) fd's
listener.close()
def prepare():
trace = CallTrace(returnValues={'addTimer': 'timerToken'})
suspend = Suspend(doNext=trace.doNext, timeout=3.14, onTimeout=trace.onTimeout)
suspend(reactor=trace, whenDone=trace.whenDone)
self.assertEquals(['doNext', 'addTimer', 'suspend'], trace.calledMethodNames())
trace.calledMethods.reset()
suspend._timedOut()
self.assertEquals(['onTimeout', 'whenDone'], trace.calledMethodNames())
self.assertEquals(True, suspend._settled)
trace.calledMethods.reset()
return trace, suspend
def testSuspendThrowBackwardsCompatibleWithInstanceOnlyThrow_YouWillMissTracebackHistory(self):
with Reactor() as reactor:
suspend = Suspend()
def handler(**httpvars):
yield 'before suspend'
yield suspend
try:
suspend.getResult()
self.fail()
except ValueError, e:
tbstring = format_exc()
yield "result = %s" % tbstring
yield 'after suspend'
listener = MyMockSocket()
port = PortNumberGenerator.next()
httpserver = HttpServer(reactor, port, handler, sok=listener)
httpserver.listen()
reactor.removeReader(listener) # avoid new connections
httpserver._acceptor._accept()
self.assertEquals(1, len(reactor._fds))
reactor.step()
reactor.step()
reactor.step()
self.assertEquals(reactor, suspend._reactor)
self.assertEquals(0, len(reactor._fds))
def raiser():
raise ValueError("BAD VALUE")
try:
raiser()
except:
exc_value = exc_info()[1]
suspend.throw(exc_value)
self.assertEquals(1, len(reactor._fds))
reactor.step()
reactor.step()
reactor.step()
expectedTraceback = ignoreLineNumbers("""Traceback (most recent call last):
File "%(__file__)s", line 201, in handler
suspend.getResult()
File "%(suspend.py)s", line 62, in getResult
raise self._exception[0], self._exception[1], self._exception[2]
ValueError: BAD VALUE
""" % fileDict)
self.assertEquals(3, len(listener.data))
self.assertEquals('before suspend', listener.data[0])
self.assertEqualsWS("result = %s" % expectedTraceback, ignoreLineNumbers(listener.data[1]))
self.assertEquals('after suspend', listener.data[2])
self.assertEquals(0, len(reactor._fds))
# cleanup (most) fd's
listener.close()
def _suspend(self, clientIdentifier, prefix, sets, **ignored):
suspend = Suspend()
suspend.oaiListResumeMask = dict(
prefix=prefix,
sets=set() if sets is None else set(sets)
)
if clientIdentifier in self._register:
self._register.pop(clientIdentifier).throw(exc_type=ForcedResumeException, exc_value=ForcedResumeException("Aborting suspended request because of new request for the same OaiClient with identifier: %s." % clientIdentifier), exc_traceback=None)
elif len(self._register) == self._maximumSuspendedConnections:
self._register.pop(choice(self._register.keys())).throw(exc_type=ForcedResumeException, exc_value=ForcedResumeException("OAI x-wait connection has been forcefully resumed."), exc_traceback=None)
sys.stderr.write("Too many suspended connections in SuspendRegister. One random connection has been resumed.\n")
self._register[clientIdentifier] = suspend
yield suspend
suspend.getResult()
def sleep(seconds):
def doNext():
suspend = yield # from Suspend.__call__
yield # Wait for timeout
yield # wait for GC
g = doNext(); g.next() # + autostart
s = Suspend(doNext=g.send, timeout=seconds, onTimeout=g.next)
yield s
try:
s.getResult()
except TimeoutException:
pass
raise StopIteration(None)
def httprequest(host, port, request, body=None, headers=None, proxyServer=None, ssl=False, prio=None, handlePartialResponse=None, method='GET', timeout=None, maxResponseSize=None):
g = _do(method, host=host, port=port, request=request, headers=headers, proxyServer=proxyServer, body=body, ssl=ssl, prio=prio, handlePartialResponse=handlePartialResponse, maxResponseSize=maxResponseSize)
kw = {}
if timeout is not None:
def onTimeout():
g.throw(TimeoutException, TimeoutException(), None)
kw = {
'timeout': timeout,
'onTimeout': onTimeout,
}
s = Suspend(doNext=g.send, **kw)
yield s
result = s.getResult()
raise StopIteration(result)
def httprequest1_1(self, host, port, request, body=None, headers=None, secure=False, prio=None, method='GET', timeout=None):
g = _do(method=method, host=host, port=port, request=request, headers=headers, body=body, secure=secure, prio=prio, observable=self)
kw = {}
if timeout is not None:
def onTimeout():
g.throw(TimeoutException, TimeoutException(), None)
kw = {
'timeout': timeout,
'onTimeout': onTimeout,
}
s = Suspend(doNext=g.send, **kw)
yield s
result = s.getResult()
raise StopIteration(result)
def prepare():
trace = CallTrace(returnValues={'addTimer': 'timerToken'})
suspend = Suspend(doNext=trace.doNext,
timeout=3.14,
onTimeout=trace.onTimeout)
suspend(reactor=trace, whenDone=trace.whenDone)
self.assertEquals(['doNext', 'addTimer', 'suspend'],
trace.calledMethodNames())
trace.calledMethods.reset()
suspend._timedOut()
self.assertEquals(['onTimeout', 'whenDone'],
trace.calledMethodNames())
self.assertEquals(True, suspend._settled)
trace.calledMethods.reset()
return trace, suspend
def httprequest1_1(self, host, port, request, body=None, headers=None, secure=False, prio=None, method='GET', timeout=None, bodyMaxSize=None):
g = _do(method=method, host=host, port=port, request=request, headers=headers, body=body, bodyMaxSize=bodyMaxSize, secure=secure, prio=prio, observable=self)
kw = {}
if timeout is not None:
def onTimeout():
g.throw(TimeoutException, TimeoutException(), None)
kw = {
'timeout': timeout,
'onTimeout': onTimeout,
}
s = Suspend(doNext=g.send, **kw)
yield s
result = s.getResult()
raise StopIteration(result)
def testSuspendProtocol(self):
with Reactor() as reactor:
suspend = Suspend()
def handler(**httpvars):
yield 'before suspend'
yield suspend
yield "result = %s" % suspend.getResult()
yield 'after suspend'
listener = MyMockSocket()
port = PortNumberGenerator.next()
httpserver = HttpServer(reactor, port, handler, sok=listener)
httpserver.listen()
reactor.removeReader(listener) # avoid new connections
httpserver._acceptor._accept()
self.assertEquals(1, len(reactor._fds))
self.assertEquals([READ_INTENT],
[v.intent for v in reactor._fds.values()])
reactor.step()
self.assertEquals(1, len(reactor._fds))
self.assertEquals([WRITE_INTENT],
[v.intent for v in reactor._fds.values()])
reactor.step()
reactor.step()
self.assertEquals(reactor, suspend._reactor)
self.assertEquals(0, len(reactor._fds))
suspend.resume('RESPONSE')
self.assertEquals(1, len(reactor._fds))
self.assertEquals([WRITE_INTENT],
[v.intent for v in reactor._fds.values()])
reactor.step()
reactor.step()
reactor.step()
self.assertEquals(
['before suspend', 'result = RESPONSE', 'after suspend'],
listener.data)
self.assertEquals(0, len(reactor._fds))
# cleanup (most) fd's
listener.close()
def getSuspend():
trace.calledMethods.reset()
suspend = Suspend(doNext=trace.doNext)
self.assertEquals([], trace.calledMethodNames())
suspend(reactor=trace, whenDone=trace.whenDone)
self.assertEquals(['doNext', 'suspend'], trace.calledMethodNames())
doNextM, suspendM = trace.calledMethods
self.assertEquals(((suspend, ), {}),
(doNextM.args, doNextM.kwargs))
self.assertEquals(((), {}), (suspendM.args, suspendM.kwargs))
trace.calledMethods.reset()
return suspend
def httprequest(host,
port,
request,
body=None,
headers=None,
proxyServer=None,
ssl=False,
prio=None,
handlePartialResponse=None,
method='GET',
timeout=None,
maxResponseSize=None):
g = _do(method,
host=host,
port=port,
request=request,
headers=headers,
proxyServer=proxyServer,
body=body,
ssl=ssl,
prio=prio,
handlePartialResponse=handlePartialResponse,
maxResponseSize=maxResponseSize)
kw = {}
if timeout is not None:
def onTimeout():
g.throw(TimeoutException, TimeoutException(), None)
kw = {
'timeout': timeout,
'onTimeout': onTimeout,
}
s = Suspend(doNext=g.send, **kw)
yield s
result = s.getResult()
raise StopIteration(result)
def _suspend(self, clientIdentifier, prefix, sets, **ignored):
suspend = Suspend()
suspend.oaiListResumeMask = dict(
prefix=prefix, sets=set() if sets is None else set(sets))
if clientIdentifier in self._register:
self._register.pop(clientIdentifier).throw(
exc_type=ForcedResumeException,
exc_value=ForcedResumeException(
"Aborting suspended request because of new request for the same OaiClient with identifier: %s."
% clientIdentifier),
exc_traceback=None)
elif len(self._register) == self._maximumSuspendedConnections:
self._register.pop(choice(list(self._register.keys()))).throw(
exc_type=ForcedResumeException,
exc_value=ForcedResumeException(
"OAI x-wait connection has been forcefully resumed."),
exc_traceback=None)
sys.stderr.write(
"Too many suspended connections in SuspendRegister. One random connection has been resumed.\n"
)
self._register[clientIdentifier] = suspend
yield suspend
suspend.getResult()
def testYieldSuspendFromAdd(self):
observer = CallTrace()
oaiDownloadProcessor = OaiDownloadProcessor(
path="/oai",
metadataPrefix="oai_dc",
workingDirectory=self.tempdir,
xWait=False)
oaiDownloadProcessor.addObserver(observer)
suspend = Suspend()
observer.returnValues['add'] = (x for x in [suspend])
yields = list(
compose(
oaiDownloadProcessor.handle(
parse(StringIO(LISTRECORDS_RESPONSE % '')))))
self.assertEqual([suspend, None], yields)
def testSuspendTimingOut(self):
# with calltrace; happy path
trace = CallTrace(returnValues={'addTimer': 'timerToken'})
suspend = Suspend(doNext=trace.doNext, timeout=3.14, onTimeout=trace.onTimeout)
self.assertEquals([], trace.calledMethodNames())
self.assertEquals(None, suspend._timer)
suspend(reactor=trace, whenDone=trace.whenDone)
self.assertEquals(['doNext', 'addTimer', 'suspend'], trace.calledMethodNames())
doNextM, addTimerM, suspendM = trace.calledMethods
self.assertEquals(((suspend,), {}), (doNextM.args, doNextM.kwargs))
self.assertEquals(((), {'seconds': 3.14, 'callback': suspend._timedOut}), (addTimerM.args, addTimerM.kwargs))
self.assertEquals(((), {}), (suspendM.args, suspendM.kwargs))
self.assertEquals('timerToken', suspend._timer)
trace.calledMethods.reset()
suspend._timedOut()
self.assertEquals(['onTimeout', 'whenDone'], trace.calledMethodNames())
onTimeoutM, whenDoneM = trace.calledMethods
self.assertEquals(((), {}), (onTimeoutM.args, onTimeoutM.kwargs))
self.assertEquals(((), {}), (whenDoneM.args, whenDoneM.kwargs))
self.assertEquals(None, suspend._timer)
self.assertRaises(TimeoutException, lambda: suspend.getResult())
def prepare(_onTimeout):
# Use indirect tracing of onTimeout call (otherwise CallTrace internals leak into the stacktrace).
trace = CallTrace(returnValues={'addTimer': 'timerToken'})
onTimeoutM = trace.onTimeout
def onTimeoutTraced():
onTimeoutM() # We have been called Watson!
return _onTimeout()
trace.onTimeout = onTimeoutTraced
suspend = Suspend(doNext=trace.doNext,
timeout=3.14,
onTimeout=trace.onTimeout)
suspend(reactor=trace, whenDone=trace.whenDone)
self.assertEquals(['doNext', 'addTimer', 'suspend'],
trace.calledMethodNames())
trace.calledMethods.reset()
return trace, suspend
def prepare():
trace = CallTrace(returnValues={'addTimer': 'timerToken'})
suspend = Suspend(doNext=trace.doNext,
timeout=3.14,
onTimeout=trace.onTimeout)
self.assertEquals([], trace.calledMethodNames())
suspend(reactor=trace, whenDone=trace.whenDone)
self.assertEquals(['doNext', 'addTimer', 'suspend'],
trace.calledMethodNames())
doNextM, addTimerM, suspendM = trace.calledMethods
self.assertEquals(((suspend, ), {}),
(doNextM.args, doNextM.kwargs))
self.assertEquals(((), {
'seconds': 3.14,
'callback': suspend._timedOut
}), (addTimerM.args, addTimerM.kwargs))
self.assertEquals(((), {}), (suspendM.args, suspendM.kwargs))
trace.calledMethods.reset()
return trace, suspend
def httppost(**_kwargs):
kwargs.append(_kwargs)
s = Suspend()
yield s
result = s.getResult()
raise StopIteration(result)
def testGetNoneResult(self):
reactor = CallTrace('reactor')
s = Suspend()
s(reactor, whenDone=lambda:None)
s.resume()
self.assertEquals(None, s.getResult())
def testGetResultRaisesException(self):
reactor = CallTrace('reactor')
s = Suspend()
s(reactor, whenDone=lambda:None)
s.throw(ValueError('bad value'))
self.assertRaises(ValueError, s.getResult)
def suspending():
s = Suspend(whileSuspended().send)
yield s
raise StopIteration(s.getResult())
def httpget(host, port, request, headers=None):
s = Suspend(_do('GET', host, port, request, headers=headers).send)
yield s
result = s.getResult()
raise StopIteration(result)
def suspending():
s = Suspend(whileSuspended().send)
yield s
raise StopIteration(s.getResult())
def testSuspendTimeoutArguments(self):
self.assertRaises(ValueError, lambda: Suspend(timeout=3))
self.assertRaises(ValueError, lambda: Suspend(onTimeout=lambda: None))
Suspend()
Suspend(timeout=3, onTimeout=lambda: None)
def suspendWithDoNextException(exception):
def doNextRaising(suspend):
raise exception
suspend = Suspend(doNext=doNextRaising)
suspend(reactor=CallTrace(), whenDone="not called")
def testGetResultRaisesException(self):
reactor = CallTrace('reactor')
s = Suspend()
s(reactor, whenDone=lambda: None)
s.throw(ValueError('bad value'))
self.assertRaises(ValueError, s.getResult)
def testGetNoneResult(self):
reactor = CallTrace('reactor')
s = Suspend()
s(reactor, whenDone=lambda: None)
s.resume()
self.assertEquals(None, s.getResult())
