class CallFromThreadTestCase(unittest.TestCase):
"""Task scheduling from threads tests."""
if interfaces.IReactorThreads(reactor, None) is None:
skip = "Nothing to test without thread support"
def schedule(self, *args, **kwargs):
"""Override in subclasses."""
reactor.callFromThread(*args, **kwargs)
def testScheduling(self):
c = Counter()
for i in range(100):
self.schedule(c.add)
for i in range(100):
reactor.iterate()
self.assertEquals(c.index, 100)
def testCorrectOrder(self):
o = Order()
self.schedule(o.a)
self.schedule(o.b)
self.schedule(o.c)
reactor.iterate()
reactor.iterate()
reactor.iterate()
self.assertEquals(o.stage, 3)
def testNotRunAtOnce(self):
c = Counter()
self.schedule(c.add)
# scheduled tasks should not be run at once:
self.assertEquals(c.index, 0)
reactor.iterate()
self.assertEquals(c.index, 1)
class WaitReentrancyTest(unittest.TestCase):
if interfaces.IReactorThreads(reactor, None) is None:
skip = ("This test probably doesn't really need threads "
"but hell if I can figure out how to rewrite it "
"without them. Skipping in the absence of "
"thread-support.")
def _returnedDeferredThenWait(self):
def threadedOperation():
time.sleep(0.1)
return "Beginning"
d = threads.deferToThread(threadedOperation)
return d.addCallback(self._cbDoWait)
def _cbDoWait(self, result):
self.assertEquals(result, "Beginning")
d = defer.succeed("End")
self.assertEquals(util.wait(d), "End")
def testReturnedDeferredThenWait(self):
d = self._returnedDeferredThenWait()
self.assertRaises(util.WaitIsNotReentrantError, util.wait, d)
def _reentrantWait(self):
def threadedOperation(n):
time.sleep(n)
return n
d1 = threads.deferToThread(threadedOperation, 0.125)
d2 = threads.deferToThread(threadedOperation, 0.250)
d1.addCallback(lambda ignored: util.wait(d2))
util.wait(d1)
def testReentrantWait(self):
self.assertRaises(util.WaitIsNotReentrantError, self._reentrantWait)
def test_twoWaitImplementations(self):
# If this test times out, then wait is being re-entered.
tc = TestMktemp('test_name')
tc._timedOut = False # whitebox
d = defer.Deferred()
def _runsInsideWait(r):
d = defer.Deferred()
self.assertRaises(util.WaitIsNotReentrantError, util.wait, d)
d.addCallback(utils.suppressWarnings(_runsInsideWait, *suppress))
reactor.callLater(0, d.callback, 'yo')
tc._wait(d)
test_twoWaitImplementations.timeout = 4
class TimeoutQueueTest(unittest.TestCase):
"""
Test L{timeoutqueue.TimeoutQueue} class.
"""
def tearDown(self):
del self.q
def put(self):
time.sleep(1)
self.q.put(1)
def test_timeout(self):
q = self.q = timeoutqueue.TimeoutQueue()
try:
q.wait(1)
except timeoutqueue.TimedOut:
pass
else:
self.fail("Didn't time out")
test_timeout.suppress = [timeoutqueueSuppression]
def test_get(self):
q = self.q = timeoutqueue.TimeoutQueue()
start = time.time()
threading.Thread(target=self.put).start()
q.wait(1.5)
assert time.time() - start < 2
result = q.get(0)
if result != 1:
self.fail("Didn't get item we put in")
test_get.suppress = [timeoutqueueSuppression]
def test_deprecation(self):
"""
Test that L{timeoutqueue.TimeoutQueue} prints a warning message.
"""
def createQueue():
return timeoutqueue.TimeoutQueue()
self.q = self.assertWarns(
DeprecationWarning, "timeoutqueue is deprecated since Twisted 8.0",
__file__, createQueue)
if interfaces.IReactorThreads(reactor, None) is None:
test_get.skip = "No thread support, no way to test putting during a blocked get"
else:
global threading
import threading
class ReconnectTestBase:
"""Test the asynchronous DB-API code with reconnect."""
if interfaces.IReactorThreads(reactor, None) is None:
skip = "ADB-API requires threads, no way to test without them"
def wait(self, d, timeout=10.0):
return unittest.wait(d, timeout=timeout)
def setUp(self):
if self.good_sql is None:
raise unittest.SkipTest('no good sql for reconnect test')
self.startDB()
self.dbpool = self.makePool(cp_max=1,
cp_reconnect=True,
cp_good_sql=self.good_sql)
self.dbpool.start()
self.wait(self.dbpool.runOperation(simple_table_schema))
def tearDown(self):
self.wait(self.dbpool.runOperation('DROP TABLE simple'))
self.dbpool.close()
self.stopDB()
def testPool(self):
sql = "select count(1) from simple"
row = self.wait(self.dbpool.runQuery(sql))
self.failUnless(int(row[0][0]) == 0, "Table not empty")
# reach in and close the connection manually
self.dbpool.connections.values()[0].close()
if not self.early_reconnect:
try:
self.wait(self.dbpool.runQuery(sql))
self.fail('no exception')
except ConnectionLost:
pass
except:
self.fail('not connection lost')
row = self.wait(self.dbpool.runQuery(sql))
self.failUnless(int(row[0][0]) == 0, "Table not empty")
sql = "select * from NOTABLE" # bad sql
try:
self.wait(self.dbpool.runQuery(sql))
self.fail('no exception')
except ConnectionLost:
self.fail('connection lost exception')
except:
pass
def _bail(self):
from twisted.internet import reactor
d = defer.Deferred()
reactor.addSystemEventTrigger('after', 'shutdown',
lambda: d.callback(None))
reactor.fireSystemEvent('shutdown') # radix's suggestion
treactor = interfaces.IReactorThreads(reactor, None)
if treactor is not None:
treactor.suggestThreadPoolSize(0)
# As long as TestCase does crap stuff with the reactor we need to
# manually shutdown the reactor here, and that requires util.wait
# :(
# so that the shutdown event completes
unittest.TestCase('mktemp')._wait(d)
class CallFromThreadStopsAndWakeUpTests(TestCase):
@skipIf(
not interfaces.IReactorThreads(reactor, None),
"Nothing to wake up for without thread support",
)
def testWakeUp(self):
# Make sure other threads can wake up the reactor
d = Deferred()
def wake():
time.sleep(0.1)
# callFromThread will call wakeUp for us
reactor.callFromThread(d.callback, None)
reactor.callInThread(wake)
return d
def _stopCallFromThreadCallback(self):
self.stopped = True
def _callFromThreadCallback(self, d):
reactor.callFromThread(self._callFromThreadCallback2, d)
reactor.callLater(0, self._stopCallFromThreadCallback)
def _callFromThreadCallback2(self, d):
try:
self.assertTrue(self.stopped)
except:
# Send the error to the deferred
d.errback()
else:
d.callback(None)
def testCallFromThreadStops(self):
"""
Ensure that callFromThread from inside a callFromThread
callback doesn't sit in an infinite loop and lets other
things happen too.
"""
self.stopped = False
d = defer.Deferred()
reactor.callFromThread(self._callFromThreadCallback, d)
return d
class TimeoutQueueTest(unittest.TestCase):
def setUp(self):
self.q = timeoutqueue.TimeoutQueue()
def tearDown(self):
del self.q
def put(self):
time.sleep(1)
self.q.put(1)
def testTimeout(self):
q = self.q
try:
q.wait(1)
except timeoutqueue.TimedOut:
pass
else:
raise AssertionError, "didn't time out"
def testGet(self):
q = self.q
start = time.time()
threading.Thread(target=self.put).start()
q.wait(1.5)
assert time.time() - start < 2
result = q.get(0)
if result != 1:
raise AssertionError, "didn't get item we put in"
if interfaces.IReactorThreads(reactor, None) is None:
testGet.skip = "No thread support, no way to test putting during a blocked get"
else:
global threading
import threading
class ReconnectTestBase:
"""Test the asynchronous DB-API code with reconnect."""
if interfaces.IReactorThreads(reactor, None) is None:
skip = "ADB-API requires threads, no way to test without them"
def setUp(self):
if self.good_sql is None:
raise unittest.SkipTest('no good sql for reconnect test')
self.startDB()
self.dbpool = self.makePool(cp_max=1,
cp_reconnect=True,
cp_good_sql=self.good_sql)
self.dbpool.start()
return self.dbpool.runOperation(simple_table_schema)
def tearDown(self):
d = self.dbpool.runOperation('DROP TABLE simple')
d.addCallback(lambda res: self.dbpool.close())
d.addCallback(lambda res: self.stopDB())
return d
def testPool(self):
d = defer.succeed(None)
d.addCallback(self._testPool_1)
d.addCallback(self._testPool_2)
if not self.early_reconnect:
d.addCallback(self._testPool_3)
d.addCallback(self._testPool_4)
d.addCallback(self._testPool_5)
return d
def _testPool_1(self, res):
sql = "select count(1) from simple"
d = self.dbpool.runQuery(sql)
def _check(row):
self.failUnless(int(row[0][0]) == 0, "Table not empty")
d.addCallback(_check)
return d
def _testPool_2(self, res):
# reach in and close the connection manually
self.dbpool.connections.values()[0].close()
def _testPool_3(self, res):
sql = "select count(1) from simple"
d = defer.maybeDeferred(self.dbpool.runQuery, sql)
d.addCallbacks(lambda res: self.fail('no exception'),
lambda f: f.trap(ConnectionLost))
return d
def _testPool_4(self, res):
sql = "select count(1) from simple"
d = self.dbpool.runQuery(sql)
def _check(row):
self.failUnless(int(row[0][0]) == 0, "Table not empty")
d.addCallback(_check)
return d
def _testPool_5(self, res):
sql = "select * from NOTABLE" # bad sql
d = defer.maybeDeferred(self.dbpool.runQuery, sql)
d.addCallbacks(lambda res: self.fail('no exception'),
lambda f: self.failIf(f.check(ConnectionLost)))
return d
class ADBAPITestBase:
"""Test the asynchronous DB-API code."""
openfun_called = {}
if interfaces.IReactorThreads(reactor, None) is None:
skip = "ADB-API requires threads, no way to test without them"
def setUp(self):
self.startDB()
self.dbpool = self.makePool(cp_openfun=self.openfun)
self.dbpool.start()
def tearDown(self):
d = self.dbpool.runOperation('DROP TABLE simple')
d.addCallback(lambda res: self.dbpool.close())
d.addCallback(lambda res: self.stopDB())
return d
def openfun(self, conn):
self.openfun_called[conn] = True
def checkOpenfunCalled(self, conn=None):
if not conn:
self.failUnless(self.openfun_called)
else:
self.failUnless(self.openfun_called.has_key(conn))
def testPool(self):
d = self.dbpool.runOperation(simple_table_schema)
if self.test_failures:
d.addCallback(self._testPool_1_1)
d.addCallback(self._testPool_1_2)
d.addCallback(self._testPool_1_3)
d.addCallback(self._testPool_1_4)
d.addCallback(lambda res: log.flushErrors())
d.addCallback(self._testPool_2)
d.addCallback(self._testPool_3)
d.addCallback(self._testPool_4)
d.addCallback(self._testPool_5)
d.addCallback(self._testPool_6)
d.addCallback(self._testPool_7)
d.addCallback(self._testPool_8)
d.addCallback(self._testPool_9)
return d
def _testPool_1_1(self, res):
d = defer.maybeDeferred(self.dbpool.runQuery, "select * from NOTABLE")
d.addCallbacks(lambda res: self.fail('no exception'), lambda f: None)
return d
def _testPool_1_2(self, res):
d = defer.maybeDeferred(self.dbpool.runOperation,
"deletexxx from NOTABLE")
d.addCallbacks(lambda res: self.fail('no exception'), lambda f: None)
return d
def _testPool_1_3(self, res):
d = defer.maybeDeferred(self.dbpool.runInteraction,
self.bad_interaction)
d.addCallbacks(lambda res: self.fail('no exception'), lambda f: None)
return d
def _testPool_1_4(self, res):
d = defer.maybeDeferred(self.dbpool.runWithConnection,
self.bad_withConnection)
d.addCallbacks(lambda res: self.fail('no exception'), lambda f: None)
return d
def _testPool_2(self, res):
# verify simple table is empty
sql = "select count(1) from simple"
d = self.dbpool.runQuery(sql)
def _check(row):
self.failUnless(int(row[0][0]) == 0, "Interaction not rolled back")
self.checkOpenfunCalled()
d.addCallback(_check)
return d
def _testPool_3(self, res):
sql = "select count(1) from simple"
inserts = []
# add some rows to simple table (runOperation)
for i in range(self.num_iterations):
sql = "insert into simple(x) values(%d)" % i
inserts.append(self.dbpool.runOperation(sql))
d = defer.gatherResults(inserts)
def _select(res):
# make sure they were added (runQuery)
sql = "select x from simple order by x"
d = self.dbpool.runQuery(sql)
return d
d.addCallback(_select)
def _check(rows):
self.failUnless(
len(rows) == self.num_iterations, "Wrong number of rows")
for i in range(self.num_iterations):
self.failUnless(len(rows[i]) == 1, "Wrong size row")
self.failUnless(rows[i][0] == i, "Values not returned.")
d.addCallback(_check)
return d
def _testPool_4(self, res):
# runInteraction
d = self.dbpool.runInteraction(self.interaction)
d.addCallback(lambda res: self.assertEquals(res, "done"))
return d
def _testPool_5(self, res):
# withConnection
d = self.dbpool.runWithConnection(self.withConnection)
d.addCallback(lambda res: self.assertEquals(res, "done"))
return d
def _testPool_6(self, res):
# Test a withConnection cannot be closed
d = self.dbpool.runWithConnection(self.close_withConnection)
return d
def _testPool_7(self, res):
# give the pool a workout
ds = []
for i in range(self.num_iterations):
sql = "select x from simple where x = %d" % i
ds.append(self.dbpool.runQuery(sql))
dlist = defer.DeferredList(ds, fireOnOneErrback=True)
def _check(result):
for i in range(self.num_iterations):
self.failUnless(result[i][1][0][0] == i, "Value not returned")
dlist.addCallback(_check)
return dlist
def _testPool_8(self, res):
# now delete everything
ds = []
for i in range(self.num_iterations):
sql = "delete from simple where x = %d" % i
ds.append(self.dbpool.runOperation(sql))
dlist = defer.DeferredList(ds, fireOnOneErrback=True)
return dlist
def _testPool_9(self, res):
# verify simple table is empty
sql = "select count(1) from simple"
d = self.dbpool.runQuery(sql)
def _check(row):
self.failUnless(
int(row[0][0]) == 0,
"Didn't successfully delete table contents")
self.checkConnect()
d.addCallback(_check)
return d
def checkConnect(self):
"""Check the connect/disconnect synchronous calls."""
conn = self.dbpool.connect()
self.checkOpenfunCalled(conn)
curs = conn.cursor()
curs.execute("insert into simple(x) values(1)")
curs.execute("select x from simple")
res = curs.fetchall()
self.failUnlessEqual(len(res), 1)
self.failUnlessEqual(len(res[0]), 1)
self.failUnlessEqual(res[0][0], 1)
curs.execute("delete from simple")
curs.execute("select x from simple")
self.failUnlessEqual(len(curs.fetchall()), 0)
curs.close()
self.dbpool.disconnect(conn)
def interaction(self, transaction):
transaction.execute("select x from simple order by x")
for i in range(self.num_iterations):
row = transaction.fetchone()
self.failUnless(len(row) == 1, "Wrong size row")
self.failUnless(row[0] == i, "Value not returned.")
# should test this, but gadfly throws an exception instead
#self.failUnless(transaction.fetchone() is None, "Too many rows")
return "done"
def bad_interaction(self, transaction):
if self.can_rollback:
transaction.execute("insert into simple(x) values(0)")
transaction.execute("select * from NOTABLE")
def withConnection(self, conn):
curs = conn.cursor()
try:
curs.execute("select x from simple order by x")
for i in range(self.num_iterations):
row = curs.fetchone()
self.failUnless(len(row) == 1, "Wrong size row")
self.failUnless(row[0] == i, "Value not returned.")
# should test this, but gadfly throws an exception instead
#self.failUnless(transaction.fetchone() is None, "Too many rows")
finally:
curs.close()
return "done"
def close_withConnection(self, conn):
conn.close()
def bad_withConnection(self, conn):
curs = conn.cursor()
try:
curs.execute("select * from NOTABLE")
finally:
curs.close()
self.fail(
(
"The child process failed to produce "
"the desired results:\n"
" Reason for termination was: {!r}\n"
" Output stream was: {!r}\n"
" Error stream was: {!r}\n"
).format(reason.getErrorMessage(), output, b"".join(error))
)
helperDeferred.addCallback(cbFinished)
return helperDeferred
@skipIf(
not interfaces.IReactorThreads(reactor, None),
"Nothing to test without thread support",
)
class CallFromThreadTests(TestCase):
"""
Task scheduling from threads tests.
"""
def setUp(self):
self.counter = 0
self.deferred = Deferred()
def schedule(self, *args, **kwargs):
"""
Override in subclasses.
"""
class ReconnectTestBase:
"""
Test the asynchronous DB-API code with reconnect.
"""
if interfaces.IReactorThreads(reactor, None) is None:
skip = "ADB-API requires threads, no way to test without them"
def extraSetUp(self):
"""
Skip the test if C{good_sql} is unavailable. Otherwise, set up the
database, create a connection pool pointed at it, and set up a simple
schema in it.
"""
if self.good_sql is None:
raise unittest.SkipTest("no good sql for reconnect test")
self.startDB()
self.dbpool = self.makePool(cp_max=1,
cp_reconnect=True,
cp_good_sql=self.good_sql)
self.dbpool.start()
return self.dbpool.runOperation(simple_table_schema)
def tearDown(self):
d = self.dbpool.runOperation("DROP TABLE simple")
d.addCallback(lambda res: self.dbpool.close())
d.addCallback(lambda res: self.stopDB())
return d
def test_pool(self):
d = defer.succeed(None)
d.addCallback(self._testPool_1)
d.addCallback(self._testPool_2)
if not self.early_reconnect:
d.addCallback(self._testPool_3)
d.addCallback(self._testPool_4)
d.addCallback(self._testPool_5)
return d
def _testPool_1(self, res):
sql = "select count(1) from simple"
d = self.dbpool.runQuery(sql)
def _check(row):
self.assertTrue(int(row[0][0]) == 0, "Table not empty")
d.addCallback(_check)
return d
def _testPool_2(self, res):
# reach in and close the connection manually
list(self.dbpool.connections.values())[0].close()
def _testPool_3(self, res):
sql = "select count(1) from simple"
d = defer.maybeDeferred(self.dbpool.runQuery, sql)
d.addCallbacks(lambda res: self.fail("no exception"), lambda f: None)
return d
def _testPool_4(self, res):
sql = "select count(1) from simple"
d = self.dbpool.runQuery(sql)
def _check(row):
self.assertTrue(int(row[0][0]) == 0, "Table not empty")
d.addCallback(_check)
return d
def _testPool_5(self, res):
self.flushLoggedErrors()
sql = "select * from NOTABLE" # bad sql
d = defer.maybeDeferred(self.dbpool.runQuery, sql)
d.addCallbacks(
lambda res: self.fail("no exception"),
lambda f: self.assertFalse(f.check(ConnectionLost)),
)
return d
class InterfaceTestCase(unittest.TestCase):
_called = 0
def _callback(self, deferred, x, **d):
"""Callback for testCallLater"""
self.assertEquals(x, 1)
self.assertEquals(d, {'a': 1})
self._called = 1
self._calledTime = time.time()
deferred.callback(None)
def testCallLater(self):
# add and remove a callback
def bad():
raise RuntimeError, "this shouldn't have been called"
i = reactor.callLater(0.1, bad)
i.cancel()
self.assertRaises(error.AlreadyCancelled, i.cancel)
d = defer.Deferred()
i = reactor.callLater(0.5, self._callback, d, 1, a=1)
start = time.time()
def check(ignored):
self.assertApproximates(self._calledTime, start + 0.5, 0.2)
self.assertRaises(error.AlreadyCalled, i.cancel)
del self._called
del self._calledTime
d.addCallback(check)
return d
def testCallLaterDelayAndReset(self):
clock = Clock()
clock.install()
try:
callbackTimes = [None, None]
def resetCallback():
callbackTimes[0] = clock()
def delayCallback():
callbackTimes[1] = clock()
ireset = reactor.callLater(2, resetCallback)
idelay = reactor.callLater(3, delayCallback)
clock.pump(reactor, [0, 1])
ireset.reset(2) # (now)1 + 2 = 3
idelay.delay(3) # (orig)3 + 3 = 6
clock.pump(reactor, [0, 1])
self.assertIdentical(callbackTimes[0], None)
self.assertIdentical(callbackTimes[0], None)
clock.pump(reactor, [0, 1])
self.assertEquals(callbackTimes[0], 3)
self.assertEquals(callbackTimes[1], None)
clock.pump(reactor, [0, 3])
self.assertEquals(callbackTimes[1], 6)
finally:
clock.uninstall()
def testCallLaterTime(self):
d = reactor.callLater(10, lambda: None)
try:
self.failUnless(d.getTime() - (time.time() + 10) < 1)
finally:
d.cancel()
def testCallInNextIteration(self):
calls = []
def f1():
calls.append('f1')
reactor.callLater(0.0, f2)
def f2():
calls.append('f2')
reactor.callLater(0.0, f3)
def f3():
calls.append('f3')
reactor.callLater(0, f1)
self.assertEquals(calls, [])
reactor.iterate()
self.assertEquals(calls, ['f1'])
reactor.iterate()
self.assertEquals(calls, ['f1', 'f2'])
reactor.iterate()
self.assertEquals(calls, ['f1', 'f2', 'f3'])
def testCallLaterOrder(self):
l = []
l2 = []
def f(x):
l.append(x)
def f2(x):
l2.append(x)
def done():
self.assertEquals(l, range(20))
def done2():
self.assertEquals(l2, range(10))
for n in range(10):
reactor.callLater(0, f, n)
for n in range(10):
reactor.callLater(0, f, n + 10)
reactor.callLater(0.1, f2, n)
reactor.callLater(0, done)
reactor.callLater(0.1, done2)
d = Deferred()
reactor.callLater(0.2, d.callback, None)
return d
testCallLaterOrder.todo = "See bug 1396"
testCallLaterOrder.skip = "Trial bug, todo doesn't work! See bug 1397"
def testCallLaterOrder2(self):
# This time destroy the clock resolution so that it fails reliably
# even on systems that don't have a crappy clock resolution.
def seconds():
return int(time.time())
from twisted.internet import base
from twisted.python import runtime
base_original = base.seconds
runtime_original = runtime.seconds
base.seconds = seconds
runtime.seconds = seconds
def cleanup(x):
runtime.seconds = runtime_original
base.seconds = base_original
return x
return maybeDeferred(self.testCallLaterOrder).addBoth(cleanup)
testCallLaterOrder2.todo = "See bug 1396"
testCallLaterOrder2.skip = "Trial bug, todo doesn't work! See bug 1397"
def testDelayedCallStringification(self):
# Mostly just make sure str() isn't going to raise anything for
# DelayedCalls within reason.
dc = reactor.callLater(0, lambda x, y: None, 'x', y=10)
str(dc)
dc.reset(5)
str(dc)
dc.cancel()
str(dc)
dc = reactor.callLater(0,
lambda: None,
x=[({
'hello': u'world'
}, 10j), reactor],
*range(10))
str(dc)
dc.cancel()
str(dc)
def calledBack(ignored):
str(dc)
d = Deferred().addCallback(calledBack)
dc = reactor.callLater(0, d.callback, None)
str(dc)
return d
def testDelayedCallSecondsOverride(self):
"""
Test that the C{seconds} argument to DelayedCall gets used instead of
the default timing function, if it is not None.
"""
def seconds():
return 10
dc = base.DelayedCall(5, lambda: None, (), {}, lambda dc: None,
lambda dc: None, seconds)
self.assertEquals(dc.getTime(), 5)
dc.reset(3)
self.assertEquals(dc.getTime(), 13)
def testWakeUp(self):
# Make sure other threads can wake up the reactor
d = Deferred()
def wake():
time.sleep(0.1)
# callFromThread will call wakeUp for us
reactor.callFromThread(d.callback, None)
reactor.callInThread(wake)
return d
if interfaces.IReactorThreads(reactor, None) is None:
testWakeUp.skip = "Nothing to wake up for without thread support"
class ThreadedFlowTest(unittest.TestCase):
if interfaces.IReactorThreads(reactor, None) is None:
skip = ("No thread support in reactor, "
"cannot test threaded flow constructs.")
def testThreaded(self):
expect = [5,4,3,2,1]
d = flow.Deferred(Threaded(CountIterator(5)))
d.addCallback(self.assertEquals, expect)
return d
def testThreadedError(self):
# is this the expected behaviour?
def iterator():
yield 1
raise ValueError
d = flow.Deferred(Threaded(iterator()))
return self.assertFailure(d, ValueError)
def testThreadedSleep(self):
expect = [5,4,3,2,1]
d = flow.Deferred(Threaded(CountIterator(5)))
sleep(.5)
d.addCallback(self.assertEquals, expect)
return d
def testQueryIterator(self):
try:
from pyPgSQL import PgSQL
dbpool = PgSQL
c = dbpool.connect()
r = c.cursor()
r.execute("SELECT 'x'")
r.fetchone()
except:
# PostgreSQL is not installed or bad permissions
return
expect = [['one'],['two'],['three']]
sql = """
(SELECT 'one')
UNION ALL
(SELECT 'two')
UNION ALL
(SELECT 'three')
"""
d = flow.Deferred(Threaded(QueryIterator(dbpool, sql)))
d.addCallback(self.assertEquals, expect)
return d
def testThreadedImmediate(self):
"""
The goal of this test is to test the callback mechanism with
regard to threads, namely to assure that results can be
accumulated before they are needed; and that left-over results
are immediately made available on the next round (even though
the producing thread has shut down). This is a very tough thing
to test due to the timing issues. So it may fail on some
platforms, I'm not sure.
"""
expect = [5,4,3,2,1]
result = []
f = Threaded(CountIterator(5))
d = defer.Deferred()
def process():
coop = f._yield()
if f.results:
result.extend(f.results)
del f.results[:len(result)]
reactor.callLater(0, process)
return
if coop:
sleep(.3)
reactor.callLater(0, coop.callLater, process)
return
if f.stop:
reactor.callLater(0, d.callback, result)
reactor.callLater(0, process)
d.addCallback(self.assertEquals, expect)
return d
class ReflectorTestBase:
"""
Base class for testing reflectors.
@ivar reflector: The reflector created during setup.
"""
if interfaces.IReactorThreads(reactor, None) is None:
skip = "No thread support, no reflector tests"
count = 100 # a parameter used for running iterative tests
def randomizeRow(self, row):
return randomizeRow(row, self.nulls_ok, self.trailing_spaces_ok)
def extraSetUp(self):
"""
Create and store a reference to a SQL reflector for use by the tests.
"""
d = self.createReflector()
d.addCallback(self._cbSetUp)
return d
def _cbSetUp(self, reflector):
self.reflector = reflector
def tearDown(self):
return self.destroyReflector()
def destroyReflector(self):
pass
def test_reflector(self):
"""
Full featured tests of reflector.
"""
# create one row to work with
row = TestRow()
row.assignKeyAttr("key_string", "first")
values = self.randomizeRow(row)
# save it
d = self.reflector.insertRow(row)
def _loadBack(_):
# now load it back in
whereClause = [("key_string", EQUAL, "first")]
d = self.reflector.loadObjectsFrom(tableName,
whereClause=whereClause)
return d.addCallback(self.gotData)
def _getParent(_):
# make sure it came back as what we saved
self.failUnless(len(self.data) == 1, "no row")
parent = self.data[0]
self.failUnless(rowMatches(parent, values), "no match")
return parent
d.addCallback(_loadBack)
d.addCallback(_getParent)
d.addCallback(self._cbTestReflector)
return d
test_reflector.suppress = [rowObjectSuppression, reflectorSuppression]
def _cbTestReflector(self, parent):
# create some child rows
test_values = {}
inserts = []
child_values = {}
for i in range(0, self.num_iterations):
row = ChildRow()
row.assignKeyAttr("childId", i)
values = self.randomizeRow(row)
values['test_key'] = row.test_key = "first"
child_values[i] = values
inserts.append(self.reflector.insertRow(row))
row = None
#del inserts
d = defer.gatherResults(inserts)
values = [None]
def _loadObjects(_):
d = self.reflector.loadObjectsFrom(childTableName, parentRow=parent)
return d.addCallback(self.gotData)
def _checkLoadObjects(_):
self.failUnless(len(self.data) == self.num_iterations,
"no rows on query")
self.failUnless(len(parent.childRows) == self.num_iterations,
"did not load child rows: %d" % len(parent.childRows))
for child in parent.childRows:
self.failUnless(rowMatches(child, child_values[child.childId]),
"child %d does not match" % child.childId)
def _checkLoadObjects2(_):
self.failUnless(len(self.data) == self.num_iterations,
"no rows on query")
self.failUnless(len(parent.childRows) == self.num_iterations,
"child rows added twice!: %d" % len(parent.childRows))
def _changeParent(_):
# now change the parent
values[0] = self.randomizeRow(parent)
return self.reflector.updateRow(parent)
def _loadBack(_):
# now load it back in
whereClause = [("key_string", EQUAL, "first")]
d = self.reflector.loadObjectsFrom(tableName, whereClause=whereClause)
return d.addCallback(self.gotData)
def _checkLoadBack(_):
# make sure it came back as what we saved
self.failUnless(len(self.data) == 1, "no row")
parent = self.data[0]
self.failUnless(rowMatches(parent, values[0]), "no match")
# save parent
test_values[parent.key_string] = values[0]
parent = None
def _saveMoreTestRows(_):
# save some more test rows
ds = []
for i in range(0, self.num_iterations):
row = TestRow()
row.assignKeyAttr("key_string", "bulk%d"%i)
test_values[row.key_string] = self.randomizeRow(row)
ds.append(self.reflector.insertRow(row))
return defer.gatherResults(ds)
def _loadRowsBack(_):
# now load them all back in
d = self.reflector.loadObjectsFrom("testTable")
return d.addCallback(self.gotData)
def _checkRowsBack(_):
# make sure they are the same
self.failUnless(len(self.data) == self.num_iterations + 1,
"query did not get rows")
for row in self.data:
self.failUnless(rowMatches(row, test_values[row.key_string]),
"child %s does not match" % row.key_string)
def _changeRows(_):
# now change them all
ds = []
for row in self.data:
test_values[row.key_string] = self.randomizeRow(row)
ds.append(self.reflector.updateRow(row))
d = defer.gatherResults(ds)
return d.addCallback(_cbChangeRows)
def _cbChangeRows(_):
self.data = None
def _deleteRows(_):
# now delete them
ds = []
for row in self.data:
ds.append(self.reflector.deleteRow(row))
d = defer.gatherResults(ds)
return d.addCallback(_cbChangeRows)
def _checkRowsDeleted(_):
self.failUnless(len(self.data) == 0, "rows were not deleted")
d.addCallback(_loadObjects)
d.addCallback(_checkLoadObjects)
d.addCallback(_loadObjects)
d.addCallback(_checkLoadObjects2)
d.addCallback(_changeParent)
d.addCallback(_loadBack)
d.addCallback(_checkLoadBack)
d.addCallback(_saveMoreTestRows)
d.addCallback(_loadRowsBack)
d.addCallback(_checkRowsBack)
d.addCallback(_changeRows)
d.addCallback(_loadRowsBack)
d.addCallback(_checkRowsBack)
d.addCallback(_deleteRows)
d.addCallback(_loadRowsBack)
d.addCallback(_checkRowsDeleted)
return d
def test_saveAndDelete(self):
"""
Create a row and then try to delete it.
"""
# create one row to work with
row = TestRow()
row.assignKeyAttr("key_string", "first")
values = self.randomizeRow(row)
# save it
d = self.reflector.insertRow(row)
def _deleteRow(_):
# delete it
return self.reflector.deleteRow(row)
d.addCallback(_deleteRow)
return d
test_saveAndDelete.suppress = [rowObjectSuppression, reflectorSuppression]
def gotData(self, data):
self.data = data
from __future__ import generators
import time
from OPSI.web2.test.test_server import BaseCase
from OPSI.web2 import resource
from twisted.internet import reactor, interfaces
from twisted.python import log
if interfaces.IReactorThreads(reactor, None) is not None:
from OPSI.web2.wsgi import WSGIResource as WSGI
else:
WSGI = None
class TestError(Exception):
pass
class TestContainer(BaseCase):
wait_timeout = 10.0
def flushErrors(self, result, error):
log.flushErrors(error)
return result
def test_getContainedResource(self):
"""Test that non-blocking WSGI applications render properly."""
def application(environ, start_response):
status = '200 OK'
response_headers = [('Content-type', 'text/html')]
writer = start_response(status, response_headers)
class CallFromThreadTests(unittest.TestCase):
"""
Task scheduling from threads tests.
"""
if interfaces.IReactorThreads(reactor, None) is None:
skip = "Nothing to test without thread support"
def setUp(self):
self.counter = 0
self.deferred = Deferred()
def schedule(self, *args, **kwargs):
"""
Override in subclasses.
"""
reactor.callFromThread(*args, **kwargs)
def test_lotsOfThreadsAreScheduledCorrectly(self):
"""
L{IReactorThreads.callFromThread} can be used to schedule a large
number of calls in the reactor thread.
"""
def addAndMaybeFinish():
self.counter += 1
if self.counter == 100:
self.deferred.callback(True)
for i in range(100):
self.schedule(addAndMaybeFinish)
return self.deferred
def test_threadsAreRunInScheduledOrder(self):
"""
Callbacks should be invoked in the order they were scheduled.
"""
order = []
def check(_):
self.assertEqual(order, [1, 2, 3])
self.deferred.addCallback(check)
self.schedule(order.append, 1)
self.schedule(order.append, 2)
self.schedule(order.append, 3)
self.schedule(reactor.callFromThread, self.deferred.callback, None)
return self.deferred
def test_scheduledThreadsNotRunUntilReactorRuns(self):
"""
Scheduled tasks should not be run until the reactor starts running.
"""
def incAndFinish():
self.counter = 1
self.deferred.callback(True)
self.schedule(incAndFinish)
# Callback shouldn't have fired yet.
self.assertEqual(self.counter, 0)
return self.deferred
class ReflectorTestBase:
"""Base class for testing reflectors."""
if interfaces.IReactorThreads(reactor, None) is None:
skip = "No thread support, no reflector tests"
count = 100 # a parameter used for running iterative tests
def wait(self, d, timeout=10.0):
return unittest.wait(d, timeout=timeout)
def randomizeRow(self, row):
return randomizeRow(row, self.nulls_ok, self.trailing_spaces_ok)
def setUp(self):
self.reflector = self.createReflector()
def tearDown(self):
self.destroyReflector()
def destroyReflector(self):
pass
def testReflector(self):
# create one row to work with
row = TestRow()
row.assignKeyAttr("key_string", "first")
values = self.randomizeRow(row)
# save it
self.wait(self.reflector.insertRow(row))
# now load it back in
whereClause = [("key_string", EQUAL, "first")]
d = self.reflector.loadObjectsFrom(tableName, whereClause=whereClause)
d.addCallback(self.gotData)
self.wait(d)
# make sure it came back as what we saved
self.failUnless(len(self.data) == 1, "no row")
parent = self.data[0]
self.failUnless(rowMatches(parent, values), "no match")
# create some child rows
inserts = []
child_values = {}
for i in range(0, self.num_iterations):
row = ChildRow()
row.assignKeyAttr("childId", i)
values = self.randomizeRow(row)
values['test_key'] = row.test_key = "first"
child_values[i] = values
inserts.append(self.reflector.insertRow(row))
row = None
self.wait(defer.gatherResults(inserts), timeout=self.num_iterations)
del inserts
d = self.reflector.loadObjectsFrom(childTableName, parentRow=parent)
d.addCallback(self.gotData)
self.wait(d)
self.failUnless(
len(self.data) == self.num_iterations, "no rows on query")
self.failUnless(
len(parent.childRows) == self.num_iterations,
"did not load child rows: %d" % len(parent.childRows))
for child in parent.childRows:
self.failUnless(rowMatches(child, child_values[child.childId]),
"child %d does not match" % child.childId)
# loading these objects a second time should not re-add them
# to the parentRow.
d = self.reflector.loadObjectsFrom(childTableName, parentRow=parent)
d.addCallback(self.gotData)
self.wait(d)
self.failUnless(
len(self.data) == self.num_iterations, "no rows on query")
self.failUnless(
len(parent.childRows) == self.num_iterations,
"child rows added twice!: %d" % len(parent.childRows))
# now change the parent
values = self.randomizeRow(parent)
self.wait(self.reflector.updateRow(parent))
parent = None
# now load it back in
whereClause = [("key_string", EQUAL, "first")]
d = self.reflector.loadObjectsFrom(tableName, whereClause=whereClause)
d.addCallback(self.gotData)
self.wait(d)
# make sure it came back as what we saved
self.failUnless(len(self.data) == 1, "no row")
parent = self.data[0]
self.failUnless(rowMatches(parent, values), "no match")
# save parent
test_values = {}
test_values[parent.key_string] = values
parent = None
# save some more test rows
for i in range(0, self.num_iterations):
row = TestRow()
row.assignKeyAttr("key_string", "bulk%d" % i)
test_values[row.key_string] = self.randomizeRow(row)
self.wait(self.reflector.insertRow(row))
row = None
# now load them all back in
d = self.reflector.loadObjectsFrom("testTable")
d.addCallback(self.gotData)
self.wait(d, 100.0)
# make sure they are the same
self.failUnless(
len(self.data) == self.num_iterations + 1,
"query did not get rows")
for row in self.data:
self.failUnless(rowMatches(row, test_values[row.key_string]),
"child %s does not match" % row.key_string)
# now change them all
for row in self.data:
test_values[row.key_string] = self.randomizeRow(row)
self.wait(self.reflector.updateRow(row))
self.data = None
# load'em back
d = self.reflector.loadObjectsFrom("testTable")
d.addCallback(self.gotData)
self.wait(d)
# make sure they are the same
self.failUnless(
len(self.data) == self.num_iterations + 1,
"query did not get rows")
for row in self.data:
self.failUnless(rowMatches(row, test_values[row.key_string]),
"child %s does not match" % row.key_string)
# now delete them
for row in self.data:
self.wait(self.reflector.deleteRow(row))
self.data = None
# load'em back
d = self.reflector.loadObjectsFrom("testTable")
d.addCallback(self.gotData)
self.wait(d)
self.failUnless(len(self.data) == 0, "rows were not deleted")
# create one row to work with
row = TestRow()
row.assignKeyAttr("key_string", "first")
values = self.randomizeRow(row)
# save it
self.wait(self.reflector.insertRow(row))
# delete it
self.wait(self.reflector.deleteRow(row))
def gotData(self, data):
self.data = data
