def testTerminate(self): N = 100 M = 100 def accounting(counts): counts[thread.get_ident()] += 1 for i in xrange(N): counts = collections.defaultdict(int) args = (counts,) pool = ThreadPool(2) for j in xrange(M): pool.apply_async(accounting, args) pool.terminate() pool.join() total_counts = sum(counts.values()) self.assertLessEqual(total_counts, M)
def testClose(self): N = 100 M = 100 counts = collections.defaultdict(int) def accounting(): counts[thread.get_ident()] += 1 for i in xrange(N): counts.clear() pool = ThreadPool(2) for j in xrange(M): pool.apply_async(accounting) pool.close() pool.join() total_counts = sum(counts.itervalues()) self.assertEqual(total_counts, M)
def setUp(self): from chorde.clients.async import AsyncCacheProcessor from chorde.clients.inproc import InprocCacheClient from chorde.threadpool import ThreadPool self.client = AsyncCacheProcessor(1, InprocCacheClient(100), threadpool = functools.partial(ThreadPool(1).subqueue, "meh"))
def setUpClient(self): from chorde.clients.inproc import InprocCacheClient from chorde.clients.async import AsyncWriteCacheClient from chorde.threadpool import ThreadPool return AsyncWriteCacheClient(InprocCacheClient(100), 100, 1, threadpool = functools.partial(ThreadPool(1).subqueue, "meh"))
def setUp(self): self.pool = ThreadPool()
class MultiQueueTest(TestCase): def setUp(self): self.pool = ThreadPool() def tearDown(self): self.pool.terminate() def testQueuePrio(self): self.pool.set_queueprio(1, "a") self.pool.set_queueprio(5, "b") self.assertEqual(1, self.pool.queueprio("a")) self.assertEqual(5, self.pool.queueprio("b")) def testFairness(self): # Calibrate for low-latency (needed by the test) # Should be fine due to zero-copy slicing self.pool.max_batch = 50 terminate = [] M = 50 counts = collections.defaultdict(int) def accounting(i): counts[thread.get_ident()] += 1 def killit(i): while not terminate: self.pool.apply_async(accounting, (i,)) time.sleep(0) # needed to avoid GIL issues that skew test results threads = [ Thread(target=killit, args=(i,)) for i in xrange(M) ] for t in threads: t.start() time.sleep(0.1) # let it fill up terminate.append(None) for t in threads: t.join() t0 = time.time() self.pool.apply(sum, (counts.itervalues(),), queue = "Johnny") t1 = time.time() self.assertLess(t1-t0, 0.025) def testWeighting(self): # Calibrate for maximum fairness accuracy (needed by test) self.pool.max_batch = 100 self.pool.min_batch = 1 counts = collections.defaultdict(int) refcounts = collections.defaultdict(int) def accounting(i): counts[i] += 1 self.pool.set_queueprio(3,"mean") self.pool.set_queueprio(1,"simple") qsequence = ["mean", "simple"] for i in xrange(10000): for q in qsequence: self.pool.apply_async(accounting, (q,), queue=q) refcounts[q] += 1 countsnap = self.pool.apply(counts.copy, queue = "mean") self.assertLess(countsnap["simple"]*2, countsnap["mean"]) self.pool.join(60) def testWrapper(self): # Calibrate for maximum fairness accuracy (needed by test) self.pool.max_batch = 100 self.pool.min_batch = 1 counts = collections.defaultdict(int) simple = self.pool.subqueue("simple", 1) mean = self.pool.subqueue("mean", 3) def accounting(i): counts[i] += 1 qsequence = [(mean,"mean"), (simple,"simple")] for i in xrange(10000): for q,qname in qsequence: q.apply_async(accounting, (qname,)) countsnap = mean.apply(counts.copy) self.pool.join(60) self.assertLess(countsnap["simple"]*2, countsnap["mean"])
def setUp(self): self.pool = ThreadPool().subqueue("something")
class ThreadpoolTest(TestCase): def setUp(self): self.pool = ThreadPool() def tearDown(self): self.join_close(self.pool, 60) @staticmethod def join_close(pool, timeout): pool.close() pool.join(timeout) @staticmethod def join_continue(pool, timeout): pool.join(timeout) def testCleanupCallbacks(self): if not hasattr(self.pool, 'register_cleanup_callback'): self.skipTest("Not implemented") called = set() def callback(): called.add(None) self.pool.register_cleanup_callback(callback) self.pool.apply(time.time) time.sleep(0.01) self.assertTrue(called) def testCleanupCallbackErrors(self): if not hasattr(self.pool, 'register_cleanup_callback'): self.skipTest("Not implemented") def raiser(): raise AssertionError self.pool.register_cleanup_callback(raiser) self.pool.apply(time.time, timeout=0.1) time.sleep(0.1) self.pool.apply(time.time, timeout=0.1) # Just not erroring out or blocking def testLazyStart(self): if not hasattr(self.pool, 'check_started'): self.skipTest("Not implemented") self.assertFalse(self.pool.check_started()) self.pool.apply(time.time, timeout=0.1) self.assertTrue(self.pool.check_started()) def testExplicitStart(self): if not hasattr(self.pool, 'check_started'): self.skipTest("Not implemented") self.assertFalse(self.pool.check_started()) self.pool.start() self.assertTrue(self.pool.check_started()) def testWorkerRespawnOnFork(self): if not hasattr(self.pool, '_ThreadPool__pid'): self.skipTest("Not implemented") self.pool.start() workers = getattr(self.pool, '_ThreadPool__workers') nworkers = len(workers) self.assertTrue(self.pool.is_started()) self.assertTrue(nworkers > 0) # Simulate a fork (pid change, workers die) setattr(self.pool, '_ThreadPool__pid', 0) workers[0].terminate(wait=False) self.assertFalse(self.pool.is_started()) # Make sure it re-starts self.pool.assert_started() self.assertTrue(self.pool.is_started()) workers2 = getattr(self.pool, '_ThreadPool__workers') nworkers2 = len(workers2) self.assertEqual(nworkers, nworkers2) self.assertFalse(workers2[0] is workers[0]) def testAsyncLatency(self): # Warm up the pool self.pool.apply(lambda:None) for i in xrange(100): t0 = time.time() ev = Event() self.pool.apply_async(ev.set) ev.wait() t1 = time.time() self.assertLess(t1-t0, 0.05) def testAsyncBlocking(self): # Warm up the pool self.pool.apply(lambda:None) for i in xrange(100): t0 = time.time() ev = Event() self.pool.apply_async(ev.set) t1 = time.time() self.assertTrue(ev.wait(1)) self.assertLess(t1-t0, 0.05) def testSyncLatency(self): # Warm up the pool self.pool.apply(lambda:None) for i in xrange(100): t0 = time.time() t1 = self.pool.apply(time.time) self.assertLess(t1-t0, 0.05) def testExceptions(self): def raiseme(): raise RuntimeError self.assertRaises(RuntimeError, self.pool.apply, raiseme) def testConcurrency(self): N = 10000 M = 50 counts = collections.defaultdict(int) def accounting(i): counts[thread.get_ident()] += 1 def killit(i): for j in xrange(N): self.pool.apply_async(accounting, (i,)) threads = [ Thread(target=killit, args=(i,)) for i in xrange(M) ] for t in threads: t.start() for t in threads: t.join() self.join_continue(self.pool, 60) total_counts = self.pool.apply(sum, (counts.itervalues(),)) self.assertEqual(total_counts, N*M) def testClose(self): N = 100 M = 100 counts = collections.defaultdict(int) def accounting(): counts[thread.get_ident()] += 1 for i in xrange(N): counts.clear() pool = ThreadPool(2) for j in xrange(M): pool.apply_async(accounting) pool.close() pool.join() total_counts = sum(counts.itervalues()) self.assertEqual(total_counts, M) def testTerminate(self): N = 100 M = 100 def accounting(counts): counts[thread.get_ident()] += 1 for i in xrange(N): counts = collections.defaultdict(int) args = (counts,) pool = ThreadPool(2) for j in xrange(M): pool.apply_async(accounting, args) pool.terminate() pool.join() total_counts = sum(counts.values()) self.assertLessEqual(total_counts, M)
def setup(self): return dict( threadpool=ThreadPool(self.nthreads) )