def test_dispatcher_run(self): n_threads = 3 n_numbers = 1000 rst = {} ordered = [] def _collect_by_tid(ii): tid = threading.current_thread().ident if tid not in rst: rst[tid] = [] rst[tid].append(ii) return ii def _collect(ii): ordered.append(ii) k3jobq.run(list(range(n_numbers)), [ (_collect_by_tid, n_threads, lambda x: x % n_threads), (_collect, 1), ]) self.assertEqual(n_threads, len(list(rst.keys()))) for arr in list(rst.values()): m = arr[0] % n_threads for i in arr: # a thread receives args with the same mod by `n_threads` self.assertEqual(m, i % n_threads) # with dispatcher, output are ordered self.assertEqual([x for x in range(n_numbers)], ordered)
def test_default_timeout_is_not_too_large(self): # Issue: threading.Thread.join does not accept a very large timeout # value def _sleep_1(args): time.sleep(0.02) k3jobq.run(list(range(1)), [_sleep_1])
def test_generator(self): def gen(args): for i in range(3): yield i time.sleep(0.1) def collect(args): rst.append(args) rst = [] k3jobq.run(list(range(3)), [(gen, 2), collect], keep_order=True) self.assertEqual([0, 1, 2] * 3, rst, "generator should extract all before next") rst = [] k3jobq.run(list(range(3)), [(gen, 2), collect], keep_order=False) self.assertEqual(set([0, 1, 2]), set(rst), "generator should get all") self.assertEqual(9, len(rst), 'nr of elts')
def test_sequential(self): cases = ( ([0, 1, 2], [add1], [1, 2, 3]), ([0, 1, 2], [add1, multi2], [2, 4, 6]), (list(range(3)), [add1, (multi2, 1)], [2, 4, 6]), (list(range(100)), [add1, (multi2, 1), discard_even], []), (list(range(100)), [add1, discard_even], list(range(1, 101, 2))), (list(range(1024 * 10)), [add1, multi2], list(range(2, 1024 * 10 * 2 + 2, 2))), ) def collect(args): rst.append(args) for inp, workers, out in cases: rst = [] k3jobq.run(inp, workers + [collect]) self.assertEqual(out, rst)
def test_limit_job_speed(self): job_num = 300 job_speed = 100 def entry_iter(): for ii in range(job_num): yield ii def empty(num): pass t0 = time.time() k3jobq.run(entry_iter(), [ (k3jobq.limit_job_speed(job_speed, 1), 1), (empty, 10), ]) self.assertEqual(int(job_num / job_speed), int(time.time() - t0))
def test_concurrent(self): cases = ( (list(range(100)), [add1, (multi2_sleep, 10)], list(range(2, 202, 2))), (list(range(100)), [add1, (multi2_sleep, 10)], list(range(2, 202, 2))), (list(range(1024 * 10)), [add1, (multi2, 4)], list(range(2, 1024 * 10 * 2 + 2, 2))), ) def collect(args): rst.append(args) for inp, workers, out in cases: rst = [] k3jobq.run(inp, workers + [collect]) self.assertEqual(set(out), set(rst)) rst = [] k3jobq.run(inp, workers + [collect], keep_order=True) self.assertEqual(out, rst)
def test_exception(self): # Add a handler, or python complains "no handler assigned # to...." jl = logging.getLogger('pykit.k3jobq') jl.addHandler(logging.NullHandler()) def err_on_even(args): if args % 2 == 0: raise Exception('even number') else: return args def collect(args): rst.append(args) rst = [] k3jobq.run(list(range(10)), [err_on_even, collect]) self.assertEqual(list(range(1, 10, 2)), rst) # remove NullHandler jl.handlers = []
def test_timeout(self): def _sleep_1(args): sleep_got.append(args) time.sleep(0.1) return args def collect(args): rst.append(args) # collect quits before it get any item from its input queue rst = [] sleep_got = [] k3jobq.run(range(10), [_sleep_1, collect], timeout=0.05) time.sleep(0.2) self.assertEqual([0], sleep_got) self.assertEqual([], rst) rst = [] sleep_got = [] k3jobq.run(range(10), [_sleep_1, collect], timeout=0.15) time.sleep(0.2) self.assertEqual([0, 1], sleep_got) self.assertEqual([0], rst)
def _start_jobq_in_thread(self, n_items, n_worker, keep_order=False): def _sleep_1(args): time.sleep(0.1) return args def _nothing(args): return args probe = {} th = threading.Thread(target=lambda: k3jobq.run( list(range(n_items)), [(_sleep_1, n_worker), _nothing], probe=probe, keep_order=keep_order, )) th.daemon = True th.start() return th, probe
#!/usr/bin/env python import k3jobq if __name__ == "__main__": def add1(args): return args + 1 def multi2(args): return args * 2 def printarg(args): print(args) k3jobq.run([0, 1, 2], [add1, printarg]) # > 1 # > 2 # > 3 k3jobq.run((0, 1, 2), [add1, multi2, printarg]) # > 2 # > 4 # > 6 # Specify number of threads for each job: # Job 'multi2' uses 1 thread. # This is the same as the above example. k3jobq.run(list(range(3)), [add1, (multi2, 1), printarg]) # > 2