def test_dispatcher_run(self):
n_threads = 3
n_numbers = 1000
rst = {}
ordered = []
def _collect_by_tid(ii):
tid = threading.current_thread().ident
if tid not in rst:
rst[tid] = []
rst[tid].append(ii)
return ii
def _collect(ii):
ordered.append(ii)
k3jobq.run(list(range(n_numbers)), [
(_collect_by_tid, n_threads, lambda x: x % n_threads),
(_collect, 1),
])
self.assertEqual(n_threads, len(list(rst.keys())))
for arr in list(rst.values()):
m = arr[0] % n_threads
for i in arr:
# a thread receives args with the same mod by `n_threads`
self.assertEqual(m, i % n_threads)
# with dispatcher, output are ordered
self.assertEqual([x for x in range(n_numbers)], ordered)
def test_default_timeout_is_not_too_large(self):
# Issue: threading.Thread.join does not accept a very large timeout
# value
def _sleep_1(args):
time.sleep(0.02)
k3jobq.run(list(range(1)), [_sleep_1])
def test_generator(self):
def gen(args):
for i in range(3):
yield i
time.sleep(0.1)
def collect(args):
rst.append(args)
rst = []
k3jobq.run(list(range(3)), [(gen, 2), collect], keep_order=True)
self.assertEqual([0, 1, 2] * 3, rst,
"generator should extract all before next")
rst = []
k3jobq.run(list(range(3)), [(gen, 2), collect], keep_order=False)
self.assertEqual(set([0, 1, 2]), set(rst), "generator should get all")
self.assertEqual(9, len(rst), 'nr of elts')
def test_sequential(self):
cases = (
([0, 1, 2], [add1], [1, 2, 3]),
([0, 1, 2], [add1, multi2], [2, 4, 6]),
(list(range(3)), [add1, (multi2, 1)], [2, 4, 6]),
(list(range(100)), [add1, (multi2, 1), discard_even], []),
(list(range(100)), [add1, discard_even], list(range(1, 101, 2))),
(list(range(1024 * 10)), [add1, multi2],
list(range(2, 1024 * 10 * 2 + 2, 2))),
)
def collect(args):
rst.append(args)
for inp, workers, out in cases:
rst = []
k3jobq.run(inp, workers + [collect])
self.assertEqual(out, rst)
def test_limit_job_speed(self):
job_num = 300
job_speed = 100
def entry_iter():
for ii in range(job_num):
yield ii
def empty(num):
pass
t0 = time.time()
k3jobq.run(entry_iter(), [
(k3jobq.limit_job_speed(job_speed, 1), 1),
(empty, 10),
])
self.assertEqual(int(job_num / job_speed), int(time.time() - t0))
def test_concurrent(self):
cases = (
(list(range(100)), [add1,
(multi2_sleep, 10)], list(range(2, 202, 2))),
(list(range(100)), [add1,
(multi2_sleep, 10)], list(range(2, 202, 2))),
(list(range(1024 * 10)), [add1, (multi2, 4)],
list(range(2, 1024 * 10 * 2 + 2, 2))),
)
def collect(args):
rst.append(args)
for inp, workers, out in cases:
rst = []
k3jobq.run(inp, workers + [collect])
self.assertEqual(set(out), set(rst))
rst = []
k3jobq.run(inp, workers + [collect], keep_order=True)
self.assertEqual(out, rst)
def test_exception(self):
# Add a handler, or python complains "no handler assigned
# to...."
jl = logging.getLogger('pykit.k3jobq')
jl.addHandler(logging.NullHandler())
def err_on_even(args):
if args % 2 == 0:
raise Exception('even number')
else:
return args
def collect(args):
rst.append(args)
rst = []
k3jobq.run(list(range(10)), [err_on_even, collect])
self.assertEqual(list(range(1, 10, 2)), rst)
# remove NullHandler
jl.handlers = []
def test_timeout(self):
def _sleep_1(args):
sleep_got.append(args)
time.sleep(0.1)
return args
def collect(args):
rst.append(args)
# collect quits before it get any item from its input queue
rst = []
sleep_got = []
k3jobq.run(range(10), [_sleep_1, collect], timeout=0.05)
time.sleep(0.2)
self.assertEqual([0], sleep_got)
self.assertEqual([], rst)
rst = []
sleep_got = []
k3jobq.run(range(10), [_sleep_1, collect], timeout=0.15)
time.sleep(0.2)
self.assertEqual([0, 1], sleep_got)
self.assertEqual([0], rst)
def _start_jobq_in_thread(self, n_items, n_worker, keep_order=False):
def _sleep_1(args):
time.sleep(0.1)
return args
def _nothing(args):
return args
probe = {}
th = threading.Thread(target=lambda: k3jobq.run(
list(range(n_items)),
[(_sleep_1, n_worker), _nothing],
probe=probe,
keep_order=keep_order,
))
th.daemon = True
th.start()
return th, probe
#!/usr/bin/env python
import k3jobq
if __name__ == "__main__":
def add1(args):
return args + 1
def multi2(args):
return args * 2
def printarg(args):
print(args)
k3jobq.run([0, 1, 2], [add1, printarg])
# > 1
# > 2
# > 3
k3jobq.run((0, 1, 2), [add1, multi2, printarg])
# > 2
# > 4
# > 6
# Specify number of threads for each job:
# Job 'multi2' uses 1 thread.
# This is the same as the above example.
k3jobq.run(list(range(3)), [add1, (multi2, 1), printarg])
# > 2
