def test_set_thread_num_keep_order(self):
def _pass(args):
return args
rst = []
jm = k3jobq.JobManager([_pass, rst.append], keep_order=True)
setter = {'running': True}
def _change_thread_nr():
while setter['running']:
jm.set_thread_num(_pass, random.randint(1, 4))
time.sleep(0.5)
ths = []
for ii in range(3):
th = k3thread.daemon(_change_thread_nr)
ths.append(th)
n = 10240
for i in range(n):
jm.put(i)
jm.join()
rst.sort()
for i in range(n):
self.assertEqual(i, rst[i])
setter['running'] = False
for th in ths:
th.join()
def test_set_thread_num(self):
def _pass(args):
return args
rst = []
jm = k3jobq.JobManager([_pass, rst.append])
for invalid in (0, -1, 1.1):
self.assertRaises(AssertionError, jm.set_thread_num, _pass,
invalid)
n = 10240
for i in range(n):
jm.put(i)
# change thread number every 91 put
if i % 91:
# randomly change thread number
jm.set_thread_num(_pass, i % 3 + 1)
jm.join()
rst.sort()
for i in range(n):
self.assertEqual(i, rst[i])
def test_set_thread_num_with_object_method(self):
"""
In python2, `x = X(); x.meth is x.meth` results in a `False`.
Every time to retrieve a method, python creates a new **bound** function.
See https://stackoverflow.com/questions/15977808/why-dont-methods-have-reference-equality
"""
class X(object):
def meth(self):
pass
x = X()
jm = k3jobq.JobManager([x.meth])
before = jm.stat()
self.assertEqual(1, before['workers'][0]['nr_worker'])
# This should not raise JobWorkerNotFound.
jm.set_thread_num(x.meth, 2)
after = jm.stat()
self.assertEqual(2, after['workers'][0]['nr_worker'])
jm.join()
def test_dispatcher_job_manager(self):
n_threads = 3
n_numbers = 1000
rst = {}
ordered = []
def _collect_by_tid(ii):
tid = threading.current_thread().ident
if tid not in rst:
rst[tid] = []
rst[tid].append(ii)
return ii
def _collect(ii):
ordered.append(ii)
jm = k3jobq.JobManager([
(_collect_by_tid, n_threads, lambda x: x % n_threads),
(_collect, 1),
])
# In dispatcher mode it does not allow to set thread_num to prevent out
# of order output
self.assertRaises(k3jobq.JobWorkerError, jm.set_thread_num,
_collect_by_tid, 10)
for i in range(n_numbers):
jm.put(i)
st = jm.stat()
dd(st)
self.assertIn('dispatcher', st['workers'][0])
dstat = st['workers'][0]['dispatcher']
self.assertEqual(n_threads, len(dstat))
for ds in dstat:
self.assertIn('input', ds)
self.assertIn('output', ds)
jm.join()
self.assertEqual(n_threads, len(list(rst.keys())))
for arr in list(rst.values()):
m = arr[0] % n_threads
for i in arr:
# a thread receives args with the same mod by `n_threads`
self.assertEqual(m, i % n_threads)
# with dispatcher, output are ordered
self.assertEqual([x for x in range(n_numbers)], ordered)
def test_stat_on_builtin_method(self):
rst = []
jm = k3jobq.JobManager([rst.append])
# stat() read attribute `func.__module__`.
# But for builtin method, there is no __module__ attribute.
# this should not raise
jm.stat()
jm.join()
def test_join_timeout(self):
def _sleep(args):
time.sleep(0.1)
jm = k3jobq.JobManager([_sleep])
t0 = time.time()
for i in range(3):
jm.put(i)
jm.join(timeout=0.1)
t1 = time.time()
self.assertTrue(0.09 < t1 - t0 < 0.11)
def test_stat(self):
def _pass(args):
return args
jm = k3jobq.JobManager([_pass])
for i in range(3):
jm.put(i)
time.sleep(0.01)
st = jm.stat()
# each element get in twice: _pass and _blackhole
self.assertEqual((i + 1) * 2, st['in'])
self.assertEqual((i + 1) * 2, st['out'])
self.assertEqual(0, st['doing'])
jm.join()
def test_manager(self):
rst = []
def _sleep(args):
time.sleep(0.1)
return args
def _collect(args):
rst.append(args)
jm = k3jobq.JobManager([_sleep, _collect])
t0 = time.time()
for i in range(3):
jm.put(i)
jm.join()
t1 = time.time()
self.assertEqual([0, 1, 2], rst)
self.assertTrue(-0.05 < t1 - t0 - 0.3 < 0.05)