def test_set_thread_num_keep_order(self):
def _pass(args):
return args
rst = []
jm = jobq.JobManager([_pass, rst.append], keep_order=True)
setter = {'running': True}
def _change_thread_nr():
while setter['running']:
jm.set_thread_num(_pass, random.randint(1, 4))
time.sleep(0.5)
ths = []
for ii in range(3):
th = threadutil.start_daemon_thread(_change_thread_nr)
ths.append(th)
n = 10240
for i in range(n):
jm.put(i)
jm.join()
rst.sort()
for i in range(n):
self.assertEqual(i, rst[i])
setter['running'] = False
for th in ths:
th.join()
def test_set_thread_num(self):
def _pass(args):
return args
rst = []
jm = jobq.JobManager([_pass, rst.append])
for invalid in (0, -1, 1.1):
self.assertRaises(
AssertionError, jm.set_thread_num, _pass, invalid)
n = 10240
for i in range(n):
jm.put(i)
# change thread number every 91 put
if i % 91:
# randomly change thread number
jm.set_thread_num(_pass, i % 3 + 1)
jm.join()
rst.sort()
for i in range(n):
self.assertEqual(i, rst[i])
def test_set_thread_num_with_object_method(self):
"""
In python2, `x = X(); x.meth is x.meth` results in a `False`.
Every time to retrieve a method, python creates a new **bound** function.
See https://stackoverflow.com/questions/15977808/why-dont-methods-have-reference-equality
"""
class X(object):
def meth(self):
pass
x = X()
jm = jobq.JobManager([x.meth])
before = jm.stat()
self.assertEqual(1, before['workers'][0]['nr_worker'])
# This should not raise JobWorkerNotFound.
jm.set_thread_num(x.meth, 2)
after = jm.stat()
self.assertEqual(2, after['workers'][0]['nr_worker'])
jm.join()
def run(ip='127.0.0.1', port=63482, jobq_thread_count=10):
JobdWebSocketApplication.jobq_mgr = jobq.JobManager([(_parse_request,
jobq_thread_count)])
WebSocketServer(
(ip, port),
Resource(OrderedDict({'/': JobdWebSocketApplication})),
).serve_forever()
def test_dispatcher_job_manager(self):
n_threads = 3
n_numbers = 1000
rst = {}
ordered = []
def _collect_by_tid(ii):
tid = threading.current_thread().ident
if tid not in rst:
rst[tid] = []
rst[tid].append(ii)
return ii
def _collect(ii):
ordered.append(ii)
jm = jobq.JobManager([
(_collect_by_tid, n_threads, lambda x: x % n_threads),
(_collect, 1),
])
# In dispatcher mode it does not allow to set thread_num to prevent out
# of order output
self.assertRaises(jobq.JobWorkerError, jm.set_thread_num, _collect_by_tid, 10)
for i in range(n_numbers):
jm.put(i)
st = jm.stat()
dd(st)
self.assertIn('dispatcher', st['workers'][0])
dstat = st['workers'][0]['dispatcher']
self.assertEqual(n_threads, len(dstat))
for ds in dstat:
self.assertIn('input', ds)
self.assertIn('output', ds)
jm.join()
self.assertEqual(n_threads, len(rst.keys()))
for arr in rst.values():
m = arr[0] % n_threads
for i in arr:
# a thread receives args with the same mod by `n_threads`
self.assertEqual(m, i % n_threads)
# with dispatcher, output are ordered
self.assertEqual([x for x in range(n_numbers)],
ordered)
def test_stat_on_builtin_method(self):
rst = []
jm = jobq.JobManager([rst.append])
# stat() read attribute `func.__module__`.
# But for builtin method, there is no __module__ attribute.
# this should not raise
jm.stat()
jm.join()
def test_join_timeout(self):
def _sleep(args):
time.sleep(0.1)
jm = jobq.JobManager([_sleep])
t0 = time.time()
for i in range(3):
jm.put(i)
jm.join(timeout=0.1)
t1 = time.time()
self.assertTrue(0.09 < t1 - t0 < 0.11)
def test_stat(self):
def _pass(args):
return args
jm = jobq.JobManager([_pass])
for i in range(3):
jm.put(i)
time.sleep(0.01)
st = jm.stat()
# each element get in twice: _pass and _blackhole
self.assertEqual((i + 1) * 2, st['in'])
self.assertEqual((i + 1) * 2, st['out'])
self.assertEqual(0, st['doing'])
jm.join()
def test_manager(self):
rst = []
def _sleep(args):
time.sleep(0.1)
return args
def _collect(args):
rst.append(args)
jm = jobq.JobManager([_sleep, _collect])
t0 = time.time()
for i in range(3):
jm.put(i)
jm.join()
t1 = time.time()
self.assertEqual([0, 1, 2], rst)
self.assertTrue(-0.05 < t1 - t0 - 0.3 < 0.05)
elif cmd == 'user':
mb.create_user(args.username,
args.password,
host=args.host,
privileges=[args.privilege.split(':', 1)],
binlog=(args.binlog == 1))
else:
raise ValueError('unsupported command: ' + repr(cmd))
return jobq.EmptyRst
def output(rst):
if callable(rst):
rst()
else:
print json.dumps(rst, indent=2)
jm = jobq.JobManager([(worker, args.jobs), (output, 1)])
for port in ports:
jm.put(port)
jm.join()
if len(rsts) == len(ports):
sys.exit(0)
else:
sys.exit(1)