def test_duplicate_items(self):
"""
one item on task_queue should be one item off the results_queue
"""
def no_work(i):
return i
qe = QueueExecute(6, no_work)
#qe.print_log = True
qe.run()
input_size = 200
for a in range(input_size):
qe.add_task_item(a)
qe.finished_adding_items()
r = {}
for result in qe.get_result():
#print "got result", result
r[result.input] = result.result_value
qe.join()
self.assertEqual(len(r.keys()), input_size)
def test_callable(self):
"""
Workers use a callable object instead of a function.
"""
qe = QueueExecute()
for a in "0123456789abcdef":
qe.add_task_item(a)
# finish called before any processing has been done
qe.finished_adding_items()
for i in range(2):
qe.add_worker(CallBasedCryptoChallenge(i))
# start processing
qe.run()
r = {}
worker_ids_seen = set()
for result in qe.get_result():
#print "got result", result
r[result.input] = result.result_value
worker_ids_seen.add(str(result.worker_id))
qe.join()
self.got_correct_results(r)
# All worker processes should have done some work
worker_ids_l = list(worker_ids_seen)
worker_ids_l.sort()
self.assertEqual(','.join(worker_ids_l), '0,1')
def test_extra_process(self):
"""
After calling .run(), add another process.
"""
qe = QueueExecute(2, crypto_challenge)
for a in "0123456789abcdef":
qe.add_task_item(a)
# finish called before any processing has been done
qe.finished_adding_items()
# add another worker (must be done before .run())
qe.add_worker(crypto_challenge)
# start processing
qe.run()
r = {}
worker_ids_seen = set()
for result in qe.get_result():
#print "got result", result
r[result.input] = result.result_value
worker_ids_seen.add(str(result.worker_id))
qe.join()
self.got_correct_results(r)
# All worker processes should have done some work
worker_ids_l = list(worker_ids_seen)
worker_ids_l.sort()
self.assertEqual(','.join(worker_ids_l), '0,1,2')
def test_zero_items(self):
"""
If no items are given to process, it shouldn't hang waiting.
"""
qe = QueueExecute(3, crypto_challenge)
#qe.print_log = True
qe.finished_adding_items()
qe.run()
for result in qe.get_result():
raise ValueError("Shouldn't get here")
qe.join()
def test_basic_usage(self):
qe = QueueExecute(6, crypto_challenge)
#qe.print_log = True
qe.run()
for a in "0123456789abcdef":
qe.add_task_item(a)
qe.finished_adding_items()
r = {}
for result in qe.get_result():
#print "got result", result
r[result.input] = result.result_value
qe.join()
self.got_correct_results(r)
def test_attrib_based(self):
qe = QueueExecute(3, crypto_challenge_attrib_based)
#qe.print_log = True
for a in "0123456789abcdef":
ab = AttribBased(data={'item':a})
#ab_str = pickle.dumps(ab)
qe.add_task_item(ab)
qe.run()
qe.finished_adding_items()
r = {}
for result in qe.get_result():
r[result.input.item] = result.result_value
qe.join()
self.got_correct_results(r)
def test_separate_generator(self):
"""
add some items to input queue and then "one in one out" of
QueueExecute
"""
def no_work(i):
return i
def generator():
i=0
while True:
yield i
i += 1
parallel_processes = 5
records_generator = generator()
queue_processor = QueueExecute()
for task_id in range(parallel_processes):
sub_processor = no_work
queue_processor.add_worker(sub_processor)
queue_processor.run()
initial_buffer_size = parallel_processes * 5
r_count = 0
for _record in records_generator:
#print "pre sending:%s" % _record
queue_processor.add_task_item(_record)
r_count += 1
if r_count >= initial_buffer_size:
break
result_generator = queue_processor.get_result()
# one in one out from here
r = []
for _record in records_generator:
#print "post sending:%s" % _record
# careful - a StopIteration here would be a fail because there
# are def. results pending
result = next(result_generator)
r.append(result.result_value)
if _record > 100:
break
# add record to queue
queue_processor.add_task_item(_record)
# end of input stream signalled to sub-processes
queue_processor.finished_adding_items()
# wait for remaining results
for result in result_generator:
r.append(result.result_value)
# wait for processes to finish
queue_processor.join()
self.assertEqual(101, len(r))