def test_exception_lambda():
pool = ProcessPool(ExceptionLambdaWorker, 1)
f1 = pool.submit_job()
with pytest.raises(Exception) as excinfo:
f1.result()
assert "JobFailedException: ExceptionWithLambda(annoying)" == str(excinfo.value)
pool.join()
assert pool.terminated
def test_exception_multiarg():
pool = ProcessPool(KWArgExceptionWorker, 1)
f1 = pool.submit_job()
with pytest.raises(JobFailedException) as excinfo:
f1.result()
assert "response" in str(excinfo.value)
pool.join()
assert pool.terminated
def test_exception_traceback():
pool = ProcessPool(CrashingSquareNumberWorker, 1)
f1 = pool.submit_job("transform", 13)
exc = f1.exception()
assert exc is not None
exc_traceback = "".join(traceback.TracebackException.from_exception(exc).format())
assert "Traceback (most recent call last)" in exc_traceback
pool.join()
assert pool.terminated
def test_plain_exception():
pool = ProcessPool(CrashingSquareNumberWorker, 1)
f1 = pool.submit_job("transform", 13)
with pytest.raises(JobFailedException) as excinfo:
f1.result()
assert "ValueError" == str(excinfo.value.original_exception_type)
assert "JobFailedException: ValueError(Got unlucky!)" == str(excinfo.value)
pool.join()
assert pool.terminated
def test_nested():
num_workers = 10
pool = ProcessPool(PoolWorker, 10)
jobs = [pool.submit_job(i) for i in range(num_workers)]
results = [j.result() for j in jobs]
assert [0, 0, 1, 5, 14, 30, 55, 91, 140, 204] == results
assert not pool.terminated
assert not pool.shutting_down
pool.join()
assert pool.terminated
def test_many():
class SquareNumberWorker:
def run(self, num, *args):
return num * num
pool = ProcessPool(SquareNumberWorker, 4)
futures = [pool.submit_job(i) for i in range(100)]
result = [f.result() for f in futures]
assert [n * n for n in range(100)] == result
pool.join()
def test_simple():
pool = ProcessPool(SquareNumberWorker, 1)
f1 = pool.submit_job("transform", 2)
f2 = pool.submit_job("transform", 3)
assert 4 == f1.result()
assert 9 == f2.result()
assert not pool.terminated
assert not pool.shutting_down
pool.join()
assert pool.terminated
def test_pickle_bad_arg_given():
bad_arg = []
for _ in range(10000):
bad_arg = [0, bad_arg]
pool = ProcessPool(CrashingSquareNumberWorker, 1)
f1 = pool.submit_job(bad_arg)
with pytest.raises(Exception) as excinfo: # not working
f1.result()
assert "maximum recursion depth exceeded while pickling" in str(excinfo.value)
assert not pool.terminated
assert not pool.shutting_down
pool.join()
assert pool.terminated
def test_submit_or_join_after_join():
pool = ProcessPool(SquareNumberWorker, 1)
f1 = pool.submit_job("transform", 0)
assert 0 == f1.result()
pool.join()
assert pool.terminated
with pytest.raises(ProcessPoolShutDownException):
pool.submit_job("transform", 13)
with pytest.raises(ProcessPoolShutDownException):
pool.join()
assert pool.terminated
def test_submit_or_join_after_terminate():
pool = ProcessPool(CrashingSquareNumberWorker, 1)
f1 = pool.submit_job("transform", 0)
pool.terminate()
with pytest.raises(WorkerDiedException) as excinfo:
f1.result()
assert excinfo.value.code < 0
with pytest.raises(ProcessPoolShutDownException):
pool.submit_job("transform", 13)
with pytest.raises(ProcessPoolShutDownException):
pool.join()
assert pool.terminated
def test_pickle_bad_arg_returned():
class BadObjectReturner:
def run(self, *args):
bad_list = []
for _ in range(10000):
bad_list = [0, bad_list]
return bad_list
pool = ProcessPool(BadObjectReturner, 1)
f1 = pool.submit_job(":(")
with pytest.raises(Exception) as excinfo: # not working
f1.result()
assert "maximum recursion depth exceeded while pickling" in str(excinfo.value)
assert not pool.terminated
assert not pool.shutting_down
pool.join()
assert pool.terminated
def test_run_job_exception():
pool = ProcessPool(CrashingSquareNumberWorker, 1)
assert 16 == pool.run_job("transform", 4)
with pytest.raises(JobFailedException):
pool.run_job("transform", 13)
pool.join()
assert pool.terminated
def test_crash():
pool = ProcessPool(CrashingSquareNumberWorker, 1)
f1 = pool.submit_job("transform", num=1)
f2 = pool.submit_job("transform", 1337)
f3 = pool.submit_job("transform", 3)
assert not pool.terminated
assert not pool.shutting_down
pool.join()
assert pool.terminated
assert 1 == f1.result()
assert 9 == f3.result()
with pytest.raises(WorkerDiedException):
f2.result()
def run(self, num_workers):
pool = ProcessPool(SquareNumberWorker, num_workers)
jobs = [pool.submit_job("transform", i) for i in range(num_workers)]
results = sum([j.result() for j in jobs])
pool.join()
return results
'~/segment1')
segmentBatch.clear()
segmentVectorBatch.clear()
except Exception as e:
print(e)
pass
return
modeldatatwitter = loadModelTwitter()
#modeldatatopics = loadModelTopics()
scriptindex = sys.argv[1]
if scriptindex == '1':
print('Generate/Serialise Topic Vectors')
pool = ProcessPool(TopicVectorComputationWorker, 8)
print('Process Pool Initialised')
futures = [
pool.submit_job(400, "twitter", "topicDataFile0" + str(i))
for i in [0, 1, 2, 3, 4, 5, 6, 7]
]
topicVectors = [f.result for f in futures]
pool.join()
else:
print('Generate/Serialise Segment Vectors')
generateSegmentsVectorFile(modeldatatwitter, 400, "twitter",
"segmenttwittervectors.txt")