def _replicate_multiprocessing(function,
no_replicates,
no_processes,
subkey_pattern=SUBSTATE_KEY_PATTERN):
"""Replicate the experiment defined by the function in multiple processes
"""
with processing_state_context():
pool = multiprocessing.Pool(processes=no_processes)
manager = multiprocessing.Manager()
ready_queue = manager.Queue() # pylint: disable=no-member
result_threads = []
for i in range(no_replicates):
# Create a target function to be run in a separate process
# using a separate state context
target = TargetCreator(function, ready_queue, subkey_pattern % i)
result = pool.apply_async(target)
# Use a thread to wait for the result
waiter = threading.Thread(target=result.get)
waiter.start()
result_threads.append(waiter)
# Make sure the process is really running (probably not necessary)
while not ready_queue.get():
pass # pragma: no cover
# Wait for the pool, then join it
log.debug("Closing pool")
pool.close()
log.debug("Joining pool")
pool.join()
# Make sure all the results are in
log.debug("Joining threads")
for thread in result_threads:
thread.join()
def test_after_exception(self):
"""Test deleting bad state in 4 processes
"""
state['a'] = 12
try:
with processing_state_context():
raise RuntimeError
except RuntimeError:
pass
self.assertEqual(state['a'], 12)
def test_after_exception(self):
"""Test deleting bad state in 4 processes
"""
state['a'] = 12
try:
with processing_state_context():
raise RuntimeError
except RuntimeError:
pass
self.assertEqual(state['a'], 12)
def test_raises_on_deleting(self):
"""Test deleting bad state in 4 processes
"""
with processing_state_context():
n_jobs = 200
pool = multiprocessing.Pool(processes=4)
results = []
for _ in range(n_jobs):
results.append(pool.apply_async(worker5))
pool.close()
pool.join()
for i in range(n_jobs):
self.assertTrue(results[i].get())
def test_deleting(self):
"""Test deleting state in 4 processes
"""
with processing_state_context():
n_jobs = 2
pool = multiprocessing.Pool(processes=4)
results = []
for i in range(n_jobs):
results.append(pool.apply_async(worker2, (i, )))
pool.close()
pool.join()
for i in range(n_jobs):
self.assertNotIn(str(i), state)
def test_raises_on_deleting(self):
"""Test deleting bad state in 4 processes
"""
with processing_state_context():
n_jobs = 200
pool = multiprocessing.Pool(processes=4)
results = []
for _ in range(n_jobs):
results.append(pool.apply_async(worker5))
pool.close()
pool.join()
for i in range(n_jobs):
self.assertTrue(results[i].get())
def test_deleting(self):
"""Test deleting state in 4 processes
"""
with processing_state_context():
n_jobs = 2
pool = multiprocessing.Pool(processes=4)
results = []
for i in range(n_jobs):
results.append(pool.apply_async(worker2, (i,)))
pool.close()
pool.join()
for i in range(n_jobs):
self.assertNotIn(str(i), state)
def test_basic_functionality(self):
"""Test setting, getting state in 4 processes
"""
with processing_state_context():
n_jobs = 2
pool = multiprocessing.Pool(processes=4)
results = []
for i in range(n_jobs):
results.append(pool.apply_async(worker1, (i, )))
pool.close()
pool.join()
for i in range(n_jobs):
self.assertTrue(results[i].get())
self.assertEqual(state[str(i)], i)
def test_basic_functionality(self):
"""Test setting, getting state in 4 processes
"""
with processing_state_context():
n_jobs = 2
pool = multiprocessing.Pool(processes=4)
results = []
for i in range(n_jobs):
results.append(pool.apply_async(worker1, (i,)))
pool.close()
pool.join()
for i in range(n_jobs):
self.assertTrue(results[i].get())
self.assertEqual(state[str(i)], i)
