def _replicate_multiprocessing(function, no_replicates, no_processes, subkey_pattern=SUBSTATE_KEY_PATTERN): """Replicate the experiment defined by the function in multiple processes """ with processing_state_context(): pool = multiprocessing.Pool(processes=no_processes) manager = multiprocessing.Manager() ready_queue = manager.Queue() # pylint: disable=no-member result_threads = [] for i in range(no_replicates): # Create a target function to be run in a separate process # using a separate state context target = TargetCreator(function, ready_queue, subkey_pattern % i) result = pool.apply_async(target) # Use a thread to wait for the result waiter = threading.Thread(target=result.get) waiter.start() result_threads.append(waiter) # Make sure the process is really running (probably not necessary) while not ready_queue.get(): pass # pragma: no cover # Wait for the pool, then join it log.debug("Closing pool") pool.close() log.debug("Joining pool") pool.join() # Make sure all the results are in log.debug("Joining threads") for thread in result_threads: thread.join()
def test_after_exception(self): """Test deleting bad state in 4 processes """ state['a'] = 12 try: with processing_state_context(): raise RuntimeError except RuntimeError: pass self.assertEqual(state['a'], 12)
def test_raises_on_deleting(self): """Test deleting bad state in 4 processes """ with processing_state_context(): n_jobs = 200 pool = multiprocessing.Pool(processes=4) results = [] for _ in range(n_jobs): results.append(pool.apply_async(worker5)) pool.close() pool.join() for i in range(n_jobs): self.assertTrue(results[i].get())
def test_deleting(self): """Test deleting state in 4 processes """ with processing_state_context(): n_jobs = 2 pool = multiprocessing.Pool(processes=4) results = [] for i in range(n_jobs): results.append(pool.apply_async(worker2, (i, ))) pool.close() pool.join() for i in range(n_jobs): self.assertNotIn(str(i), state)
def test_deleting(self): """Test deleting state in 4 processes """ with processing_state_context(): n_jobs = 2 pool = multiprocessing.Pool(processes=4) results = [] for i in range(n_jobs): results.append(pool.apply_async(worker2, (i,))) pool.close() pool.join() for i in range(n_jobs): self.assertNotIn(str(i), state)
def test_basic_functionality(self): """Test setting, getting state in 4 processes """ with processing_state_context(): n_jobs = 2 pool = multiprocessing.Pool(processes=4) results = [] for i in range(n_jobs): results.append(pool.apply_async(worker1, (i, ))) pool.close() pool.join() for i in range(n_jobs): self.assertTrue(results[i].get()) self.assertEqual(state[str(i)], i)
def test_basic_functionality(self): """Test setting, getting state in 4 processes """ with processing_state_context(): n_jobs = 2 pool = multiprocessing.Pool(processes=4) results = [] for i in range(n_jobs): results.append(pool.apply_async(worker1, (i,))) pool.close() pool.join() for i in range(n_jobs): self.assertTrue(results[i].get()) self.assertEqual(state[str(i)], i)