def _test_workspace_concurrency(tmpdir, timeout, max_procs): """ WorkSpace concurrency test. We merely check that no exception or deadlock happens. """ base_dir = str(tmpdir) err_q = mp_context.Queue() purged_q = mp_context.Queue() stop_evt = mp_context.Event() ws = WorkSpace(base_dir) # Make sure purging only happens in the child processes ws._purge_leftovers = lambda: None # Run a bunch of child processes that will try to purge concurrently NPROCS = 2 if sys.platform == "win32" else max_procs processes = [ mp_context.Process( target=_workspace_concurrency, args=(base_dir, purged_q, err_q, stop_evt) ) for i in range(NPROCS) ] for p in processes: p.start() n_created = 0 n_purged = 0 try: t1 = time() while time() - t1 < timeout: # Add a bunch of locks, and simulate forgetting them. # The concurrent processes should try to purge them. for i in range(50): d = ws.new_work_dir(prefix="workspace-concurrency-") d._finalizer.detach() n_created += 1 sleep(1e-2) finally: stop_evt.set() for p in processes: p.join() # Any errors? try: err = err_q.get_nowait() except Empty: pass else: raise err try: while True: n_purged += purged_q.get_nowait() except Empty: pass # We attempted to purge most directories at some point assert n_purged >= 0.5 * n_created > 0 return n_created, n_purged
async def parent_process_coroutine(): worker_ready = mp_context.Event() worker = AsyncProcess(target=_worker_process, args=(worker_ready, child_pipe)) await worker.start() # Wait for the child process to have started. worker_ready.wait() # Exit immediately, without doing any process teardown (including atexit # and 'finally:' blocks) as if by SIGKILL. This should cause # worker_process to also exit. os._exit(255)
def _test_workspace_concurrency(tmpdir, timeout, max_procs): base_dir = str(tmpdir) err_q = mp_context.Queue() purged_q = mp_context.Queue() stop_evt = mp_context.Event() ws = WorkSpace(base_dir) # Make sure purging only happens in the child processes ws._purge_leftovers = lambda: None NPROCS = 2 if sys.platform == 'win32' else max_procs processes = [ mp_context.Process(target=_workspace_concurrency, args=(base_dir, purged_q, err_q, stop_evt)) for i in range(NPROCS) ] for p in processes: p.start() n_created = 0 n_purged = 0 try: t1 = time() while time() - t1 < timeout: # Add a bunch of locks, and simulate forgetting them for i in range(50): d = ws.new_work_dir(prefix='workspace-concurrency-') d._finalizer.detach() n_created += 1 sleep(1e-2) finally: stop_evt.set() for p in processes: p.join() # Any errors? try: err = err_q.get_nowait() except Empty: pass else: raise err try: while True: n_purged += purged_q.get_nowait() except Empty: pass return n_created, n_purged
def test_workspace_concurrency(tmpdir): """WorkSpace concurrency test. We merely check that no exception or deadlock happens. """ base_dir = str(tmpdir) err_q = mp_context.Queue() purged_q = mp_context.Queue() stop_evt = mp_context.Event() ws = WorkSpace(base_dir) # Make sure purging only happens in the child processes ws._purge_leftovers = lambda: None # Windows (or at least Windows GitHub CI) has been observed to be exceptionally # slow. Don't stress it too much. max_procs = 2 if WINDOWS else 16 # Run a bunch of child processes that will try to purge concurrently barrier = mp_context.Barrier(parties=max_procs + 1) processes = [ mp_context.Process( target=_workspace_concurrency, args=(base_dir, purged_q, err_q, stop_evt, barrier), ) for _ in range(max_procs) ] for p in processes: p.start() barrier.wait() n_created = 0 n_purged = 0 t1 = time() try: # On Linux, you will typically end with n_created > 10.000 # On Windows, it can take 60 seconds to create 50 locks! while time() - t1 < 10: # Add a bunch of locks and simulate forgetting them. # The concurrent processes should try to purge them. for _ in range(100): d = ws.new_work_dir(prefix="workspace-concurrency-") d._finalizer.detach() n_created += 1 finally: stop_evt.set() for p in processes: p.join() # Any errors? try: err = err_q.get_nowait() except queue.Empty: pass else: raise err try: while True: n_purged += purged_q.get_nowait() except queue.Empty: pass # We attempted to purge most directories at some point assert n_purged >= 0.5 * n_created > 0