def test_submit_during_shutdown_no_deadlock(): proceed = Event() submit_more_done = [False] executor = CancelOnShutdownExecutor(Executors.thread_pool(max_workers=2)) def submit_more(): proceed.wait() assert_that( calling(executor.submit).with_args(lambda: None), raises(RuntimeError, "cannot schedule new futures after shutdown"), ) submit_more_done[0] = True futures = [executor.submit(submit_more) for _ in (1, 2, 3)] # Let threads proceed only after shutdown has already started... def set_soon(): time.sleep(0.10) proceed.set() proceed_f = Executors.thread_pool(max_workers=1).submit(set_soon) executor.shutdown(wait=True) proceed_f.result() # It should have reached here without deadlocking # All futures should have completed assert all([f.done() for f in futures]) # And the tests in submit_more should have run assert_that(submit_more_done[0], equal_to(True))
def test_throttle_dynamic_raises_cannot_construct(): """Can't make a ThrottleExecutor if count immediately raises""" error = RuntimeError("oops") def raise_error(): raise error with pytest.raises(RuntimeError) as exc_info: Executors.sync().with_throttle(count=raise_error) assert exc_info.value is error
def __init__( self, url, errata, target="cdn", koji_source=None, threads=4, timeout=60 * 60 * 4, ): """Create a new source. Parameters: url (src) Base URL of Errata Tool, e.g. "http://errata.example.com", "https://errata.example.com:8123". errata (str, list[str]) Advisory ID(s) to be used as push item source. If a single string is given, multiple IDs may be comma-separated. target (str) The target type used when querying Errata Tool. The target type may influence the content produced, depending on the Errata Tool settings. Valid values include at least: "cdn", "rhn", "ftp". koji_source (str) URL of a koji source associated with this Errata Tool instance. threads (int) Number of threads used for concurrent queries to Errata Tool and koji. timeout (int) Number of seconds after which an error is raised, if no progress is made during queries to Errata Tool. """ self._url = url self._errata = list_argument(errata) self._executor = Executors.thread_pool( max_workers=threads).with_retry() # We set aside a separate thread pool for koji so that there are separate # queues for ET and koji calls, yet we avoid creating a new thread pool for # each koji source. self._koji_executor = Executors.thread_pool( max_workers=threads).with_retry() self._koji_cache = {} self._koji_source_url = koji_source self._target = target self._timeout = timeout self._tls = threading.local()
def mount_fstab(mounts): """Wait on each mount, then make sure all fstab is mounted""" from more_executors import Executors, ExceptionRetryPolicy def mount_path(path): log.info(f"Waiting for '{path}' to be mounted...") try: run(f"mount {path}", timeout=120) except Exception as e: exc_type, _, _ = sys.exc_info() log.error(f"mount of path '{path}' failed: {exc_type}: {e}") raise e log.info(f"Mount point '{path}' was mounted.") MAX_MOUNT_TIMEOUT = 60 * 5 future_list = [] retry_policy = ExceptionRetryPolicy(max_attempts=40, exponent=1.6, sleep=1.0, max_sleep=16.0) with Executors.thread_pool().with_timeout(MAX_MOUNT_TIMEOUT).with_retry( retry_policy=retry_policy) as exe: for path in mounts: future = exe.submit(mount_path, path) future_list.append(future) # Iterate over futures, checking for exceptions for future in as_completed(future_list): try: future.result() except Exception as e: raise e
def test_submit_during_shutdown(): proceed = Event() futures = [] submit_more_done = [False] executor = Executors.thread_pool(max_workers=2).with_cancel_on_shutdown() futures = [executor.submit(proceed.wait) for _ in (1, 2, 3)] def submit_more(f): assert_that(f, equal_to(futures[2])) assert_that( calling(executor.submit).with_args(lambda: None), raises(RuntimeError, "cannot schedule new futures after shutdown"), ) submit_more_done[0] = True futures[2].add_done_callback(submit_more) # Shut it down... executor.shutdown(wait=False) proceed.set() # That should have cancelled futures[2] assert_that(futures[2].cancelled()) # And the tests in submit_more should have run assert_that(submit_more_done[0], equal_to(True))
def __init__(self, pulp_hostname, pulp_auth, dry_run, ubiconfig_filename_list=None, ubiconfig_dir_or_url=None, insecure=False, workers_count=4, output_repos=None, **kwargs): # legacy client implemeted in this repo, it's expected to be replaced by pubtools.pulplib.Client self.pulp = Pulp(pulp_hostname, pulp_auth, insecure) self._pulp_hostname = pulp_hostname self._pulp_auth = pulp_auth self._insecure = insecure self._pulp_client = None self.dry_run = dry_run self.output_repos = output_repos self._executor = Executors.thread_pool( max_workers=workers_count).with_retry() self._ubiconfig_list = None self._ubiconfig_filename_list = ubiconfig_filename_list self._ubiconfig_dir_or_url = ubiconfig_dir_or_url self._content_sets = kwargs.get("content_sets", None) self._repo_ids = kwargs.get("repo_ids", None) self._version = kwargs.get("version", None) self._content_set_regex = kwargs.get("content_set_regex", None) self._ubiconfig_map = None
def test_or_with_nocancel(): calls = set() def delayed_call(delay): time.sleep(delay) calls.add(delay) return delay executor = Executors.thread_pool(max_workers=2) futures = [ executor.submit(delayed_call, x) for x in (0.5, 0.1, 0.2, 1.0, 1.1) ] futures = [f_nocancel(f) for f in futures] future = f_or(*futures) result = future.result() # Result comes from the future which completed first assert result == 0.1 # Only this calls should have completed so far assert calls == set([0.1]) # But if we wait on this... assert futures[-1].result() == 1.1 # Then they've all completed now, thanks to nocancel assert calls == set([0.1, 0.2, 0.5, 1.0, 1.1])
def _do_parallel_requests(self, make_request, data_items): """ Call given function with given data items in parallel, collect responses. Args: make_request (function): a function that does the actual request. Must accept a single argument: a data item. Must return a `requests.models.Response` object. data_items (list): a list of arbitrary objects to be passed individually to `make_request()`. The number of parallel requests is defined by `DEFAULT_REQUEST_THREADS_LIMIT` (can be overridden by the user) and of course by the number of actually available threads. If a response fails consistently (see `PyxisSession` for retry policy), the execution is terminated and an informative error is raised. See `PyxisClient._handle_json_response()` for details. Returns: list(dict): list of dictionaries extracted from responses. """ with Executors.thread_pool(max_workers=self.threads_limit).with_map( self._handle_json_response) as executor: futures = [ executor.submit(make_request, data) for data in data_items ] return [f.result() for f in as_completed(futures)]
def test_and_with_nocancel(): calls = set() error = RuntimeError("simulated error") def delayed_call(delay): if delay is error: raise error time.sleep(delay) calls.add(delay) return delay executor = Executors.thread_pool(max_workers=2) futures = [ executor.submit(delayed_call, x) for x in (0.5, error, 0.2, 1.1, 1.2) ] futures = [f_nocancel(f) for f in futures] future = f_and(*futures) exception = future.exception() # error should have been propagated assert exception is error # nothing else has been called yet assert not calls # but if we wait on the last future... assert futures[-1].result() == 1.2 # then all calls were made thanks to nocancel assert calls == set([0.5, 0.2, 1.1, 1.2])
def run(self): with Executors.thread_pool(max_workers=CHECKSUM_THREADS, name="checksummer") as exc: for item in self.iter_input(): # Use a heuristic to try to hand off the item onto the next # phase as quickly as possible. # # - in general we use a thread pool to do calculations in parallel. # # - but if we probably already have sums, we don't want to put # that item onto the back of a potentially long queue where it # may have to wait a long time, when it could be passsed on # immediately. # # Hence we handle some items synchronously and others not. if not item.blocking_checksums: # with_checksums (probably) won't block so just do # it immediately, thus letting the next phase get hold # of the item more quickly. self.put_output(self._get_sums(item)) else: # with_checksums (probably) will block so put it onto # the thread pool's queue. f = exc.submit(self._get_sums, item) self.put_future_output(f)
def test_broken_policy(caplog): """Executor logs without retry if policy raises exception""" policy_error = RuntimeError("error from policy") fn_error = RuntimeError("error from fn") class BrokenRetryPolicy(RetryPolicy): def should_retry(self, attempt, future): raise policy_error executor = Executors.thread_pool().with_retry(retry_policy=BrokenRetryPolicy()) def fn(): raise fn_error # Start the callable future = executor.submit(fn) # It should fail with the error raised from the fn assert future.exception(10.0) is fn_error if not caplog: # TODO: remove me when py2.6 support is dropped return records = caplog.records error_log = records[-1] # It should have logged a message about the error assert "Exception while evaluating retry policy" in error_log.message assert "BrokenRetryPolicy" in error_log.message assert "error from policy" in error_log.exc_text
def test_throttle_dynamic(): throttle = [0] executor = Executors.thread_pool(max_workers=8).with_throttle( count=lambda: throttle[0]) tester = ThrottleTester() futures = [] for _ in range(0, 50): futures.append(executor.submit(tester)) # Currently, it should not be possible for any to execute time.sleep(0.5) assert not tester.entered assert tester.running_count == 0 # If we raise the throttle... throttle[0] = 1 # Then one should be able to start assert_soon(lambda: tester.running_count == 1) # If we raise it again... throttle[0] = 2 # Let the first one complete so it wakes faster tester.proceed() # Now two should be able to start assert_soon(lambda: tester.running_count == 2) # If we unset throttle entirely... throttle[0] = None # Then all threads should be used assert_soon(lambda: tester.running_count == 8) # Throttle back to zero throttle[0] = 0 # Let them all finish tester.proceed_all() # There should NOT be any more able to run assert tester.running_count == 0 time.sleep(0.5) assert tester.running_count == 0 # Raise throttle again works fine throttle[0] = 6 assert_soon(lambda: tester.running_count == 6) # Let them all finish for _ in range(0, 100): tester.proceed() assert_soon(lambda: len(tester.exited) == 50)
def test_timeout(): def sleep_then_return(x): time.sleep(x) return x with Executors.thread_pool(max_workers=2) as executor: futures = [ executor.submit(sleep_then_return, x) for x in [0.5, 0.5, 0.5, 0.5, 0.5] ] futures[3] = f_nocancel(futures[3]) futures = [f_timeout(f, 0.02) for f in futures] try: futures[-1].result() raise AssertionError("Should have failed") except CancelledError: # expected pass assert futures[0].result() == 0.5 assert futures[1].result() == 0.5 assert futures[2].cancelled() assert futures[3].result() == 0.5 assert futures[4].cancelled()
def __init__(self, url, threads=4, timeout=60 * 60): """Create a new source. Parameters: url (list[str]) URL(s) of locally accessible staging directories, e.g. ``"/mnt/staging/my-content-for-push"``. These directories must follow the documented layout for staging areas. threads (int) Number of threads used for concurrent loading of files. timeout (int) Number of seconds after which an error is raised, if no progress is made during each step. """ super(StagedSource, self).__init__() self._url = list_argument(url) self._threads = threads self._timeout = timeout # Note: this executor does not have a retry. # NFS already does a lot of its own retries. self._executor = (Executors.thread_pool( name="pushsource-staged", max_workers=threads).with_timeout( timeout).with_cancel_on_shutdown())
def test_no_rebind(): bound = Executors.sync().bind(mult10) try: bound.bind(mult2) raise AssertionError("Chained bind should have failed!") # pragma: no cover except AttributeError: pass
def test_bind_with_callable(): async_mult2 = Executors.thread_pool().bind(mult_class(2)) inputs = [0, 1, 2] futures = [async_mult2(x) for x in inputs] results = [f.result() for f in futures] assert results == [0, 2, 4]
def test_bind_with_partial(): async_mult2 = Executors.thread_pool().bind(partial(mult, 2)) inputs = [0, 1, 2] futures = [async_mult2(x) for x in inputs] results = [f.result() for f in futures] assert results == [0, 2, 4]
def test_single_bind(): async_mult2 = Executors.thread_pool().bind(mult2) inputs = [0, 1, 2] futures = [async_mult2(x) for x in inputs] results = [f.result() for f in futures] assert results == [0, 2, 4]
def test_flat_bind(): bound_async_mult = Executors.thread_pool().flat_bind(async_mult) inputs = [(0, 1), (2, 3), (4, 5)] expected_results = [0, 6, 20] futures = [bound_async_mult(x, y) for (x, y) in inputs] results = [f.result() for f in futures] assert results == expected_results
def timeout_executor(): global EXECUTOR_REF # pylint: disable=global-statement with LOCK: executor = EXECUTOR_REF and EXECUTOR_REF() if not executor: executor = Executors.sync().with_flat_map( lambda x: x).with_timeout(None) EXECUTOR_REF = weakref.ref(executor) return executor
def test_bind_then_map(): async_mult200 = ( Executors.thread_pool().with_map(mult10).bind(mult2).with_map(mult10) ) inputs = [0, 1, 2] futures = [async_mult200(x) for x in inputs] results = [f.result() for f in futures] assert results == [0, 200, 400]
def __init__(self, input_repos, ubi_config, workers=8): self._input_repos = input_repos self._ubi_config = ubi_config # executor for this class, not adding retries because for pulp # we use executor from pulplib self._executor = Executors.thread_pool(max_workers=workers) self.binary_rpms = None self.debug_rpms = None self.source_rpms = None
def test_with_map(asyncio): with Executors.thread_pool().with_map( lambda x: [x, x]).with_asyncio() as executor: f = executor.submit(sleep_then_return, 0.01, "abc") result = asyncio.get_event_loop().run_until_complete(f) # MapExecutor should have worked, as usual assert_that(result, equal_to(["abc", "abc"])) assert_that(result, equal_to(f.result()))
def test_nocancel(): executor = Executors.thread_pool(max_workers=2) futures = [f_nocancel(executor.submit(delay_then, x)) for x in [1, 2, 3, 4, 5]] for f in futures: # Should not be able to cancel it even though most # are not started yet assert not f.cancel() assert [f.result() for f in futures] == [1, 2, 3, 4, 5]
def __init__(self, pulp_hostname, pulp_auth, dry_run, ubiconfig_filename_list=None, ubiconfig_dir_or_url=None, insecure=False, workers_count=4, output_repos=None, **kwargs): self.pulp = Pulp(pulp_hostname, pulp_auth, insecure) self.dry_run = dry_run self.output_repos = output_repos self._executor = Executors.thread_pool(max_workers=workers_count).with_retry() self.ubiconfig_list = self._load_ubiconfig(ubiconfig_filename_list, ubiconfig_dir_or_url, content_sets=kwargs.get('content_sets', None), repo_ids=kwargs.get('repo_ids', None))
def timeout_executor(): global EXECUTOR_REF # pylint: disable=global-statement with LOCK: executor = EXECUTOR_REF and EXECUTOR_REF() if not executor: # TODO: consider rethinking this and keeping one # executor alive until atexit() instead. executor = (Executors.sync( name="internal").with_flat_map(lambda x: x).with_timeout(None)) EXECUTOR_REF = weakref.ref(executor) return executor
def test_as_completed_with_timeout_reset_slow_caller(): # simulates slow caller, as_completed_with_timeout_reset() won't raise # spurious TimeoutError executor = Executors.thread_pool(1) ft_1 = executor.submit(sleep, 0.2) ft_2 = executor.submit(sleep, 0.3) futures = [ft_1, ft_2] for ft in as_completed_with_timeout_reset(futures, timeout=0.5): ft.result() # simulate slow caller sleep(1.5)
def test_as_completed_with_timeout_reset_raises_timeout_error(): # tests raised TimeoutError when futures are slow executor = Executors.thread_pool(1) ft_1 = executor.submit(sleep, 1) ft_2 = executor.submit(sleep, 0.5) futures = [ft_1, ft_2] with pytest.raises(TimeoutError) as exc: for _ in as_completed_with_timeout_reset(futures, timeout=0.1): pass assert str(exc.value) == "2 (of 2) futures unfinished"
def test_run(asyncio): with AsyncioExecutor(Executors.thread_pool()) as executor: f = executor.submit(sleep_then_return, 0.01, "abc") # The result should not be available yet assert_that(calling(f.result), raises(asyncio.InvalidStateError)) # Running in event loop should work result = asyncio.get_event_loop().run_until_complete(f) # It should have given the expected value, both from run_until_complete # and f.result() assert_that(result, equal_to("abc")) assert_that(result, equal_to(f.result()))
def test_or_propagate_traceback(): def inner_test_fn(): raise RuntimeError("oops") def my_test_fn(inner_fn): inner_fn() executor = Executors.thread_pool() future = f_or( executor.submit(my_test_fn), executor.submit(my_test_fn), executor.submit(my_test_fn, inner_test_fn), ) assert_in_traceback(future, "inner_test_fn")