def test_bind_with_callable(): async_mult2 = Executors.thread_pool().bind(mult_class(2)) inputs = [0, 1, 2] futures = [async_mult2(x) for x in inputs] results = [f.result() for f in futures] assert results == [0, 2, 4]
def test_bind_with_partial(): async_mult2 = Executors.thread_pool().bind(partial(mult, 2)) inputs = [0, 1, 2] futures = [async_mult2(x) for x in inputs] results = [f.result() for f in futures] assert results == [0, 2, 4]
def test_single_bind(): async_mult2 = Executors.thread_pool().bind(mult2) inputs = [0, 1, 2] futures = [async_mult2(x) for x in inputs] results = [f.result() for f in futures] assert results == [0, 2, 4]
def test_flat_bind(): bound_async_mult = Executors.thread_pool().flat_bind(async_mult) inputs = [(0, 1), (2, 3), (4, 5)] expected_results = [0, 6, 20] futures = [bound_async_mult(x, y) for (x, y) in inputs] results = [f.result() for f in futures] assert results == expected_results
def test_bind_then_map(): async_mult200 = ( Executors.thread_pool().with_map(mult10).bind(mult2).with_map(mult10) ) inputs = [0, 1, 2] futures = [async_mult200(x) for x in inputs] results = [f.result() for f in futures] assert results == [0, 200, 400]
def __init__(self, input_repos, ubi_config, workers=8): self._input_repos = input_repos self._ubi_config = ubi_config # executor for this class, not adding retries because for pulp # we use executor from pulplib self._executor = Executors.thread_pool(max_workers=workers) self.binary_rpms = None self.debug_rpms = None self.source_rpms = None
def test_with_map(asyncio): with Executors.thread_pool().with_map( lambda x: [x, x]).with_asyncio() as executor: f = executor.submit(sleep_then_return, 0.01, "abc") result = asyncio.get_event_loop().run_until_complete(f) # MapExecutor should have worked, as usual assert_that(result, equal_to(["abc", "abc"])) assert_that(result, equal_to(f.result()))
def test_nocancel(): executor = Executors.thread_pool(max_workers=2) futures = [f_nocancel(executor.submit(delay_then, x)) for x in [1, 2, 3, 4, 5]] for f in futures: # Should not be able to cancel it even though most # are not started yet assert not f.cancel() assert [f.result() for f in futures] == [1, 2, 3, 4, 5]
def __init__(self, pulp_hostname, pulp_auth, dry_run, ubiconfig_filename_list=None, ubiconfig_dir_or_url=None, insecure=False, workers_count=4, output_repos=None, **kwargs): self.pulp = Pulp(pulp_hostname, pulp_auth, insecure) self.dry_run = dry_run self.output_repos = output_repos self._executor = Executors.thread_pool(max_workers=workers_count).with_retry() self.ubiconfig_list = self._load_ubiconfig(ubiconfig_filename_list, ubiconfig_dir_or_url, content_sets=kwargs.get('content_sets', None), repo_ids=kwargs.get('repo_ids', None))
def test_as_completed_with_timeout_reset_slow_caller(): # simulates slow caller, as_completed_with_timeout_reset() won't raise # spurious TimeoutError executor = Executors.thread_pool(1) ft_1 = executor.submit(sleep, 0.2) ft_2 = executor.submit(sleep, 0.3) futures = [ft_1, ft_2] for ft in as_completed_with_timeout_reset(futures, timeout=0.5): ft.result() # simulate slow caller sleep(1.5)
def test_as_completed_with_timeout_reset_raises_timeout_error(): # tests raised TimeoutError when futures are slow executor = Executors.thread_pool(1) ft_1 = executor.submit(sleep, 1) ft_2 = executor.submit(sleep, 0.5) futures = [ft_1, ft_2] with pytest.raises(TimeoutError) as exc: for _ in as_completed_with_timeout_reset(futures, timeout=0.1): pass assert str(exc.value) == "2 (of 2) futures unfinished"
def test_run(asyncio): with AsyncioExecutor(Executors.thread_pool()) as executor: f = executor.submit(sleep_then_return, 0.01, "abc") # The result should not be available yet assert_that(calling(f.result), raises(asyncio.InvalidStateError)) # Running in event loop should work result = asyncio.get_event_loop().run_until_complete(f) # It should have given the expected value, both from run_until_complete # and f.result() assert_that(result, equal_to("abc")) assert_that(result, equal_to(f.result()))
def test_success(): with TimeoutExecutor(Executors.thread_pool(), TIMEOUT) as executor: f = executor.submit(sleep_and_return, TIMEOUT / 10, "abc") # Should complete successfully assert_that(f.result(), equal_to("abc")) assert_that(f.done()) assert_that(not f.cancelled()) # Should remain completed successfully through the timeout time.sleep(TIMEOUT * 2) assert_that(f.done()) assert_that(not f.cancelled())
def test_or_propagate_traceback(): def inner_test_fn(): raise RuntimeError("oops") def my_test_fn(inner_fn): inner_fn() executor = Executors.thread_pool() future = f_or( executor.submit(my_test_fn), executor.submit(my_test_fn), executor.submit(my_test_fn, inner_test_fn), ) assert_in_traceback(future, "inner_test_fn")
def test_map(): sem = Semaphore(0) with Executors.thread_pool() as exc: # This future cannot possibly proceed until we unblock the semaphore. f = exc.submit(sem.acquire) f = f_proxy(f) # If bug #278 exists, we will hang here indefinitely. f = f_map(f, lambda _: 123) # If bug is fixed, future is still not evaluated. # Let it proceed now. sem.release() assert f.result() == 123
def __init__(self, threads, url, **retry_args): self._executor = (Executors.thread_pool( name="pushsource-errata-client", max_workers=threads).with_retry( **retry_args).with_cancel_on_shutdown()) self._url = url self._tls = threading.local() self._get_advisory_cdn_metadata = partial(self._call_et, "get_advisory_cdn_metadata") self._get_advisory_cdn_file_list = partial( self._call_et, "get_advisory_cdn_file_list") self._get_advisory_cdn_docker_file_list = partial( self._call_et, "get_advisory_cdn_docker_file_list") self._get_ftp_paths = partial(self._call_et, "get_ftp_paths")
def test_and_order_async(): executor = Executors.thread_pool(max_workers=2) def delay_then(delay, value): time.sleep(delay) return value f_inputs = [ executor.submit(delay_then, 0.1, 123), executor.submit(delay_then, 0.05, 456), ] future = f_and(*f_inputs) assert_future_equal(future, 123)
def __init__(self, pulp_hostname, pulp_auth, dry_run, ubiconfig_filename_list=None, ubiconfig_dir_or_url=None, insecure=False, workers_count=4): self.ubiconfig_list = self._load_ubiconfig(ubiconfig_filename_list, ubiconfig_dir_or_url) self.pulp = Pulp(pulp_hostname, pulp_auth, insecure) self.dry_run = dry_run self._executor = Executors.thread_pool( max_workers=workers_count).with_retry()
def __init__( self, access_id=None, access_key=None, session_token=None, default_region=None, ): with mock.patch("boto3.Session") as mocked_sessions: mocked_sessions.return_value = mocked_sessions Client.__init__( self, access_id, access_key, session_token, default_region ) # Only use one retry attempt on tests self._executor = Executors.thread_pool(max_workers=4).with_retry( max_attempts=1 )
def test_cancels(): proceed = Event() count = 1000 running_count = 0 canceled_count = 0 exception_count = 0 completed_count = 0 futures = [] executor = Executors.thread_pool(max_workers=2).with_cancel_on_shutdown() futures = [executor.submit(proceed.wait) for _ in range(0, count)] # I'm using wait=False here since otherwise it could block on the 2 threads # currently in progress to finish their work items. I can't see a way to # make the test fully synchronized, and using wait=True, without deadlock. executor.shutdown(wait=False) # Now let those two threads complete (if they've started) proceed.set() # Collect status of the futures. for future in futures: if future.running(): running_count += 1 elif future.cancelled(): canceled_count += 1 elif future.exception(): exception_count += 1 elif future.done(): completed_count += 1 # No futures should have failed assert_that(exception_count, equal_to(0)) # Could have been anywhere from 0..2 futures running assert_that(running_count, is_in((0, 1, 2))) # Could have been anywhere from 0..2 futures completed assert_that(completed_count, is_in((0, 1, 2))) # All others should have been cancelled assert_that(canceled_count, less_than_or_equal_to(count)) assert_that(canceled_count, greater_than_or_equal_to(count - 2)) # Harmless to call shutdown again executor.shutdown()
def test_cancel_pending(): called = [] with Executors.thread_pool( max_workers=1).with_timeout(TIMEOUT) as executor: f1 = executor.submit(sleep_and_return, 1.0, "abc") f2 = executor.submit(called.append, True) # f2 should be cancelled while f1 was still running assert_that(calling(f2.result), raises(CancelledError)) # And it should not have been invoked at all assert_that(called, equal_to([])) # Meanwhile, f1 was not cancelable at the timeout, so it should have # completed assert_that(f1.result(), equal_to("abc"))
def test_and_cancels(): calls = set() error = RuntimeError("simulated error") def delayed_call(delay): if delay is error: raise error time.sleep(delay) calls.add(delay) return delay executor = Executors.thread_pool(max_workers=2) with executor: futures = [ executor.submit(delayed_call, x) for x in (0.5, error, 0.2, 2.0, 3.0) ] future = f_and(*futures) exception = future.exception() # error should have been propagated assert exception is error # 0.5 definitely should have been invoked assert 0.5 in calls # up to 1 more of the calls might have succeeded, # but we can't say which. # Why: because the cancels race with the thread pool, # once 'error' completes then f_and and the thread pool # are both trying to grab the next futures ASAP, # and either cancel or run could win. assert len(calls) in (1, 2) # This could not have been cancelled since it was # submitted earlier than the terminal assert futures[0].done() # The future which terminated the and() assert futures[1].done() # at least 2 of the remaining futures should have been cancelled assert len([f for f in futures if f.cancelled()]) >= 2
def test_and_propagate_traceback(): def inner_test_fn(): raise RuntimeError("oops") def my_test_fn(inner_fn=None): if inner_fn: inner_fn() return True executor = Executors.thread_pool() futures = [ executor.submit(my_test_fn), executor.submit(my_test_fn), executor.submit(my_test_fn, inner_fn=inner_test_fn), executor.submit(my_test_fn), ] future = f_and(*futures) assert_in_traceback(future, "inner_test_fn")
def test_throttle(block): THREADS = 8 COUNT = 3 samples = [] running_now = [] lock = Lock() def record(x): with lock: running_now.append(x) samples.append(running_now[:]) # Need to ensure the function takes some time to run # so the futures don't complete as fast as we submit them time.sleep(0.001) with lock: running_now.remove(x) futures = [] executor = ThrottleExecutor(Executors.thread_pool(max_workers=THREADS), count=COUNT, block=block) with executor: for i in range(0, 1000): future = executor.submit(record, i) futures.append(future) # They should all be able to complete for future in futures: future.result(30.0) # Now have a look at running counts max_len = 0 for i, sample in enumerate(samples): # There should never have been more futures running than the limit sample_len = len(sample) assert_that(sample_len, less_than_or_equal_to(COUNT), "failed at step %s" % i) max_len = max(max_len, sample_len) # It should have been able to run up to the limit too assert_that(max_len, equal_to(COUNT))
def test_throttle_dynamic_raises_uses_previous(caplog): """ThrottleExecutor uses last throttle value if the callable raises""" called = [] def throttle_count(): if called: raise RuntimeError("oops") called.append(None) return 1 tester = ThrottleTester() executor = Executors.thread_pool(max_workers=2).with_throttle( count=throttle_count) f1 = executor.submit(tester) f2 = executor.submit(tester) f3 = executor.submit(tester) # There should be one running assert_soon(lambda: tester.running_count == 1) tester.proceed() # First future should be able to complete f1.result(10.0) # There should still one running assert_soon(lambda: tester.running_count == 1) tester.proceed() tester.proceed() # They can all complete f2.result(10.0) f3.result(10.0) if caplog: # It should have logged a warning about the error message = "\n".join(caplog.messages) assert "Error evaluating throttle count" in message
def __init__(self, url, max_retry_sleep=None, **kwargs): """Create a new UD cache flush client. Arguments: url (str) Base URL of UD cache flushing API. max_retry_sleep (float) Max number of seconds to sleep between retries. Mainly provided so that tests can reduce the time needed to retry. kwargs Remaining arguments are used to initialize the requests.Session() used within this class (e.g. "verify", "auth"). """ self._url = url self._tls = threading.local() retry_args = {} if max_retry_sleep: retry_args["max_sleep"] = max_retry_sleep self._session_attrs = kwargs self._executor = (Executors.thread_pool().with_map( self._check_http_response).with_retry(**retry_args))
def __init__( self, access_id=None, access_key=None, session_token=None, default_region=None, workers_count=4, retry_count=3, ): self._access_key_id = access_id self._access_key = access_key self._session_token = session_token self._default_region = default_region self._session = boto3.Session( aws_access_key_id=self._access_key_id, aws_secret_access_key=self._access_key, aws_session_token=self._session_token, region_name=self._default_region, ) self._executor = Executors.thread_pool( max_workers=workers_count).with_retry(max_attempts=retry_count)
def test_or_cancels(): calls = set() def delayed_call(delay): time.sleep(delay) calls.add(delay) return delay executor = Executors.thread_pool(max_workers=2) with executor: futures = [ executor.submit(delayed_call, x) for x in (0.5, 0.1, 0.2, 2.0, 3.0) ] future = f_or(*futures) result = future.result() # Result comes from the future which completed first assert result == 0.1 assert 0.1 in calls # There could have been one more call, since f_or # cancelling other futures races with threadpool trying # to pick them up assert len(calls) in (2, 3) # This could not have been cancelled since it was # submitted earlier than the winner assert futures[0].done() # The winner 0.1 assert futures[1].done() # at least 2 of the remaining futures should have been cancelled assert len([f for f in futures if f.cancelled()]) >= 2
def executor(): yield Executors.thread_pool(max_workers=1).with_retry()
def __init__( self, url, dest=None, rpm=None, module_build=None, signing_key=None, basedir=None, threads=4, timeout=60 * 30, cache=None, executor=None, ): """Create a new source. Parameters: url (str) URL of the XML-RPC endpoint for koji hub. dest (str, list[str]) The destination(s) to fill in for push items created by this source. If omitted, all push items have empty destinations. rpm (list[str, int]) RPM identifier(s). Can be koji IDs (integers) or filenames. The source will yield all RPMs identified by this list. module_build (list[str, int]) Build identifier(s). Can be koji IDs (integers) or build NVRs. The source will yield all modulemd files belonging to these build(s). signing_key (list[str]) GPG signing key ID(s). If provided, content must be signed using one of the provided keys. Include ``None`` if unsigned should also be permitted. Keys should be listed in the order of preference. basedir (str) Base directory of koji content (e.g. /mnt/koji). This directory must be readable locally. threads (int) Number of threads used for concurrent queries to koji. timeout (int) Number of seconds after which an error is raised, if no progress is made during queries to koji. cache (dict) A cache used to retain the results of XML-RPC calls, to avoid repeated calls. Providing a cache only gives a benefit if the current process will create more than a single instance of KojiSource. In this case, some calls may be avoided by passing the same cache to each instance. executor (concurrent.futures.Executor) A custom executor used to submit calls to koji. """ self._url = url self._rpm = [try_int(x) for x in list_argument(rpm)] self._module_build = [try_int(x) for x in list_argument(module_build)] self._signing_key = list_argument(signing_key) self._dest = list_argument(dest) self._timeout = timeout self._pathinfo = koji.PathInfo(basedir) self._cache = {} if cache is None else cache self._threads = threads self._executor = ( executor or Executors.thread_pool(max_workers=threads).with_retry())