def test_bind_with_callable():
async_mult2 = Executors.thread_pool().bind(mult_class(2))
inputs = [0, 1, 2]
futures = [async_mult2(x) for x in inputs]
results = [f.result() for f in futures]
assert results == [0, 2, 4]
def test_bind_with_partial():
async_mult2 = Executors.thread_pool().bind(partial(mult, 2))
inputs = [0, 1, 2]
futures = [async_mult2(x) for x in inputs]
results = [f.result() for f in futures]
assert results == [0, 2, 4]
def test_single_bind():
async_mult2 = Executors.thread_pool().bind(mult2)
inputs = [0, 1, 2]
futures = [async_mult2(x) for x in inputs]
results = [f.result() for f in futures]
assert results == [0, 2, 4]
def test_flat_bind():
bound_async_mult = Executors.thread_pool().flat_bind(async_mult)
inputs = [(0, 1), (2, 3), (4, 5)]
expected_results = [0, 6, 20]
futures = [bound_async_mult(x, y) for (x, y) in inputs]
results = [f.result() for f in futures]
assert results == expected_results
def test_bind_then_map():
async_mult200 = (
Executors.thread_pool().with_map(mult10).bind(mult2).with_map(mult10)
)
inputs = [0, 1, 2]
futures = [async_mult200(x) for x in inputs]
results = [f.result() for f in futures]
assert results == [0, 200, 400]
def __init__(self, input_repos, ubi_config, workers=8):
self._input_repos = input_repos
self._ubi_config = ubi_config
# executor for this class, not adding retries because for pulp
# we use executor from pulplib
self._executor = Executors.thread_pool(max_workers=workers)
self.binary_rpms = None
self.debug_rpms = None
self.source_rpms = None
def test_with_map(asyncio):
with Executors.thread_pool().with_map(
lambda x: [x, x]).with_asyncio() as executor:
f = executor.submit(sleep_then_return, 0.01, "abc")
result = asyncio.get_event_loop().run_until_complete(f)
# MapExecutor should have worked, as usual
assert_that(result, equal_to(["abc", "abc"]))
assert_that(result, equal_to(f.result()))
def test_nocancel():
executor = Executors.thread_pool(max_workers=2)
futures = [f_nocancel(executor.submit(delay_then, x)) for x in [1, 2, 3, 4, 5]]
for f in futures:
# Should not be able to cancel it even though most
# are not started yet
assert not f.cancel()
assert [f.result() for f in futures] == [1, 2, 3, 4, 5]
def __init__(self, pulp_hostname, pulp_auth, dry_run, ubiconfig_filename_list=None,
ubiconfig_dir_or_url=None, insecure=False, workers_count=4, output_repos=None,
**kwargs):
self.pulp = Pulp(pulp_hostname, pulp_auth, insecure)
self.dry_run = dry_run
self.output_repos = output_repos
self._executor = Executors.thread_pool(max_workers=workers_count).with_retry()
self.ubiconfig_list = self._load_ubiconfig(ubiconfig_filename_list, ubiconfig_dir_or_url,
content_sets=kwargs.get('content_sets', None),
repo_ids=kwargs.get('repo_ids', None))
def test_as_completed_with_timeout_reset_slow_caller():
# simulates slow caller, as_completed_with_timeout_reset() won't raise
# spurious TimeoutError
executor = Executors.thread_pool(1)
ft_1 = executor.submit(sleep, 0.2)
ft_2 = executor.submit(sleep, 0.3)
futures = [ft_1, ft_2]
for ft in as_completed_with_timeout_reset(futures, timeout=0.5):
ft.result()
# simulate slow caller
sleep(1.5)
def test_as_completed_with_timeout_reset_raises_timeout_error():
# tests raised TimeoutError when futures are slow
executor = Executors.thread_pool(1)
ft_1 = executor.submit(sleep, 1)
ft_2 = executor.submit(sleep, 0.5)
futures = [ft_1, ft_2]
with pytest.raises(TimeoutError) as exc:
for _ in as_completed_with_timeout_reset(futures, timeout=0.1):
pass
assert str(exc.value) == "2 (of 2) futures unfinished"
def test_run(asyncio):
with AsyncioExecutor(Executors.thread_pool()) as executor:
f = executor.submit(sleep_then_return, 0.01, "abc")
# The result should not be available yet
assert_that(calling(f.result), raises(asyncio.InvalidStateError))
# Running in event loop should work
result = asyncio.get_event_loop().run_until_complete(f)
# It should have given the expected value, both from run_until_complete
# and f.result()
assert_that(result, equal_to("abc"))
assert_that(result, equal_to(f.result()))
def test_success():
with TimeoutExecutor(Executors.thread_pool(), TIMEOUT) as executor:
f = executor.submit(sleep_and_return, TIMEOUT / 10, "abc")
# Should complete successfully
assert_that(f.result(), equal_to("abc"))
assert_that(f.done())
assert_that(not f.cancelled())
# Should remain completed successfully through the timeout
time.sleep(TIMEOUT * 2)
assert_that(f.done())
assert_that(not f.cancelled())
def test_or_propagate_traceback():
def inner_test_fn():
raise RuntimeError("oops")
def my_test_fn(inner_fn):
inner_fn()
executor = Executors.thread_pool()
future = f_or(
executor.submit(my_test_fn),
executor.submit(my_test_fn),
executor.submit(my_test_fn, inner_test_fn),
)
assert_in_traceback(future, "inner_test_fn")
def test_map():
sem = Semaphore(0)
with Executors.thread_pool() as exc:
# This future cannot possibly proceed until we unblock the semaphore.
f = exc.submit(sem.acquire)
f = f_proxy(f)
# If bug #278 exists, we will hang here indefinitely.
f = f_map(f, lambda _: 123)
# If bug is fixed, future is still not evaluated.
# Let it proceed now.
sem.release()
assert f.result() == 123
def __init__(self, threads, url, **retry_args):
self._executor = (Executors.thread_pool(
name="pushsource-errata-client", max_workers=threads).with_retry(
**retry_args).with_cancel_on_shutdown())
self._url = url
self._tls = threading.local()
self._get_advisory_cdn_metadata = partial(self._call_et,
"get_advisory_cdn_metadata")
self._get_advisory_cdn_file_list = partial(
self._call_et, "get_advisory_cdn_file_list")
self._get_advisory_cdn_docker_file_list = partial(
self._call_et, "get_advisory_cdn_docker_file_list")
self._get_ftp_paths = partial(self._call_et, "get_ftp_paths")
def test_and_order_async():
executor = Executors.thread_pool(max_workers=2)
def delay_then(delay, value):
time.sleep(delay)
return value
f_inputs = [
executor.submit(delay_then, 0.1, 123),
executor.submit(delay_then, 0.05, 456),
]
future = f_and(*f_inputs)
assert_future_equal(future, 123)
def __init__(self,
pulp_hostname,
pulp_auth,
dry_run,
ubiconfig_filename_list=None,
ubiconfig_dir_or_url=None,
insecure=False,
workers_count=4):
self.ubiconfig_list = self._load_ubiconfig(ubiconfig_filename_list,
ubiconfig_dir_or_url)
self.pulp = Pulp(pulp_hostname, pulp_auth, insecure)
self.dry_run = dry_run
self._executor = Executors.thread_pool(
max_workers=workers_count).with_retry()
def __init__(
self,
access_id=None,
access_key=None,
session_token=None,
default_region=None,
):
with mock.patch("boto3.Session") as mocked_sessions:
mocked_sessions.return_value = mocked_sessions
Client.__init__(
self, access_id, access_key, session_token, default_region
)
# Only use one retry attempt on tests
self._executor = Executors.thread_pool(max_workers=4).with_retry(
max_attempts=1
)
def test_cancels():
proceed = Event()
count = 1000
running_count = 0
canceled_count = 0
exception_count = 0
completed_count = 0
futures = []
executor = Executors.thread_pool(max_workers=2).with_cancel_on_shutdown()
futures = [executor.submit(proceed.wait) for _ in range(0, count)]
# I'm using wait=False here since otherwise it could block on the 2 threads
# currently in progress to finish their work items. I can't see a way to
# make the test fully synchronized, and using wait=True, without deadlock.
executor.shutdown(wait=False)
# Now let those two threads complete (if they've started)
proceed.set()
# Collect status of the futures.
for future in futures:
if future.running():
running_count += 1
elif future.cancelled():
canceled_count += 1
elif future.exception():
exception_count += 1
elif future.done():
completed_count += 1
# No futures should have failed
assert_that(exception_count, equal_to(0))
# Could have been anywhere from 0..2 futures running
assert_that(running_count, is_in((0, 1, 2)))
# Could have been anywhere from 0..2 futures completed
assert_that(completed_count, is_in((0, 1, 2)))
# All others should have been cancelled
assert_that(canceled_count, less_than_or_equal_to(count))
assert_that(canceled_count, greater_than_or_equal_to(count - 2))
# Harmless to call shutdown again
executor.shutdown()
def test_cancel_pending():
called = []
with Executors.thread_pool(
max_workers=1).with_timeout(TIMEOUT) as executor:
f1 = executor.submit(sleep_and_return, 1.0, "abc")
f2 = executor.submit(called.append, True)
# f2 should be cancelled while f1 was still running
assert_that(calling(f2.result), raises(CancelledError))
# And it should not have been invoked at all
assert_that(called, equal_to([]))
# Meanwhile, f1 was not cancelable at the timeout, so it should have
# completed
assert_that(f1.result(), equal_to("abc"))
def test_and_cancels():
calls = set()
error = RuntimeError("simulated error")
def delayed_call(delay):
if delay is error:
raise error
time.sleep(delay)
calls.add(delay)
return delay
executor = Executors.thread_pool(max_workers=2)
with executor:
futures = [
executor.submit(delayed_call, x)
for x in (0.5, error, 0.2, 2.0, 3.0)
]
future = f_and(*futures)
exception = future.exception()
# error should have been propagated
assert exception is error
# 0.5 definitely should have been invoked
assert 0.5 in calls
# up to 1 more of the calls might have succeeded,
# but we can't say which.
# Why: because the cancels race with the thread pool,
# once 'error' completes then f_and and the thread pool
# are both trying to grab the next futures ASAP,
# and either cancel or run could win.
assert len(calls) in (1, 2)
# This could not have been cancelled since it was
# submitted earlier than the terminal
assert futures[0].done()
# The future which terminated the and()
assert futures[1].done()
# at least 2 of the remaining futures should have been cancelled
assert len([f for f in futures if f.cancelled()]) >= 2
def test_and_propagate_traceback():
def inner_test_fn():
raise RuntimeError("oops")
def my_test_fn(inner_fn=None):
if inner_fn:
inner_fn()
return True
executor = Executors.thread_pool()
futures = [
executor.submit(my_test_fn),
executor.submit(my_test_fn),
executor.submit(my_test_fn, inner_fn=inner_test_fn),
executor.submit(my_test_fn),
]
future = f_and(*futures)
assert_in_traceback(future, "inner_test_fn")
def test_throttle(block):
THREADS = 8
COUNT = 3
samples = []
running_now = []
lock = Lock()
def record(x):
with lock:
running_now.append(x)
samples.append(running_now[:])
# Need to ensure the function takes some time to run
# so the futures don't complete as fast as we submit them
time.sleep(0.001)
with lock:
running_now.remove(x)
futures = []
executor = ThrottleExecutor(Executors.thread_pool(max_workers=THREADS),
count=COUNT,
block=block)
with executor:
for i in range(0, 1000):
future = executor.submit(record, i)
futures.append(future)
# They should all be able to complete
for future in futures:
future.result(30.0)
# Now have a look at running counts
max_len = 0
for i, sample in enumerate(samples):
# There should never have been more futures running than the limit
sample_len = len(sample)
assert_that(sample_len, less_than_or_equal_to(COUNT),
"failed at step %s" % i)
max_len = max(max_len, sample_len)
# It should have been able to run up to the limit too
assert_that(max_len, equal_to(COUNT))
def test_throttle_dynamic_raises_uses_previous(caplog):
"""ThrottleExecutor uses last throttle value if the callable raises"""
called = []
def throttle_count():
if called:
raise RuntimeError("oops")
called.append(None)
return 1
tester = ThrottleTester()
executor = Executors.thread_pool(max_workers=2).with_throttle(
count=throttle_count)
f1 = executor.submit(tester)
f2 = executor.submit(tester)
f3 = executor.submit(tester)
# There should be one running
assert_soon(lambda: tester.running_count == 1)
tester.proceed()
# First future should be able to complete
f1.result(10.0)
# There should still one running
assert_soon(lambda: tester.running_count == 1)
tester.proceed()
tester.proceed()
# They can all complete
f2.result(10.0)
f3.result(10.0)
if caplog:
# It should have logged a warning about the error
message = "\n".join(caplog.messages)
assert "Error evaluating throttle count" in message
def __init__(self, url, max_retry_sleep=None, **kwargs):
"""Create a new UD cache flush client.
Arguments:
url (str)
Base URL of UD cache flushing API.
max_retry_sleep (float)
Max number of seconds to sleep between retries.
Mainly provided so that tests can reduce the time needed to retry.
kwargs
Remaining arguments are used to initialize the requests.Session()
used within this class (e.g. "verify", "auth").
"""
self._url = url
self._tls = threading.local()
retry_args = {}
if max_retry_sleep:
retry_args["max_sleep"] = max_retry_sleep
self._session_attrs = kwargs
self._executor = (Executors.thread_pool().with_map(
self._check_http_response).with_retry(**retry_args))
def __init__(
self,
access_id=None,
access_key=None,
session_token=None,
default_region=None,
workers_count=4,
retry_count=3,
):
self._access_key_id = access_id
self._access_key = access_key
self._session_token = session_token
self._default_region = default_region
self._session = boto3.Session(
aws_access_key_id=self._access_key_id,
aws_secret_access_key=self._access_key,
aws_session_token=self._session_token,
region_name=self._default_region,
)
self._executor = Executors.thread_pool(
max_workers=workers_count).with_retry(max_attempts=retry_count)
def test_or_cancels():
calls = set()
def delayed_call(delay):
time.sleep(delay)
calls.add(delay)
return delay
executor = Executors.thread_pool(max_workers=2)
with executor:
futures = [
executor.submit(delayed_call, x) for x in (0.5, 0.1, 0.2, 2.0, 3.0)
]
future = f_or(*futures)
result = future.result()
# Result comes from the future which completed first
assert result == 0.1
assert 0.1 in calls
# There could have been one more call, since f_or
# cancelling other futures races with threadpool trying
# to pick them up
assert len(calls) in (2, 3)
# This could not have been cancelled since it was
# submitted earlier than the winner
assert futures[0].done()
# The winner 0.1
assert futures[1].done()
# at least 2 of the remaining futures should have been cancelled
assert len([f for f in futures if f.cancelled()]) >= 2
def executor():
yield Executors.thread_pool(max_workers=1).with_retry()
def __init__(
self,
url,
dest=None,
rpm=None,
module_build=None,
signing_key=None,
basedir=None,
threads=4,
timeout=60 * 30,
cache=None,
executor=None,
):
"""Create a new source.
Parameters:
url (str)
URL of the XML-RPC endpoint for koji hub.
dest (str, list[str])
The destination(s) to fill in for push items created by this
source. If omitted, all push items have empty destinations.
rpm (list[str, int])
RPM identifier(s). Can be koji IDs (integers) or filenames.
The source will yield all RPMs identified by this list.
module_build (list[str, int])
Build identifier(s). Can be koji IDs (integers) or build NVRs.
The source will yield all modulemd files belonging to these
build(s).
signing_key (list[str])
GPG signing key ID(s). If provided, content must be signed
using one of the provided keys. Include ``None`` if unsigned
should also be permitted.
Keys should be listed in the order of preference.
basedir (str)
Base directory of koji content (e.g. /mnt/koji).
This directory must be readable locally.
threads (int)
Number of threads used for concurrent queries to koji.
timeout (int)
Number of seconds after which an error is raised, if no progress is
made during queries to koji.
cache (dict)
A cache used to retain the results of XML-RPC calls, to avoid
repeated calls.
Providing a cache only gives a benefit if the current process
will create more than a single instance of KojiSource. In this
case, some calls may be avoided by passing the same cache
to each instance.
executor (concurrent.futures.Executor)
A custom executor used to submit calls to koji.
"""
self._url = url
self._rpm = [try_int(x) for x in list_argument(rpm)]
self._module_build = [try_int(x) for x in list_argument(module_build)]
self._signing_key = list_argument(signing_key)
self._dest = list_argument(dest)
self._timeout = timeout
self._pathinfo = koji.PathInfo(basedir)
self._cache = {} if cache is None else cache
self._threads = threads
self._executor = (
executor
or Executors.thread_pool(max_workers=threads).with_retry())
