def test_submit_during_shutdown_no_deadlock():
proceed = Event()
submit_more_done = [False]
executor = CancelOnShutdownExecutor(Executors.thread_pool(max_workers=2))
def submit_more():
proceed.wait()
assert_that(
calling(executor.submit).with_args(lambda: None),
raises(RuntimeError, "cannot schedule new futures after shutdown"),
)
submit_more_done[0] = True
futures = [executor.submit(submit_more) for _ in (1, 2, 3)]
# Let threads proceed only after shutdown has already started...
def set_soon():
time.sleep(0.10)
proceed.set()
proceed_f = Executors.thread_pool(max_workers=1).submit(set_soon)
executor.shutdown(wait=True)
proceed_f.result()
# It should have reached here without deadlocking
# All futures should have completed
assert all([f.done() for f in futures])
# And the tests in submit_more should have run
assert_that(submit_more_done[0], equal_to(True))
def test_throttle_dynamic_raises_cannot_construct():
"""Can't make a ThrottleExecutor if count immediately raises"""
error = RuntimeError("oops")
def raise_error():
raise error
with pytest.raises(RuntimeError) as exc_info:
Executors.sync().with_throttle(count=raise_error)
assert exc_info.value is error
def __init__(
self,
url,
errata,
target="cdn",
koji_source=None,
threads=4,
timeout=60 * 60 * 4,
):
"""Create a new source.
Parameters:
url (src)
Base URL of Errata Tool, e.g. "http://errata.example.com",
"https://errata.example.com:8123".
errata (str, list[str])
Advisory ID(s) to be used as push item source.
If a single string is given, multiple IDs may be
comma-separated.
target (str)
The target type used when querying Errata Tool.
The target type may influence the content produced,
depending on the Errata Tool settings.
Valid values include at least: "cdn", "rhn", "ftp".
koji_source (str)
URL of a koji source associated with this Errata Tool
instance.
threads (int)
Number of threads used for concurrent queries to Errata Tool
and koji.
timeout (int)
Number of seconds after which an error is raised, if no progress is
made during queries to Errata Tool.
"""
self._url = url
self._errata = list_argument(errata)
self._executor = Executors.thread_pool(
max_workers=threads).with_retry()
# We set aside a separate thread pool for koji so that there are separate
# queues for ET and koji calls, yet we avoid creating a new thread pool for
# each koji source.
self._koji_executor = Executors.thread_pool(
max_workers=threads).with_retry()
self._koji_cache = {}
self._koji_source_url = koji_source
self._target = target
self._timeout = timeout
self._tls = threading.local()
def mount_fstab(mounts):
"""Wait on each mount, then make sure all fstab is mounted"""
from more_executors import Executors, ExceptionRetryPolicy
def mount_path(path):
log.info(f"Waiting for '{path}' to be mounted...")
try:
run(f"mount {path}", timeout=120)
except Exception as e:
exc_type, _, _ = sys.exc_info()
log.error(f"mount of path '{path}' failed: {exc_type}: {e}")
raise e
log.info(f"Mount point '{path}' was mounted.")
MAX_MOUNT_TIMEOUT = 60 * 5
future_list = []
retry_policy = ExceptionRetryPolicy(max_attempts=40,
exponent=1.6,
sleep=1.0,
max_sleep=16.0)
with Executors.thread_pool().with_timeout(MAX_MOUNT_TIMEOUT).with_retry(
retry_policy=retry_policy) as exe:
for path in mounts:
future = exe.submit(mount_path, path)
future_list.append(future)
# Iterate over futures, checking for exceptions
for future in as_completed(future_list):
try:
future.result()
except Exception as e:
raise e
def test_submit_during_shutdown():
proceed = Event()
futures = []
submit_more_done = [False]
executor = Executors.thread_pool(max_workers=2).with_cancel_on_shutdown()
futures = [executor.submit(proceed.wait) for _ in (1, 2, 3)]
def submit_more(f):
assert_that(f, equal_to(futures[2]))
assert_that(
calling(executor.submit).with_args(lambda: None),
raises(RuntimeError, "cannot schedule new futures after shutdown"),
)
submit_more_done[0] = True
futures[2].add_done_callback(submit_more)
# Shut it down...
executor.shutdown(wait=False)
proceed.set()
# That should have cancelled futures[2]
assert_that(futures[2].cancelled())
# And the tests in submit_more should have run
assert_that(submit_more_done[0], equal_to(True))
def __init__(self,
pulp_hostname,
pulp_auth,
dry_run,
ubiconfig_filename_list=None,
ubiconfig_dir_or_url=None,
insecure=False,
workers_count=4,
output_repos=None,
**kwargs):
# legacy client implemeted in this repo, it's expected to be replaced by pubtools.pulplib.Client
self.pulp = Pulp(pulp_hostname, pulp_auth, insecure)
self._pulp_hostname = pulp_hostname
self._pulp_auth = pulp_auth
self._insecure = insecure
self._pulp_client = None
self.dry_run = dry_run
self.output_repos = output_repos
self._executor = Executors.thread_pool(
max_workers=workers_count).with_retry()
self._ubiconfig_list = None
self._ubiconfig_filename_list = ubiconfig_filename_list
self._ubiconfig_dir_or_url = ubiconfig_dir_or_url
self._content_sets = kwargs.get("content_sets", None)
self._repo_ids = kwargs.get("repo_ids", None)
self._version = kwargs.get("version", None)
self._content_set_regex = kwargs.get("content_set_regex", None)
self._ubiconfig_map = None
def test_or_with_nocancel():
calls = set()
def delayed_call(delay):
time.sleep(delay)
calls.add(delay)
return delay
executor = Executors.thread_pool(max_workers=2)
futures = [
executor.submit(delayed_call, x) for x in (0.5, 0.1, 0.2, 1.0, 1.1)
]
futures = [f_nocancel(f) for f in futures]
future = f_or(*futures)
result = future.result()
# Result comes from the future which completed first
assert result == 0.1
# Only this calls should have completed so far
assert calls == set([0.1])
# But if we wait on this...
assert futures[-1].result() == 1.1
# Then they've all completed now, thanks to nocancel
assert calls == set([0.1, 0.2, 0.5, 1.0, 1.1])
def _do_parallel_requests(self, make_request, data_items):
"""
Call given function with given data items in parallel, collect responses.
Args:
make_request (function): a function that does the actual request.
Must accept a single argument: a data item.
Must return a `requests.models.Response` object.
data_items (list): a list of arbitrary objects to be passed
individually to `make_request()`.
The number of parallel requests is defined by
`DEFAULT_REQUEST_THREADS_LIMIT` (can be overridden by the user) and
of course by the number of actually available threads.
If a response fails consistently (see `PyxisSession` for retry policy),
the execution is terminated and an informative error is raised.
See `PyxisClient._handle_json_response()` for details.
Returns:
list(dict): list of dictionaries extracted from responses.
"""
with Executors.thread_pool(max_workers=self.threads_limit).with_map(
self._handle_json_response) as executor:
futures = [
executor.submit(make_request, data) for data in data_items
]
return [f.result() for f in as_completed(futures)]
def test_and_with_nocancel():
calls = set()
error = RuntimeError("simulated error")
def delayed_call(delay):
if delay is error:
raise error
time.sleep(delay)
calls.add(delay)
return delay
executor = Executors.thread_pool(max_workers=2)
futures = [
executor.submit(delayed_call, x) for x in (0.5, error, 0.2, 1.1, 1.2)
]
futures = [f_nocancel(f) for f in futures]
future = f_and(*futures)
exception = future.exception()
# error should have been propagated
assert exception is error
# nothing else has been called yet
assert not calls
# but if we wait on the last future...
assert futures[-1].result() == 1.2
# then all calls were made thanks to nocancel
assert calls == set([0.5, 0.2, 1.1, 1.2])
def run(self):
with Executors.thread_pool(max_workers=CHECKSUM_THREADS,
name="checksummer") as exc:
for item in self.iter_input():
# Use a heuristic to try to hand off the item onto the next
# phase as quickly as possible.
#
# - in general we use a thread pool to do calculations in parallel.
#
# - but if we probably already have sums, we don't want to put
# that item onto the back of a potentially long queue where it
# may have to wait a long time, when it could be passsed on
# immediately.
#
# Hence we handle some items synchronously and others not.
if not item.blocking_checksums:
# with_checksums (probably) won't block so just do
# it immediately, thus letting the next phase get hold
# of the item more quickly.
self.put_output(self._get_sums(item))
else:
# with_checksums (probably) will block so put it onto
# the thread pool's queue.
f = exc.submit(self._get_sums, item)
self.put_future_output(f)
def test_broken_policy(caplog):
"""Executor logs without retry if policy raises exception"""
policy_error = RuntimeError("error from policy")
fn_error = RuntimeError("error from fn")
class BrokenRetryPolicy(RetryPolicy):
def should_retry(self, attempt, future):
raise policy_error
executor = Executors.thread_pool().with_retry(retry_policy=BrokenRetryPolicy())
def fn():
raise fn_error
# Start the callable
future = executor.submit(fn)
# It should fail with the error raised from the fn
assert future.exception(10.0) is fn_error
if not caplog:
# TODO: remove me when py2.6 support is dropped
return
records = caplog.records
error_log = records[-1]
# It should have logged a message about the error
assert "Exception while evaluating retry policy" in error_log.message
assert "BrokenRetryPolicy" in error_log.message
assert "error from policy" in error_log.exc_text
def test_throttle_dynamic():
throttle = [0]
executor = Executors.thread_pool(max_workers=8).with_throttle(
count=lambda: throttle[0])
tester = ThrottleTester()
futures = []
for _ in range(0, 50):
futures.append(executor.submit(tester))
# Currently, it should not be possible for any to execute
time.sleep(0.5)
assert not tester.entered
assert tester.running_count == 0
# If we raise the throttle...
throttle[0] = 1
# Then one should be able to start
assert_soon(lambda: tester.running_count == 1)
# If we raise it again...
throttle[0] = 2
# Let the first one complete so it wakes faster
tester.proceed()
# Now two should be able to start
assert_soon(lambda: tester.running_count == 2)
# If we unset throttle entirely...
throttle[0] = None
# Then all threads should be used
assert_soon(lambda: tester.running_count == 8)
# Throttle back to zero
throttle[0] = 0
# Let them all finish
tester.proceed_all()
# There should NOT be any more able to run
assert tester.running_count == 0
time.sleep(0.5)
assert tester.running_count == 0
# Raise throttle again works fine
throttle[0] = 6
assert_soon(lambda: tester.running_count == 6)
# Let them all finish
for _ in range(0, 100):
tester.proceed()
assert_soon(lambda: len(tester.exited) == 50)
def test_timeout():
def sleep_then_return(x):
time.sleep(x)
return x
with Executors.thread_pool(max_workers=2) as executor:
futures = [
executor.submit(sleep_then_return, x)
for x in [0.5, 0.5, 0.5, 0.5, 0.5]
]
futures[3] = f_nocancel(futures[3])
futures = [f_timeout(f, 0.02) for f in futures]
try:
futures[-1].result()
raise AssertionError("Should have failed")
except CancelledError:
# expected
pass
assert futures[0].result() == 0.5
assert futures[1].result() == 0.5
assert futures[2].cancelled()
assert futures[3].result() == 0.5
assert futures[4].cancelled()
def __init__(self, url, threads=4, timeout=60 * 60):
"""Create a new source.
Parameters:
url (list[str])
URL(s) of locally accessible staging directories, e.g.
``"/mnt/staging/my-content-for-push"``.
These directories must follow the documented layout for staging areas.
threads (int)
Number of threads used for concurrent loading of files.
timeout (int)
Number of seconds after which an error is raised, if no progress is
made during each step.
"""
super(StagedSource, self).__init__()
self._url = list_argument(url)
self._threads = threads
self._timeout = timeout
# Note: this executor does not have a retry.
# NFS already does a lot of its own retries.
self._executor = (Executors.thread_pool(
name="pushsource-staged", max_workers=threads).with_timeout(
timeout).with_cancel_on_shutdown())
def test_no_rebind():
bound = Executors.sync().bind(mult10)
try:
bound.bind(mult2)
raise AssertionError("Chained bind should have failed!") # pragma: no cover
except AttributeError:
pass
def test_bind_with_callable():
async_mult2 = Executors.thread_pool().bind(mult_class(2))
inputs = [0, 1, 2]
futures = [async_mult2(x) for x in inputs]
results = [f.result() for f in futures]
assert results == [0, 2, 4]
def test_bind_with_partial():
async_mult2 = Executors.thread_pool().bind(partial(mult, 2))
inputs = [0, 1, 2]
futures = [async_mult2(x) for x in inputs]
results = [f.result() for f in futures]
assert results == [0, 2, 4]
def test_single_bind():
async_mult2 = Executors.thread_pool().bind(mult2)
inputs = [0, 1, 2]
futures = [async_mult2(x) for x in inputs]
results = [f.result() for f in futures]
assert results == [0, 2, 4]
def test_flat_bind():
bound_async_mult = Executors.thread_pool().flat_bind(async_mult)
inputs = [(0, 1), (2, 3), (4, 5)]
expected_results = [0, 6, 20]
futures = [bound_async_mult(x, y) for (x, y) in inputs]
results = [f.result() for f in futures]
assert results == expected_results
def timeout_executor():
global EXECUTOR_REF # pylint: disable=global-statement
with LOCK:
executor = EXECUTOR_REF and EXECUTOR_REF()
if not executor:
executor = Executors.sync().with_flat_map(
lambda x: x).with_timeout(None)
EXECUTOR_REF = weakref.ref(executor)
return executor
def test_bind_then_map():
async_mult200 = (
Executors.thread_pool().with_map(mult10).bind(mult2).with_map(mult10)
)
inputs = [0, 1, 2]
futures = [async_mult200(x) for x in inputs]
results = [f.result() for f in futures]
assert results == [0, 200, 400]
def __init__(self, input_repos, ubi_config, workers=8):
self._input_repos = input_repos
self._ubi_config = ubi_config
# executor for this class, not adding retries because for pulp
# we use executor from pulplib
self._executor = Executors.thread_pool(max_workers=workers)
self.binary_rpms = None
self.debug_rpms = None
self.source_rpms = None
def test_with_map(asyncio):
with Executors.thread_pool().with_map(
lambda x: [x, x]).with_asyncio() as executor:
f = executor.submit(sleep_then_return, 0.01, "abc")
result = asyncio.get_event_loop().run_until_complete(f)
# MapExecutor should have worked, as usual
assert_that(result, equal_to(["abc", "abc"]))
assert_that(result, equal_to(f.result()))
def test_nocancel():
executor = Executors.thread_pool(max_workers=2)
futures = [f_nocancel(executor.submit(delay_then, x)) for x in [1, 2, 3, 4, 5]]
for f in futures:
# Should not be able to cancel it even though most
# are not started yet
assert not f.cancel()
assert [f.result() for f in futures] == [1, 2, 3, 4, 5]
def __init__(self, pulp_hostname, pulp_auth, dry_run, ubiconfig_filename_list=None,
ubiconfig_dir_or_url=None, insecure=False, workers_count=4, output_repos=None,
**kwargs):
self.pulp = Pulp(pulp_hostname, pulp_auth, insecure)
self.dry_run = dry_run
self.output_repos = output_repos
self._executor = Executors.thread_pool(max_workers=workers_count).with_retry()
self.ubiconfig_list = self._load_ubiconfig(ubiconfig_filename_list, ubiconfig_dir_or_url,
content_sets=kwargs.get('content_sets', None),
repo_ids=kwargs.get('repo_ids', None))
def timeout_executor():
global EXECUTOR_REF # pylint: disable=global-statement
with LOCK:
executor = EXECUTOR_REF and EXECUTOR_REF()
if not executor:
# TODO: consider rethinking this and keeping one
# executor alive until atexit() instead.
executor = (Executors.sync(
name="internal").with_flat_map(lambda x: x).with_timeout(None))
EXECUTOR_REF = weakref.ref(executor)
return executor
def test_as_completed_with_timeout_reset_slow_caller():
# simulates slow caller, as_completed_with_timeout_reset() won't raise
# spurious TimeoutError
executor = Executors.thread_pool(1)
ft_1 = executor.submit(sleep, 0.2)
ft_2 = executor.submit(sleep, 0.3)
futures = [ft_1, ft_2]
for ft in as_completed_with_timeout_reset(futures, timeout=0.5):
ft.result()
# simulate slow caller
sleep(1.5)
def test_as_completed_with_timeout_reset_raises_timeout_error():
# tests raised TimeoutError when futures are slow
executor = Executors.thread_pool(1)
ft_1 = executor.submit(sleep, 1)
ft_2 = executor.submit(sleep, 0.5)
futures = [ft_1, ft_2]
with pytest.raises(TimeoutError) as exc:
for _ in as_completed_with_timeout_reset(futures, timeout=0.1):
pass
assert str(exc.value) == "2 (of 2) futures unfinished"
def test_run(asyncio):
with AsyncioExecutor(Executors.thread_pool()) as executor:
f = executor.submit(sleep_then_return, 0.01, "abc")
# The result should not be available yet
assert_that(calling(f.result), raises(asyncio.InvalidStateError))
# Running in event loop should work
result = asyncio.get_event_loop().run_until_complete(f)
# It should have given the expected value, both from run_until_complete
# and f.result()
assert_that(result, equal_to("abc"))
assert_that(result, equal_to(f.result()))
def test_or_propagate_traceback():
def inner_test_fn():
raise RuntimeError("oops")
def my_test_fn(inner_fn):
inner_fn()
executor = Executors.thread_pool()
future = f_or(
executor.submit(my_test_fn),
executor.submit(my_test_fn),
executor.submit(my_test_fn, inner_test_fn),
)
assert_in_traceback(future, "inner_test_fn")
