def test_future_set_callback_empty():
future_set = FutureSet([])
callback = mock.Mock()
future_set.add_done_callback(callback)
assert callback.call_count == 1
assert callback.call_args == mock.call(future_set)
def test_future_set_callback_empty():
future_set = FutureSet([])
callback = mock.Mock()
future_set.add_done_callback(callback)
assert callback.call_count == 1
assert callback.call_args == mock.call(future_set)
def test_future_broken_callback():
future_set = FutureSet([])
callback = mock.Mock(side_effect=Exception("Boom!"))
try:
future_set.add_done_callback(callback)
except Exception:
assert False, "should not raise"
assert callback.call_count == 1
assert callback.call_args == mock.call(future_set)
def test_future_broken_callback():
future_set = FutureSet([])
callback = mock.Mock(side_effect=Exception('Boom!'))
try:
future_set.add_done_callback(callback)
except Exception:
assert False, 'should not raise'
assert callback.call_count == 1
assert callback.call_args == mock.call(future_set)
def test_future_set_callback_error():
future_set = FutureSet([Future() for i in xrange(3)])
callback = mock.Mock()
future_set.add_done_callback(callback)
for i, future in enumerate(list(future_set)):
assert callback.call_count == 0
future.set_exception(Exception)
assert callback.call_count == 1
assert callback.call_args == mock.call(future_set)
other_callback = mock.Mock()
future_set.add_done_callback(other_callback)
assert other_callback.call_count == 1
assert other_callback.call_args == mock.call(future_set)
def test_future_set_callback_success():
future_set = FutureSet([Future() for i in xrange(3)])
callback = mock.Mock()
future_set.add_done_callback(callback)
for i, future in enumerate(list(future_set)):
assert callback.call_count == 0
future.set_result(True)
assert callback.call_count == 1
assert callback.call_args == mock.call(future_set)
other_callback = mock.Mock()
future_set.add_done_callback(other_callback)
assert other_callback.call_count == 1
assert other_callback.call_args == mock.call(future_set)
def test_future_set_callback_error():
future_set = FutureSet([Future() for i in range(3)])
callback = mock.Mock()
future_set.add_done_callback(callback)
for i, future in enumerate(list(future_set)):
assert callback.call_count == 0
future.set_exception(Exception)
assert callback.call_count == 1
assert callback.call_args == mock.call(future_set)
other_callback = mock.Mock()
future_set.add_done_callback(other_callback)
assert other_callback.call_count == 1
assert other_callback.call_args == mock.call(future_set)
def test_future_set_callback_success():
future_set = FutureSet([Future() for i in range(3)])
callback = mock.Mock()
future_set.add_done_callback(callback)
for i, future in enumerate(list(future_set)):
assert callback.call_count == 0
future.set_result(True)
assert callback.call_count == 1
assert callback.call_args == mock.call(future_set)
other_callback = mock.Mock()
future_set.add_done_callback(other_callback)
assert other_callback.call_count == 1
assert other_callback.call_args == mock.call(future_set)
def execute(*args: Sequence[Any], **kwargs: Mapping[str, Any]) -> Any:
context = type(self).__state.context
# If there is no context object already set in the thread local
# state, we are entering the delegator for the first time and need
# to create a new context.
if context is None:
from sentry.app import env # avoids a circular import
context = Context(env.request, {})
# If this thread already has an active backend for this base class,
# we can safely call that backend synchronously without delegating.
if self.base in context.backends:
backend = context.backends[self.base]
return getattr(backend, attribute_name)(*args, **kwargs)
# Binding the call arguments to named arguments has two benefits:
# 1. These values always be passed in the same form to the selector
# function and callback, regardless of how they were passed to
# the method itself (as positional arguments, keyword arguments,
# etc.)
# 2. This ensures that the given arguments are those supported by
# the base backend itself, which should be a common subset of
# arguments that are supported by all backends.
callargs = inspect.getcallargs(base_value, None, *args, **kwargs)
selected_backend_names = list(self.selector(context, attribute_name, callargs))
if not len(selected_backend_names) > 0:
raise self.InvalidBackend("No backends returned by selector!")
# Ensure that the primary backend is actually registered -- we
# don't want to schedule any work on the secondaries if the primary
# request is going to fail anyway.
if selected_backend_names[0] not in self.backends:
raise self.InvalidBackend(
f"{selected_backend_names[0]!r} is not a registered backend."
)
def call_backend_method(context: Context, backend: Service, is_primary: bool) -> Any:
# Update the thread local state in the executor to the provided
# context object. This allows the context to be propagated
# across different threads.
assert type(self).__state.context is None
type(self).__state.context = context
# Ensure that we haven't somehow accidentally entered a context
# where the backend we're calling has already been marked as
# active (or worse, some other backend is already active.)
base = self.base
assert base not in context.backends
# Mark the backend as active.
context.backends[base] = backend
try:
return getattr(backend, attribute_name)(*args, **kwargs)
except Exception as e:
# If this isn't the primary backend, we log any unexpected
# exceptions so that they don't pass by unnoticed. (Any
# exceptions raised by the primary backend aren't logged
# here, since it's assumed that the caller will log them
# from the calling thread.)
if not is_primary:
expected_raises = getattr(base_value, "__raises__", [])
if not expected_raises or not isinstance(e, tuple(expected_raises)):
logger.warning(
"%s caught in executor while calling %r on %s.",
type(e).__name__,
attribute_name,
type(backend).__name__,
exc_info=True,
)
raise
finally:
type(self).__state.context = None
# Enqueue all of the secondary backend requests first since these
# are non-blocking queue insertions. (Since the primary backend
# executor queue insertion can block, if that queue was full the
# secondary requests would have to wait unnecessarily to be queued
# until the after the primary request can be enqueued.)
# NOTE: If the same backend is both the primary backend *and* in
# the secondary backend list -- this is unlikely, but possible --
# this means that one of the secondary requests will be queued and
# executed before the primary request is queued. This is such a
# strange usage pattern that I don't think it's worth optimizing
# for.)
results = [None] * len(selected_backend_names)
for i, backend_name in enumerate(selected_backend_names[1:], 1):
try:
backend, executor = self.backends[backend_name]
except KeyError:
logger.warning(
"%r is not a registered backend and will be ignored.",
backend_name,
exc_info=True,
)
else:
results[i] = executor.submit(
functools.partial(
call_backend_method, context.copy(), backend, is_primary=False
),
priority=1,
block=False,
)
# The primary backend is scheduled last since it may block the
# calling thread. (We don't have to protect this from ``KeyError``
# since we already ensured that the primary backend exists.)
backend, executor = self.backends[selected_backend_names[0]]
results[0] = executor.submit(
functools.partial(call_backend_method, context.copy(), backend, is_primary=True),
priority=0,
block=True,
)
if self.callback is not None:
FutureSet([_f for _f in results if _f]).add_done_callback(
lambda *a, **k: self.callback(
context, attribute_name, callargs, selected_backend_names, results
)
)
result: TimedFuture = results[0]
return result.result()
def execute(*args, **kwargs):
# If this thread already has an active backend for this base class,
# we can safely call that backend synchronously without delegating.
if self in self.state.backends:
backend = type(self).state.backends[self.__backend_base]
return getattr(backend, attribute_name)(*args, **kwargs)
# Binding the call arguments to named arguments has two benefits:
# 1. These values always be passed in the same form to the selector
# function and callback, regardless of how they were passed to
# the method itself (as positional arguments, keyword arguments,
# etc.)
# 2. This ensures that the given arguments are those supported by
# the base backend itself, which should be a common subset of
# arguments that are supported by all backends.
callargs = inspect.getcallargs(base_value, None, *args, **kwargs)
selected_backend_names = list(
self.__selector_func(attribute_name, callargs))
if not len(selected_backend_names) > 0:
raise self.InvalidBackend('No backends returned by selector!')
# Ensure that the primary backend is actually registered -- we
# don't want to schedule any work on the secondaries if the primary
# request is going to fail anyway.
if selected_backend_names[0] not in self.__backends:
raise self.InvalidBackend(
'{!r} is not a registered backend.'.format(
selected_backend_names[0]))
def call_backend_method(backend):
base = self.__backend_base
active_backends = type(self).state.backends
# Ensure that we haven't somehow accidentally entered a context
# where the backend we're calling has already been marked as
# active (or worse, some other backend is already active.)
assert base not in active_backends
# Mark the backend as active.
active_backends[base] = backend
try:
return getattr(backend, attribute_name)(*args, **kwargs)
finally:
# Unmark the backend as active.
assert active_backends[base] is backend
del active_backends[base]
# Enqueue all of the secondary backend requests first since these
# are non-blocking queue insertions. (Since the primary backend
# executor queue insertion can block, if that queue was full the
# secondary requests would have to wait unnecessarily to be queued
# until the after the primary request can be enqueued.)
# NOTE: If the same backend is both the primary backend *and* in
# the secondary backend list -- this is unlikely, but possible --
# this means that one of the secondary requests will be queued and
# executed before the primary request is queued. This is such a
# strange usage pattern that I don't think it's worth optimizing
# for.)
results = [None] * len(selected_backend_names)
for i, backend_name in enumerate(selected_backend_names[1:], 1):
try:
backend, executor = self.__backends[backend_name]
except KeyError:
logger.warning(
'%r is not a registered backend and will be ignored.',
backend_name,
exc_info=True)
else:
results[i] = executor.submit(
functools.partial(call_backend_method, backend),
priority=1,
block=False,
)
# The primary backend is scheduled last since it may block the
# calling thread. (We don't have to protect this from ``KeyError``
# since we already ensured that the primary backend exists.)
backend, executor = self.__backends[selected_backend_names[0]]
results[0] = executor.submit(
functools.partial(call_backend_method, backend),
priority=0,
block=True,
)
if self.__callback_func is not None:
FutureSet(filter(None, results)).add_done_callback(
lambda *a, **k: self.__callback_func(
attribute_name,
callargs,
selected_backend_names,
results,
))
return results[0].result()
