def test_each_event_has_same_result_from_mismatched_demuxed_api_call(self):
an_elt = 'dummy message'
mismatched_result = _Simple([an_elt, an_elt])
bundle_id = 'an_id'
threshold = 5 # arbitrary, greater than 1
options = BundleOptions(element_count_threshold=threshold)
bundler = bundling.Executor(options)
events = []
# send 3 groups of elements of different sizes in the bundle
for i in range(1, 4):
got_event = bundler.schedule(
lambda x: mismatched_result,
bundle_id,
DEMUX_DESCRIPTOR,
_Simple(['%s%d' % (an_elt, i)] * i)
)
events.append(got_event)
previous_event = None
for i, event in enumerate(events):
if previous_event:
self.assertTrue(previous_event != event)
self.assertTrue(event.is_set(),
'event is not set after triggering element')
self.assertEquals(event.result, mismatched_result)
previous_event = event
def test_api_call_not_invoked_until_threshold(self):
an_elt = 'dummy message'
an_id = 'bundle_id'
api_call = _return_request
elts_for_threshold = 3
threshold = elts_for_threshold * len(an_elt) # arbitrary
options = BundleOptions(request_byte_threshold=threshold)
bundler = bundling.Executor(options)
for i in range(elts_for_threshold):
got_event = bundler.schedule(
api_call,
an_id,
SIMPLE_DESCRIPTOR,
_Simple([an_elt])
)
self.assertIsNotNone(
got_event.canceller,
'missing canceller after element #{}'.format(i))
if i + 1 < elts_for_threshold:
self.assertFalse(got_event.is_set())
self.assertIsNone(got_event.result)
else:
self.assertTrue(got_event.is_set())
self.assertEquals(_Simple([an_elt] * elts_for_threshold),
got_event.result)
def test_each_event_has_exception_when_demuxed_api_call_fails(self):
an_elt = 'dummy message'
api_call = _raise_exc
bundle_id = 'an_id'
threshold = 5 # arbitrary, greater than 1
options = BundleOptions(element_count_threshold=threshold)
bundler = bundling.Executor(options)
events = []
for i in range(threshold - 1):
got_event = bundler.schedule(api_call, bundle_id, DEMUX_DESCRIPTOR,
_Bundled(['%s%d' % (an_elt, i)]))
self.assertFalse(
got_event.is_set(),
'event unexpectedly set after element #{}'.format(i))
self.assertIsNone(got_event.result)
events.append(got_event)
last_event = bundler.schedule(
api_call, bundle_id, DEMUX_DESCRIPTOR,
_Bundled(['%s%d' % (an_elt, threshold - 1)]))
events.append(last_event)
previous_event = None
for event in events:
if previous_event:
self.assertTrue(previous_event != event)
self.assertTrue(event.is_set(),
'event is not set after triggering element')
self.assertTrue(isinstance(event.result, ValueError))
previous_event = event
def test_api_calls_are_grouped_by_bundle_id(self):
an_elt = 'dummy message'
api_call = _return_request
bundle_ids = ['id1', 'id2']
threshold = 5 # arbitrary
options = BundleOptions(element_count_threshold=threshold)
bundler = bundling.Executor(options)
for an_id in bundle_ids:
for i in range(threshold - 1):
got_event = bundler.schedule(api_call, an_id,
SIMPLE_DESCRIPTOR,
_Bundled([an_elt]))
self.assertIsNotNone(
got_event.canceller,
'missing canceller after element #{}'.format(i))
self.assertFalse(
got_event.is_set(),
'event unexpectedly set after element #{}'.format(i))
self.assertIsNone(got_event.result)
for an_id in bundle_ids:
got_event = bundler.schedule(api_call, an_id, SIMPLE_DESCRIPTOR,
_Bundled([an_elt]))
self.assertIsNotNone(got_event.canceller,
'missing expected canceller')
self.assertTrue(got_event.is_set(),
'event is not set after triggering element')
self.assertEqual(_Bundled([an_elt] * threshold), got_event.result)
def _construct_bundling(bundle_config, bundle_descriptor):
"""Helper for ``construct_settings()``.
Args:
bundle_config: A dictionary specifying a bundle parameters, the value for
'bundling' field in a method config (See ``construct_settings()`` for
information on this config.)
bundle_descriptor: A BundleDescriptor object describing the structure of
bundling for this method. If not set, this method will not bundle.
Returns:
A tuple (bundling.Executor, BundleDescriptor) that configures bundling.
The bundling.Executor may be None if this method should not bundle.
"""
if bundle_config and bundle_descriptor:
bundler = bundling.Executor(
gax.BundleOptions(
element_count_threshold=bundle_config.get(
'element_count_threshold', 0),
element_count_limit=bundle_config.get('element_count_limit',
0),
request_byte_threshold=bundle_config.get(
'request_byte_threshold', 0),
request_byte_limit=bundle_config.get('request_byte_limit', 0),
delay_threshold=bundle_config.get('delay_threshold_millis',
0)))
else:
bundler = None
return bundler
def test_schedule_passes_kwargs(self):
an_elt = 'dummy_msg'
options = BundleOptions(element_count_threshold=1)
bundle_id = 'an_id'
bundler = bundling.Executor(options)
event = bundler.schedule(_return_kwargs, bundle_id, SIMPLE_DESCRIPTOR,
_Bundled([an_elt]), {'an_option': 'a_value'})
self.assertEqual('a_value', event.result['an_option'])
def test_api_call_is_scheduled_on_timer(self, timer_class):
an_elt = 'dummy message'
an_id = 'bundle_id'
api_call = _return_request
delay_threshold = 3
options = BundleOptions(delay_threshold=delay_threshold)
bundler = bundling.Executor(options)
got_event = bundler.schedule(api_call, an_id, SIMPLE_DESCRIPTOR,
_Bundled([an_elt]))
self.assertIsNotNone(got_event, 'missing event after first request')
self.assertIsNone(got_event.result)
self.assertTrue(timer_class.called)
timer_args, timer_kwargs = timer_class.call_args_list[0]
self.assertEqual(delay_threshold, timer_args[0])
self.assertEqual({'args': [an_id]}, timer_kwargs)
timer_class.return_value.start.assert_called_once_with()
def test_bundling(self):
# pylint: disable=abstract-method, too-few-public-methods
class BundlingRequest(object):
def __init__(self, elements=None):
self.elements = elements
fake_grpc_func_descriptor = BundleDescriptor('elements', [])
bundler = bundling.Executor(BundleOptions(element_count_threshold=8))
def my_func(request, dummy_timeout):
return len(request.elements)
settings = _CallSettings(bundler=bundler,
bundle_descriptor=fake_grpc_func_descriptor,
timeout=0)
my_callable = api_callable.create_api_call(my_func, settings)
first = my_callable(BundlingRequest([0] * 3))
self.assertIsInstance(first, bundling.Event)
self.assertIsNone(first.result) # pylint: disable=no-member
second = my_callable(BundlingRequest([0] * 5))
self.assertEqual(second.result, 8) # pylint: disable=no-member
def test_each_event_has_its_result_from_a_demuxed_api_call(self):
an_elt = 'dummy message'
api_call = _return_request
bundle_id = 'an_id'
threshold = 5 # arbitrary, greater than 1
options = BundleOptions(element_count_threshold=threshold)
bundler = bundling.Executor(options)
events = []
# send 3 groups of elements of different sizes in the bundle
for i in range(1, 4):
got_event = bundler.schedule(api_call, bundle_id, DEMUX_DESCRIPTOR,
_Bundled(['%s%d' % (an_elt, i)] * i))
events.append(got_event)
previous_event = None
for i, event in enumerate(events):
index = i + 1
if previous_event:
self.assertTrue(previous_event != event)
self.assertTrue(event.is_set(),
'event is not set after triggering element')
self.assertEqual(event.result,
_Bundled(['%s%d' % (an_elt, index)] * index))
previous_event = event
