def _pfactory(self, buckets, log, got_buckets):
self.assertEqual(buckets, range(self.num_buckets))
self.fake_partitioner = FakePartitioner(log, got_buckets)
return self.fake_partitioner
class ConvergerTests(SynchronousTestCase):
"""Tests for :obj:`Converger`."""
def setUp(self):
self.log = mock_log()
self.num_buckets = 10
def _converger(self, converge_all_groups, dispatcher=None):
if dispatcher is None:
dispatcher = _get_dispatcher()
return Converger(
self.log, dispatcher, self.num_buckets,
self._pfactory, build_timeout=3600,
interval=15,
converge_all_groups=converge_all_groups)
def _pfactory(self, buckets, log, got_buckets):
self.assertEqual(buckets, range(self.num_buckets))
self.fake_partitioner = FakePartitioner(log, got_buckets)
return self.fake_partitioner
def _log_sequence(self, intents):
uid = uuid.uuid4()
exp_uid = str(uid)
return SequenceDispatcher([
(Func(uuid.uuid4), lambda i: uid),
(BoundFields(effect=mock.ANY,
fields={'otter_service': 'converger',
'converger_run_id': exp_uid}),
nested_sequence(intents)),
])
def test_buckets_acquired(self):
"""
When buckets are allocated, the result of converge_all_groups is
performed.
"""
def converge_all_groups(currently_converging, recent, _my_buckets,
all_buckets, divergent_flags, build_timeout,
interval):
return Effect(
('converge-all', currently_converging, _my_buckets,
all_buckets, divergent_flags, build_timeout, interval))
my_buckets = [0, 5]
bound_sequence = [
(GetChildren(CONVERGENCE_DIRTY_DIR),
lambda i: ['flag1', 'flag2']),
(('converge-all',
transform_eq(lambda cc: cc is converger.currently_converging,
True),
my_buckets,
range(self.num_buckets),
['flag1', 'flag2'],
3600,
15),
lambda i: 'foo')
]
sequence = self._log_sequence(bound_sequence)
converger = self._converger(converge_all_groups, dispatcher=sequence)
with sequence.consume():
result, = self.fake_partitioner.got_buckets(my_buckets)
self.assertEqual(self.successResultOf(result), 'foo')
def test_buckets_acquired_errors(self):
"""
Errors raised from performing the converge_all_groups effect are
logged, and None is the ultimate result.
"""
def converge_all_groups(currently_converging, recent, _my_buckets,
all_buckets, divergent_flags, build_timeout,
interval):
return Effect('converge-all')
bound_sequence = [
(GetChildren(CONVERGENCE_DIRTY_DIR),
lambda i: ['flag1', 'flag2']),
('converge-all', lambda i: raise_(RuntimeError('foo'))),
(LogErr(
CheckFailureValue(RuntimeError('foo')),
'converge-all-groups-error', {}), noop)
]
sequence = self._log_sequence(bound_sequence)
# relying on the side-effect of setting up self.fake_partitioner
self._converger(converge_all_groups, dispatcher=sequence)
with sequence.consume():
result, = self.fake_partitioner.got_buckets([0])
self.assertEqual(self.successResultOf(result), None)
def test_divergent_changed_not_acquired(self):
"""
When notified that divergent groups have changed and we have not
acquired our buckets, nothing is done.
"""
dispatcher = SequenceDispatcher([]) # "nothing happens"
converger = self._converger(lambda *a, **kw: 1 / 0,
dispatcher=dispatcher)
# Doesn't try to get buckets
self.fake_partitioner.get_current_buckets = lambda s: 1 / 0
converger.divergent_changed(['group1', 'group2'])
def test_divergent_changed_not_ours(self):
"""
When notified that divergent groups have changed but they're not ours,
nothing is done.
"""
dispatcher = SequenceDispatcher([]) # "nothing happens"
converger = self._converger(lambda *a, **kw: 1 / 0,
dispatcher=dispatcher)
self.fake_partitioner.current_state = PartitionState.ACQUIRED
converger.divergent_changed(['group1', 'group2'])
def test_divergent_changed(self):
"""
When notified that divergent groups have changed, and one of the groups
is associated with a bucket assigned to us, convergence is triggered,
and the list of child nodes is passed on to
:func:`converge_all_groups`.
"""
def converge_all_groups(currently_converging, recent, _my_buckets,
all_buckets, divergent_flags, build_timeout,
interval):
return Effect(('converge-all-groups', divergent_flags))
intents = [
(('converge-all-groups', ['group1', 'group2']),
noop)
]
sequence = self._log_sequence(intents)
converger = self._converger(converge_all_groups, dispatcher=sequence)
# sha1('group1') % 10 == 3
self.fake_partitioner.current_state = PartitionState.ACQUIRED
self.fake_partitioner.my_buckets = [3]
with sequence.consume():
converger.divergent_changed(['group1', 'group2'])
def pfactory(log, callable):
self.fake_partitioner = FakePartitioner(log, callable)
return self.fake_partitioner
class SchedulerServiceTests(SchedulerTests, DeferredFunctionMixin):
"""
Tests for `SchedulerService`.
"""
def setUp(self):
"""
Mock all the dependencies of SchedulingService.
This includes logging, store's fetch_and_delete, TxKazooClient stuff,
check_events_in_bucket.
"""
super(SchedulerServiceTests, self).setUp()
otter_log = patch(self, 'otter.scheduler.otter_log')
self.log = mock_log()
otter_log.bind.return_value = self.log
def pfactory(log, callable):
self.fake_partitioner = FakePartitioner(log, callable)
return self.fake_partitioner
self.scheduler_service = SchedulerService("disp",
100,
self.mock_store,
pfactory,
threshold=600)
otter_log.bind.assert_called_once_with(system='otter.scheduler')
self.scheduler_service.running = True
self.assertIdentical(self.fake_partitioner,
self.scheduler_service.partitioner)
self.check_events_in_bucket = patch(
self, 'otter.scheduler.check_events_in_bucket')
self.returns = []
self.setup_func(self.mock_store.get_oldest_event)
def test_partitioner_child(self):
"""
The Partitioner service is registered as a child of the
SchedulerService.
"""
self.assertEqual(self.scheduler_service.services,
[self.fake_partitioner])
def test_health_check_after_threshold(self):
"""
`service.health_check` returns False when trigger time is above
threshold.
"""
self.fake_partitioner.health = (True, {'buckets': [2, 3]})
now = datetime.utcnow()
returns = [{
'trigger': now - timedelta(hours=1),
'version': 'v1'
}, {
'trigger': now - timedelta(seconds=2),
'version': 'v1'
}]
self.returns = returns[:]
d = self.scheduler_service.health_check()
self.assertEqual(self.successResultOf(d), (False, {
'old_events': [returns[0]],
'buckets': [2, 3]
}))
self.mock_store.get_oldest_event.assert_has_calls(
[mock.call(2), mock.call(3)])
def test_health_check_before_threshold(self):
"""
`service.health_check` returns True when trigger time is below
threshold.
"""
self.fake_partitioner.health = (True, {'buckets': [2, 3]})
now = datetime.utcnow()
self.returns = [{
'trigger': now + timedelta(hours=1),
'version': 'v1'
}, {
'trigger': now + timedelta(seconds=2),
'version': 'v1'
}]
d = self.scheduler_service.health_check()
self.assertEqual(self.successResultOf(d), (True, {
'old_events': [],
'buckets': [2, 3]
}))
self.mock_store.get_oldest_event.assert_has_calls(
[mock.call(2), mock.call(3)])
def test_health_check_None(self):
"""
`service.health_check` returns True when there are no triggers.
"""
self.fake_partitioner.health = (True, {'buckets': [2, 3]})
self.returns = [None, None]
d = self.scheduler_service.health_check()
self.assertEqual(self.successResultOf(d), (True, {
'old_events': [],
'buckets': [2, 3]
}))
self.mock_store.get_oldest_event.assert_has_calls(
[mock.call(2), mock.call(3)])
def test_health_check_unhealthy_partitioner(self):
"""
When the partitioner service is unhealthy, the scheduler service passes
its health message through.
"""
self.fake_partitioner.health = (False, {'foo': 'bar'})
d = self.scheduler_service.health_check()
self.assertEqual(self.successResultOf(d), (False, {'foo': 'bar'}))
def test_health_check_not_running(self):
"""
`service.health_check` returns False when scheduler is stopped.
"""
self.scheduler_service.running = False
d = self.scheduler_service.health_check()
self.assertEqual(self.successResultOf(d), (False, {
'reason': 'Not running'
}))
self.assertFalse(self.mock_store.get_oldest_event.called)
def test_reset(self):
"""
reset() starts new partition based on new path.
"""
self.assertEqual(self.scheduler_service.reset('/new_path'),
'partitioner reset to /new_path')
@mock.patch('otter.scheduler.datetime')
def test_check_events_acquired(self, mock_datetime):
"""
the got_buckets callback checks events in each bucket when they are
partitoned.
"""
self.scheduler_service.log = mock.Mock()
mock_datetime.utcnow.return_value = 'utcnow'
responses = [4, 5]
self.check_events_in_bucket.side_effect = \
lambda *_: defer.succeed(responses.pop(0))
d = self.fake_partitioner.got_buckets([2, 3])
self.assertEqual(self.successResultOf(d), [4, 5])
self.scheduler_service.log.bind.assert_called_once_with(
scheduler_run_id='transaction-id', utcnow='utcnow')
log = self.scheduler_service.log.bind.return_value
self.assertEqual(self.check_events_in_bucket.mock_calls, [
mock.call(log, "disp", self.mock_store, 2, 'utcnow', 100),
mock.call(log, "disp", self.mock_store, 3, 'utcnow', 100)
])
def pfactory(log, callable):
self.fake_partitioner = FakePartitioner(log, callable)
return self.fake_partitioner
class SchedulerServiceTests(SchedulerTests, DeferredFunctionMixin):
"""
Tests for `SchedulerService`.
"""
def setUp(self):
"""
Mock all the dependencies of SchedulingService.
This includes logging, store's fetch_and_delete, TxKazooClient stuff,
check_events_in_bucket.
"""
super(SchedulerServiceTests, self).setUp()
otter_log = patch(self, "otter.scheduler.otter_log")
self.log = mock_log()
otter_log.bind.return_value = self.log
def pfactory(log, callable):
self.fake_partitioner = FakePartitioner(log, callable)
return self.fake_partitioner
self.scheduler_service = SchedulerService("disp", 100, self.mock_store, pfactory, threshold=600)
otter_log.bind.assert_called_once_with(system="otter.scheduler")
self.scheduler_service.running = True
self.assertIdentical(self.fake_partitioner, self.scheduler_service.partitioner)
self.check_events_in_bucket = patch(self, "otter.scheduler.check_events_in_bucket")
self.returns = []
self.setup_func(self.mock_store.get_oldest_event)
def test_partitioner_child(self):
"""
The Partitioner service is registered as a child of the
SchedulerService.
"""
self.assertEqual(self.scheduler_service.services, [self.fake_partitioner])
def test_health_check_after_threshold(self):
"""
`service.health_check` returns False when trigger time is above
threshold.
"""
self.fake_partitioner.health = (True, {"buckets": [2, 3]})
now = datetime.utcnow()
returns = [
{"trigger": now - timedelta(hours=1), "version": "v1"},
{"trigger": now - timedelta(seconds=2), "version": "v1"},
]
self.returns = returns[:]
d = self.scheduler_service.health_check()
self.assertEqual(self.successResultOf(d), (False, {"old_events": [returns[0]], "buckets": [2, 3]}))
self.mock_store.get_oldest_event.assert_has_calls([mock.call(2), mock.call(3)])
def test_health_check_before_threshold(self):
"""
`service.health_check` returns True when trigger time is below
threshold.
"""
self.fake_partitioner.health = (True, {"buckets": [2, 3]})
now = datetime.utcnow()
self.returns = [
{"trigger": now + timedelta(hours=1), "version": "v1"},
{"trigger": now + timedelta(seconds=2), "version": "v1"},
]
d = self.scheduler_service.health_check()
self.assertEqual(self.successResultOf(d), (True, {"old_events": [], "buckets": [2, 3]}))
self.mock_store.get_oldest_event.assert_has_calls([mock.call(2), mock.call(3)])
def test_health_check_None(self):
"""
`service.health_check` returns True when there are no triggers.
"""
self.fake_partitioner.health = (True, {"buckets": [2, 3]})
self.returns = [None, None]
d = self.scheduler_service.health_check()
self.assertEqual(self.successResultOf(d), (True, {"old_events": [], "buckets": [2, 3]}))
self.mock_store.get_oldest_event.assert_has_calls([mock.call(2), mock.call(3)])
def test_health_check_unhealthy_partitioner(self):
"""
When the partitioner service is unhealthy, the scheduler service passes
its health message through.
"""
self.fake_partitioner.health = (False, {"foo": "bar"})
d = self.scheduler_service.health_check()
self.assertEqual(self.successResultOf(d), (False, {"foo": "bar"}))
def test_health_check_not_running(self):
"""
`service.health_check` returns False when scheduler is stopped.
"""
self.scheduler_service.running = False
d = self.scheduler_service.health_check()
self.assertEqual(self.successResultOf(d), (False, {"reason": "Not running"}))
self.assertFalse(self.mock_store.get_oldest_event.called)
def test_reset(self):
"""
reset() starts new partition based on new path.
"""
self.assertEqual(self.scheduler_service.reset("/new_path"), "partitioner reset to /new_path")
@mock.patch("otter.scheduler.datetime")
def test_check_events_acquired(self, mock_datetime):
"""
the got_buckets callback checks events in each bucket when they are
partitoned.
"""
self.scheduler_service.log = mock.Mock()
mock_datetime.utcnow.return_value = "utcnow"
responses = [4, 5]
self.check_events_in_bucket.side_effect = lambda *_: defer.succeed(responses.pop(0))
d = self.fake_partitioner.got_buckets([2, 3])
self.assertEqual(self.successResultOf(d), [4, 5])
self.scheduler_service.log.bind.assert_called_once_with(scheduler_run_id="transaction-id", utcnow="utcnow")
log = self.scheduler_service.log.bind.return_value
self.assertEqual(
self.check_events_in_bucket.mock_calls,
[
mock.call(log, "disp", self.mock_store, 2, "utcnow", 100),
mock.call(log, "disp", self.mock_store, 3, "utcnow", 100),
],
)