def test___new__(self):
expected = BaseServiceInstance(
service="universe",
instance="c137",
watcher="mywatcher",
bounce_by=0,
wait_until=0,
failures=0,
bounce_timers=None,
processed_count=0,
)
assert self.service_instance == expected
expected = BaseServiceInstance(
service="universe",
instance="c137",
watcher="mywatcher",
bounce_by=0,
wait_until=0,
failures=0,
bounce_timers=None,
processed_count=0,
)
# https://github.com/python/mypy/issues/2852
assert (ServiceInstance( # type: ignore
service="universe",
instance="c137",
watcher="mywatcher",
cluster="westeros-prod",
bounce_by=0,
wait_until=0,
) == expected)
def test_run(self):
with mock.patch("time.time", autospec=True, return_value=1), mock.patch(
"time.sleep", autospec=True
) as mock_sleep, mock.patch(
"paasta_tools.deployd.workers.PaastaDeployWorker.process_service_instance",
autospec=True,
) as mock_process_service_instance:
mock_timers = mock.Mock()
mock_bounce_results = BounceResults(
bounce_again_in_seconds=None, return_code=0, bounce_timers=mock_timers
)
mock_process_service_instance.return_value = mock_bounce_results
mock_sleep.side_effect = LoopBreak
mock_si = mock.Mock(
service="universe", instance="c137", failures=0, processed_count=0
)
self.mock_instances_to_bounce.get.return_value = mock_si
with raises(LoopBreak):
self.worker.run()
mock_process_service_instance.assert_called_with(self.worker, mock_si)
assert not self.mock_instances_to_bounce.put.called
mock_bounce_results = BounceResults(
bounce_again_in_seconds=60, return_code=1, bounce_timers=mock_timers
)
mock_process_service_instance.return_value = mock_bounce_results
mock_queued_si = BaseServiceInstance(
service="universe",
instance="c137",
bounce_by=61,
wait_until=61,
watcher="Worker1",
bounce_timers=mock_timers,
failures=1,
processed_count=1,
)
with raises(LoopBreak):
self.worker.run()
mock_process_service_instance.assert_called_with(self.worker, mock_si)
self.mock_instances_to_bounce.put.assert_called_with(mock_queued_si)
mock_si = mock.Mock(
service="universe", instance="c137", failures=0, processed_count=0
)
self.mock_instances_to_bounce.get.return_value = mock_si
mock_process_service_instance.side_effect = Exception
mock_queued_si = BaseServiceInstance(
service="universe",
instance="c137",
bounce_by=61,
wait_until=61,
watcher="Worker1",
bounce_timers=mock_si.bounce_timers,
failures=1,
processed_count=1,
)
with raises(LoopBreak):
self.worker.run()
mock_process_service_instance.assert_called_with(self.worker, mock_si)
self.mock_instances_to_bounce.put.assert_called_with(mock_queued_si)
def test___new__(self):
with mock.patch(
'paasta_tools.deployd.common.get_priority',
autospec=True,
):
expected = BaseServiceInstance(
service='universe',
instance='c137',
watcher='mywatcher',
bounce_by=0,
failures=0,
bounce_timers=None,
priority=1,
)
assert self.service_instance == expected
expected = BaseServiceInstance(
service='universe',
instance='c137',
watcher='mywatcher',
bounce_by=0,
failures=0,
bounce_timers=None,
priority=2,
)
# https://github.com/python/mypy/issues/2852
assert ServiceInstance( # type: ignore
service='universe',
instance='c137',
watcher='mywatcher',
cluster='westeros-prod',
bounce_by=0,
priority=2,
) == expected
def test_get_at_risk_service_instances(self):
with mock.patch(
'paasta_tools.deployd.common.ServiceInstance.get_priority',
autospec=True,
return_value=0,
), mock.patch(
'paasta_tools.deployd.watchers.get_all_marathon_apps',
autospec=True,
) as mock_get_marathon_apps, mock.patch(
'time.time',
autospec=True,
return_value=1,
):
mock_marathon_apps = [
mock.Mock(tasks=[
mock.Mock(
host='host1',
app_id='/universe.c137.configsha.gitsha',
),
mock.Mock(
host='host2',
app_id='/universe.c138.configsha.gitsha',
),
]),
mock.Mock(tasks=[
mock.Mock(
host='host1',
app_id='/universe.c139.configsha.gitsha',
)
]),
mock.Mock(tasks=[
mock.Mock(
host='host1',
app_id='/universe.c139.configsha.gitsha',
)
]),
]
mock_get_marathon_apps.return_value = mock_marathon_apps
ret = self.watcher.get_at_risk_service_instances(['host1'])
expected = [
BaseServiceInstance(
service='universe',
instance='c137',
bounce_by=1,
watcher=self.watcher.__class__.__name__,
priority=0,
bounce_timers=None,
failures=0,
),
BaseServiceInstance(
service='universe',
instance='c139',
bounce_by=1,
watcher=self.watcher.__class__.__name__,
priority=0,
bounce_timers=None,
failures=0,
),
]
assert ret == expected
def test___new__(self):
with mock.patch(
'paasta_tools.deployd.common.ServiceInstance.get_priority',
autospec=True,
):
expected = BaseServiceInstance(
service='universe',
instance='c137',
watcher='mywatcher',
bounce_by=0,
failures=0,
bounce_timers=None,
priority=1,
)
assert self.service_instance == expected
expected = BaseServiceInstance(
service='universe',
instance='c137',
watcher='mywatcher',
bounce_by=0,
failures=0,
bounce_timers=None,
priority=2,
)
assert ServiceInstance(
service='universe',
instance='c137',
watcher='mywatcher',
cluster='westeros-prod',
bounce_by=0,
priority=2,
) == expected
def test_process_node_event(self):
with mock.patch(
'paasta_tools.deployd.common.get_priority',
autospec=True,
return_value=0,
), mock.patch(
'paasta_tools.deployd.watchers.EventType',
autospec=True,
) as mock_event_type, mock.patch(
'time.time',
autospec=True,
return_value=1,
):
mock_event_other = mock_event_type.DELETED
mock_event = mock.Mock(
type=mock_event_other,
path='/autoscaling/service/instance/instances',
)
assert not self.mock_instances_to_bounce_later.put.called
mock_event_created = mock_event_type.CREATED
mock_event = mock.Mock(
type=mock_event_created,
path='/autoscaling/service/instance/instances',
)
self.watcher.process_node_event(mock.Mock(), mock.Mock(),
mock_event)
self.mock_instances_to_bounce_later.put.assert_called_with(
BaseServiceInstance(
service='service',
instance='instance',
bounce_by=1,
bounce_timers=None,
watcher=self.watcher.__class__.__name__,
priority=0,
failures=0,
))
mock_event_changed = mock_event_type.CHANGED
mock_event = mock.Mock(
type=mock_event_changed,
path='/autoscaling/service/instance/instances',
)
self.watcher.process_node_event(mock.Mock(), mock.Mock(),
mock_event)
self.mock_instances_to_bounce_later.put.assert_called_with(
BaseServiceInstance(
service='service',
instance='instance',
bounce_by=1,
bounce_timers=None,
watcher=self.watcher.__class__.__name__,
priority=0,
failures=0,
))
def test_get_at_risk_service_instances(self):
with mock.patch(
"paasta_tools.deployd.watchers.get_marathon_apps_with_clients",
autospec=True,
) as mock_get_marathon_apps, mock.patch("time.time",
autospec=True,
return_value=1):
mock_marathon_apps = [
mock.Mock(tasks=[
mock.Mock(host="host1",
app_id="/universe.c137.configsha.gitsha"),
mock.Mock(host="host2",
app_id="/universe.c138.configsha.gitsha"),
]),
mock.Mock(tasks=[
mock.Mock(host="host1",
app_id="/universe.c139.configsha.gitsha")
]),
mock.Mock(tasks=[
mock.Mock(host="host1",
app_id="/universe.c139.configsha.gitsha")
]),
]
mock_client = mock.Mock()
mock_get_marathon_apps.return_value = [
(app, mock_client) for app in mock_marathon_apps
]
ret = self.watcher.get_at_risk_service_instances(["host1"])
expected = [
BaseServiceInstance(
service="universe",
instance="c137",
bounce_by=1,
wait_until=1,
watcher=self.watcher.__class__.__name__,
bounce_timers=None,
failures=0,
processed_count=0,
),
BaseServiceInstance(
service="universe",
instance="c139",
bounce_by=1,
wait_until=1,
watcher=self.watcher.__class__.__name__,
bounce_timers=None,
failures=0,
processed_count=0,
),
]
assert ret == expected
def test_prioritise_bouncing_services(self):
with mock.patch(
'paasta_tools.deployd.common.get_priority',
autospec=True,
return_value=0,
), mock.patch(
'paasta_tools.deployd.master.get_service_instances_that_need_bouncing',
autospec=True,
) as mock_get_service_instances_that_need_bouncing, mock.patch(
'time.time',
autospec=True,
return_value=1,
):
mock_changed_instances = (
x for x in {'universe.c137', 'universe.c138'})
mock_get_service_instances_that_need_bouncing.return_value = mock_changed_instances
self.deployd.prioritise_bouncing_services()
mock_get_service_instances_that_need_bouncing.assert_called_with(
self.deployd.marathon_clients,
'/nail/etc/services',
)
calls = [
mock.call(
BaseServiceInstance(
service='universe',
instance='c138',
watcher='DeployDaemon',
priority=0,
bounce_by=1,
bounce_timers=None,
failures=0,
)),
mock.call(
BaseServiceInstance(
service='universe',
instance='c137',
watcher='DeployDaemon',
priority=0,
bounce_by=1,
bounce_timers=None,
failures=0,
)),
]
self.deployd.instances_that_need_to_be_bounced_in_the_future.put.assert_has_calls(
calls, any_order=True)
def test_process_node_event(self):
with mock.patch("paasta_tools.deployd.watchers.EventType",
autospec=True) as mock_event_type, mock.patch(
"time.time", autospec=True, return_value=1):
mock_event_other = mock_event_type.DELETED
mock_event = mock.Mock(
type=mock_event_other,
path="/autoscaling/service/instance/instances")
assert not self.mock_instances_to_bounce.put.called
mock_event_created = mock_event_type.CREATED
mock_event = mock.Mock(
type=mock_event_created,
path="/autoscaling/service/instance/instances")
self.watcher.process_node_event(mock.Mock(), mock.Mock(),
mock_event)
self.mock_instances_to_bounce.put.assert_called_with(
BaseServiceInstance(
service="service",
instance="instance",
bounce_by=1,
wait_until=1,
bounce_timers=None,
watcher=self.watcher.__class__.__name__,
failures=0,
processed_count=0,
))
mock_event_changed = mock_event_type.CHANGED
mock_event = mock.Mock(
type=mock_event_changed,
path="/autoscaling/service/instance/instances")
self.watcher.process_node_event(mock.Mock(), mock.Mock(),
mock_event)
self.mock_instances_to_bounce.put.assert_called_with(
BaseServiceInstance(
service="service",
instance="instance",
bounce_by=1,
wait_until=1,
bounce_timers=None,
watcher=self.watcher.__class__.__name__,
failures=0,
processed_count=0,
))
def test_prioritise_bouncing_services(self):
with mock.patch(
"paasta_tools.deployd.master.get_service_instances_that_need_bouncing",
autospec=True,
) as mock_get_service_instances_that_need_bouncing, mock.patch(
"time.time", autospec=True, return_value=1
):
mock_changed_instances = (x for x in {"universe.c137", "universe.c138"})
mock_get_service_instances_that_need_bouncing.return_value = (
mock_changed_instances
)
self.deployd.prioritise_bouncing_services()
mock_get_service_instances_that_need_bouncing.assert_called_with(
self.deployd.marathon_clients, "/nail/etc/services"
)
calls = [
mock.call(
BaseServiceInstance(
service="universe",
instance="c138",
watcher="DeployDaemon",
bounce_by=1,
wait_until=1,
bounce_timers=None,
failures=0,
processed_count=0,
)
),
mock.call(
BaseServiceInstance(
service="universe",
instance="c137",
watcher="DeployDaemon",
bounce_by=1,
wait_until=1,
bounce_timers=None,
failures=0,
processed_count=0,
)
),
]
self.deployd.instances_to_bounce.put.assert_has_calls(calls, any_order=True)
def setUp(self):
self.queue = DedupedPriorityQueue("BounceQueue")
assert not self.queue.bouncing
self.mock_service_instance = BaseServiceInstance(
service='universe',
instance='c137',
watcher='tests',
priority=0,
bounce_by=1,
bounce_timers=None,
failures=0,
)
def make_si(wait_until, bounce_by):
"""Just using mock.Mock(wait_until=wait_until, bounce_by=bounce_by) mostly works, but our PriorityQueues
occasionally will compare two ServiceInstances directly, and Mocks aren't comparable unless you define an __eq__."""
return BaseServiceInstance(
service="service",
instance="instance",
bounce_by=bounce_by,
wait_until=wait_until,
watcher="watcher",
bounce_timers=None,
failures=0,
processed_count=0,
)
def test_bounce_service(self):
with mock.patch(
'paasta_tools.deployd.common.get_priority',
autospec=True,
return_value=0,
), mock.patch(
'paasta_tools.deployd.watchers.list_all_instances_for_service',
autospec=True,
) as mock_list_instances, mock.patch(
'paasta_tools.deployd.watchers.get_service_instances_needing_update',
autospec=True,
) as mock_get_service_instances_needing_update, mock.patch(
'time.time',
autospec=True,
return_value=1,
):
mock_list_instances.return_value = ['c137', 'c138']
mock_get_service_instances_needing_update.return_value = [
('universe', 'c137')
]
self.handler.bounce_service('universe')
mock_list_instances.assert_called_with(
service='universe',
clusters=[self.handler.filewatcher.cluster],
instance_type='marathon',
cache=False,
)
mock_get_service_instances_needing_update.assert_called_with(
self.handler.marathon_clients,
[
('universe', 'c137'),
('universe', 'c138'),
],
self.handler.filewatcher.cluster,
)
expected_si = BaseServiceInstance(
service='universe',
instance='c137',
bounce_by=1,
watcher='YelpSoaEventHandler',
bounce_timers=None,
priority=0,
failures=0,
)
self.mock_filewatcher.instances_to_bounce_later.put.assert_called_with(
expected_si)
assert self.mock_filewatcher.instances_to_bounce_later.put.call_count == 1
def test_get(self):
with mock.patch(
'paasta_tools.deployd.master.PaastaPriorityQueue.get', autospec=True,
) as mock_paasta_queue_get:
self.mock_service_instance = BaseServiceInstance(
service='universe',
instance='c137',
watcher='tests',
priority=0,
bounce_by=1,
bounce_timers=None,
failures=0,
)
self.queue.bouncing.add('universe.c137')
mock_paasta_queue_get.return_value = self.mock_service_instance
assert self.queue.get() is self.mock_service_instance
assert mock_paasta_queue_get.called
assert 'universe.c137' not in self.queue.bouncing
def test_bounce_service(self):
with mock.patch(
"paasta_tools.deployd.watchers.list_all_instances_for_service",
autospec=True,
) as mock_list_instances, mock.patch(
"paasta_tools.deployd.watchers.get_service_instances_needing_update",
autospec=True,
) as mock_get_service_instances_needing_update, mock.patch(
"time.time", autospec=True, return_value=1):
mock_list_instances.return_value = ["c137", "c138"]
mock_get_service_instances_needing_update.return_value = [(
"universe",
"c137",
mock.Mock(get_bounce_start_deadline=mock.Mock(return_value=0)),
)]
self.handler.bounce_service("universe")
mock_list_instances.assert_called_with(
service="universe",
clusters=[self.handler.filewatcher.cluster],
instance_type="marathon",
cache=False,
)
mock_get_service_instances_needing_update.assert_called_with(
self.handler.marathon_clients,
[("universe", "c137"), ("universe", "c138")],
self.handler.filewatcher.cluster,
)
expected_si = BaseServiceInstance(
service="universe",
instance="c137",
bounce_by=1,
wait_until=1,
watcher="YelpSoaEventHandler",
bounce_timers=None,
failures=0,
processed_count=0,
)
self.mock_filewatcher.instances_to_bounce.put.assert_called_with(
expected_si)
assert self.mock_filewatcher.instances_to_bounce.put.call_count == 1
def test_rate_limit_instances():
with mock.patch(
'paasta_tools.deployd.common.get_priority',
autospec=True,
return_value=0,
), mock.patch(
'time.time',
autospec=True,
) as mock_time:
mock_time.return_value = 1
mock_si_1 = ('universe', 'c137')
mock_si_2 = ('universe', 'c138')
ret = rate_limit_instances([mock_si_1, mock_si_2], "westeros-prod", 2,
"Custos")
expected = [
BaseServiceInstance(
service='universe',
instance='c137',
watcher='Custos',
priority=0,
bounce_by=1,
bounce_timers=None,
failures=0,
),
BaseServiceInstance(
service='universe',
instance='c138',
watcher='Custos',
priority=0,
bounce_by=31,
bounce_timers=None,
failures=0,
),
]
assert ret == expected
ret = rate_limit_instances([mock_si_1, mock_si_2],
"westeros-prod",
2,
"Custos",
priority=99)
expected = [
BaseServiceInstance(
service='universe',
instance='c137',
watcher='Custos',
priority=99,
bounce_by=1,
bounce_timers=None,
failures=0,
),
BaseServiceInstance(
service='universe',
instance='c138',
watcher='Custos',
priority=99,
bounce_by=31,
bounce_timers=None,
failures=0,
),
]
assert ret == expected
def test_run(self):
with mock.patch(
'time.time',
autospec=True,
return_value=1,
), mock.patch(
'time.sleep',
autospec=True,
) as mock_sleep, mock.patch(
'paasta_tools.deployd.workers.PaastaDeployWorker.process_service_instance',
autospec=True,
) as mock_process_service_instance:
mock_timers = mock.Mock()
mock_bounce_results = BounceResults(
bounce_again_in_seconds=None,
return_code=0,
bounce_timers=mock_timers,
)
mock_process_service_instance.return_value = mock_bounce_results
mock_sleep.side_effect = LoopBreak
mock_si = mock.Mock(
service='universe',
instance='c137',
failures=0,
priority=0,
)
self.mock_bounce_q.get.return_value = mock_si
with raises(LoopBreak):
self.worker.run()
mock_process_service_instance.assert_called_with(
self.worker, mock_si)
assert not self.mock_inbox_q.put.called
mock_bounce_results = BounceResults(
bounce_again_in_seconds=60,
return_code=1,
bounce_timers=mock_timers,
)
mock_process_service_instance.return_value = mock_bounce_results
mock_queued_si = BaseServiceInstance(
service='universe',
instance='c137',
bounce_by=61,
watcher='Worker1',
bounce_timers=mock_timers,
failures=1,
priority=0,
)
with raises(LoopBreak):
self.worker.run()
mock_process_service_instance.assert_called_with(
self.worker, mock_si)
self.mock_inbox_q.put.assert_called_with(mock_queued_si)
mock_si = mock.Mock(
service='universe',
instance='c137',
failures=0,
priority=0,
)
self.mock_bounce_q.get.return_value = mock_si
mock_process_service_instance.side_effect = Exception
mock_queued_si = BaseServiceInstance(
service='universe',
instance='c137',
bounce_by=61,
watcher='Worker1',
bounce_timers=mock_si.bounce_timers,
failures=1,
priority=0,
)
with raises(LoopBreak):
self.worker.run()
mock_process_service_instance.assert_called_with(
self.worker, mock_si)
self.mock_inbox_q.put.assert_called_with(mock_queued_si)
