class TestMaintenanceWatcher(unittest.TestCase):
def setUp(self):
self.mock_inbox_q = mock.Mock()
self.mock_marathon_client = mock.Mock()
with mock.patch(
'paasta_tools.deployd.watchers.get_marathon_client_from_config',
autospec=True):
self.watcher = MaintenanceWatcher(self.mock_inbox_q,
"westeros-prod")
def test_run(self):
with mock.patch(
'paasta_tools.deployd.watchers.get_draining_hosts',
autospec=True
) as mock_get_draining_hosts, mock.patch(
'paasta_tools.deployd.watchers.MaintenanceWatcher.get_at_risk_service_instances',
autospec=True
) as mock_get_at_risk_service_instances, mock.patch(
'time.sleep', autospec=True, side_effect=LoopBreak):
assert not self.watcher.is_ready
with raises(LoopBreak):
self.watcher.run()
assert self.watcher.is_ready
assert not mock_get_at_risk_service_instances.called
mock_get_draining_hosts.return_value = ['host1', 'host2']
mock_get_at_risk_service_instances.return_value = ['si1', 'si2']
with raises(LoopBreak):
self.watcher.run()
mock_get_at_risk_service_instances.assert_called_with(
self.watcher, ['host1', 'host2'])
calls = [mock.call('si1'), mock.call('si2')]
self.mock_inbox_q.put.assert_has_calls(calls)
mock_get_draining_hosts.return_value = ['host1', 'host2', 'host3']
with raises(LoopBreak):
self.watcher.run()
mock_get_at_risk_service_instances.assert_called_with(
self.watcher, ['host3'])
def test_get_at_risk_service_instances(self):
with mock.patch('paasta_tools.deployd.watchers.get_all_marathon_apps',
autospec=True) as mock_get_marathon_apps, mock.patch(
'time.time', autospec=True, return_value=1):
mock_marathon_apps = [
mock.Mock(tasks=[
mock.Mock(host='host1',
app_id='/universe.c137.configsha.gitsha'),
mock.Mock(host='host2',
app_id='/universe.c138.configsha.gitsha')
]),
mock.Mock(tasks=[
mock.Mock(host='host1',
app_id='/universe.c139.configsha.gitsha')
]),
mock.Mock(tasks=[
mock.Mock(host='host1',
app_id='/universe.c139.configsha.gitsha')
])
]
mock_get_marathon_apps.return_value = mock_marathon_apps
ret = self.watcher.get_at_risk_service_instances(['host1'])
expected = [
ServiceInstance(service='universe',
instance='c137',
bounce_by=1,
watcher=self.watcher.__class__.__name__,
bounce_timers=None,
failures=0),
ServiceInstance(service='universe',
instance='c139',
bounce_by=1,
watcher=self.watcher.__class__.__name__,
bounce_timers=None,
failures=0)
]
assert ret == expected
class TestMaintenanceWatcher(unittest.TestCase):
def setUp(self):
self.mock_instances_to_bounce = mock.Mock()
self.mock_marathon_client = mock.Mock()
mock_config = mock.Mock(
get_deployd_maintenance_polling_frequency=mock.Mock(
return_value=20),
get_cluster=mock.Mock(return_value="clustername"),
)
with mock.patch(
"paasta_tools.deployd.watchers.get_marathon_clients_from_config",
autospec=True,
):
self.watcher = MaintenanceWatcher(self.mock_instances_to_bounce,
"westeros-prod",
config=mock_config)
def test_get_new_draining_hosts(self):
with mock.patch("paasta_tools.deployd.watchers.get_draining_hosts",
autospec=True) as mock_get_draining_hosts:
mock_get_draining_hosts.return_value = ["host1", "host2"]
assert self.watcher.get_new_draining_hosts() == ["host1", "host2"]
assert self.watcher.draining == {"host1", "host2"}
mock_get_draining_hosts.return_value = ["host1"]
assert self.watcher.get_new_draining_hosts() == []
assert self.watcher.draining == {"host1"}
mock_get_draining_hosts.side_effect = RequestException
assert self.watcher.get_new_draining_hosts() == []
assert self.watcher.draining == {"host1"}
mock_get_draining_hosts.side_effect = None
mock_get_draining_hosts.return_value = ["host3", "host1"]
assert self.watcher.get_new_draining_hosts() == ["host3"]
assert self.watcher.draining == {"host1", "host3"}
mock_get_draining_hosts.return_value = []
assert self.watcher.get_new_draining_hosts() == []
assert self.watcher.draining == set()
def test_run(self):
with mock.patch(
"paasta_tools.deployd.watchers.MaintenanceWatcher.get_new_draining_hosts",
autospec=True,
) as mock_get_new_draining_hosts, mock.patch(
"paasta_tools.deployd.watchers.MaintenanceWatcher.get_at_risk_service_instances",
autospec=True,
) as mock_get_at_risk_service_instances, mock.patch(
"time.sleep", autospec=True, side_effect=LoopBreak):
mock_get_new_draining_hosts.return_value = []
assert not self.watcher.is_ready
with raises(LoopBreak):
self.watcher.run()
assert self.watcher.is_ready
assert not mock_get_at_risk_service_instances.called
mock_get_new_draining_hosts.return_value = ["host1", "host2"]
mock_get_at_risk_service_instances.return_value = ["si1", "si2"]
with raises(LoopBreak):
self.watcher.run()
mock_get_at_risk_service_instances.assert_called_with(
self.watcher, ["host1", "host2"])
calls = [mock.call("si1"), mock.call("si2")]
self.mock_instances_to_bounce.put.assert_has_calls(calls)
def test_get_at_risk_service_instances(self):
with mock.patch(
"paasta_tools.deployd.watchers.get_marathon_apps_with_clients",
autospec=True,
) as mock_get_marathon_apps, mock.patch("time.time",
autospec=True,
return_value=1):
mock_marathon_apps = [
mock.Mock(tasks=[
mock.Mock(host="host1",
app_id="/universe.c137.configsha.gitsha"),
mock.Mock(host="host2",
app_id="/universe.c138.configsha.gitsha"),
]),
mock.Mock(tasks=[
mock.Mock(host="host1",
app_id="/universe.c139.configsha.gitsha")
]),
mock.Mock(tasks=[
mock.Mock(host="host1",
app_id="/universe.c139.configsha.gitsha")
]),
]
mock_client = mock.Mock()
mock_get_marathon_apps.return_value = [
(app, mock_client) for app in mock_marathon_apps
]
ret = self.watcher.get_at_risk_service_instances(["host1"])
expected = [
ServiceInstance(
service="universe",
instance="c137",
bounce_by=1,
wait_until=1,
watcher=self.watcher.__class__.__name__,
failures=0,
processed_count=0,
enqueue_time=1,
bounce_start_time=1,
),
ServiceInstance(
service="universe",
instance="c139",
bounce_by=1,
wait_until=1,
watcher=self.watcher.__class__.__name__,
failures=0,
processed_count=0,
enqueue_time=1,
bounce_start_time=1,
),
]
assert ret == expected
class TestMaintenanceWatcher(unittest.TestCase):
def setUp(self):
self.mock_instances_to_bounce_later = mock.Mock()
self.mock_marathon_client = mock.Mock()
mock_config = mock.Mock(
get_deployd_maintenance_polling_frequency=mock.Mock(
return_value=20))
with mock.patch(
'paasta_tools.deployd.watchers.get_marathon_clients_from_config',
autospec=True,
):
self.watcher = MaintenanceWatcher(
self.mock_instances_to_bounce_later,
"westeros-prod",
config=mock_config,
)
def test_get_new_draining_hosts(self):
with mock.patch(
'paasta_tools.deployd.watchers.get_draining_hosts',
autospec=True,
) as mock_get_draining_hosts:
mock_get_draining_hosts.return_value = ['host1', 'host2']
assert self.watcher.get_new_draining_hosts() == ['host1', 'host2']
assert self.watcher.draining == {'host1', 'host2'}
mock_get_draining_hosts.return_value = ['host1']
assert self.watcher.get_new_draining_hosts() == []
assert self.watcher.draining == {'host1'}
mock_get_draining_hosts.side_effect = RequestException
assert self.watcher.get_new_draining_hosts() == []
assert self.watcher.draining == {'host1'}
mock_get_draining_hosts.side_effect = None
mock_get_draining_hosts.return_value = ['host3', 'host1']
assert self.watcher.get_new_draining_hosts() == ['host3']
assert self.watcher.draining == {'host1', 'host3'}
mock_get_draining_hosts.return_value = []
assert self.watcher.get_new_draining_hosts() == []
assert self.watcher.draining == set()
def test_run(self):
with mock.patch(
'paasta_tools.deployd.watchers.MaintenanceWatcher.get_new_draining_hosts',
autospec=True,
) as mock_get_new_draining_hosts, mock.patch(
'paasta_tools.deployd.watchers.MaintenanceWatcher.get_at_risk_service_instances',
autospec=True,
) as mock_get_at_risk_service_instances, mock.patch(
'time.sleep',
autospec=True,
side_effect=LoopBreak,
):
mock_get_new_draining_hosts.return_value = []
assert not self.watcher.is_ready
with raises(LoopBreak):
self.watcher.run()
assert self.watcher.is_ready
assert not mock_get_at_risk_service_instances.called
mock_get_new_draining_hosts.return_value = ['host1', 'host2']
mock_get_at_risk_service_instances.return_value = ['si1', 'si2']
with raises(LoopBreak):
self.watcher.run()
mock_get_at_risk_service_instances.assert_called_with(
self.watcher, ['host1', 'host2'])
calls = [
mock.call('si1'),
mock.call('si2'),
]
self.mock_instances_to_bounce_later.put.assert_has_calls(calls)
def test_get_at_risk_service_instances(self):
with mock.patch(
'paasta_tools.deployd.common.get_priority',
autospec=True,
return_value=0,
), mock.patch(
'paasta_tools.deployd.watchers.get_marathon_apps_with_clients',
autospec=True,
) as mock_get_marathon_apps, mock.patch(
'time.time',
autospec=True,
return_value=1,
):
mock_marathon_apps = [
mock.Mock(tasks=[
mock.Mock(
host='host1',
app_id='/universe.c137.configsha.gitsha',
),
mock.Mock(
host='host2',
app_id='/universe.c138.configsha.gitsha',
),
]),
mock.Mock(tasks=[
mock.Mock(
host='host1',
app_id='/universe.c139.configsha.gitsha',
)
]),
mock.Mock(tasks=[
mock.Mock(
host='host1',
app_id='/universe.c139.configsha.gitsha',
)
]),
]
mock_client = mock.Mock()
mock_get_marathon_apps.return_value = [
(app, mock_client) for app in mock_marathon_apps
]
ret = self.watcher.get_at_risk_service_instances(['host1'])
expected = [
BaseServiceInstance(
service='universe',
instance='c137',
bounce_by=1,
watcher=self.watcher.__class__.__name__,
priority=0,
bounce_timers=None,
failures=0,
),
BaseServiceInstance(
service='universe',
instance='c139',
bounce_by=1,
watcher=self.watcher.__class__.__name__,
priority=0,
bounce_timers=None,
failures=0,
),
]
assert ret == expected