def test_policy(self):
"""Test scale policy."""
zkclient = mock.Mock()
alerter = mock.Mock()
state = {
'scheduled': {
'foo.bar': ['foo.bar#1', 'foo.bar#2'],
},
'monitors': {
'foo.bar': {
'count': 2,
'available': 2,
'rate': 1.0,
'last_update': 100,
},
},
'suspended': {},
}
time.time.return_value = 101
parametrizes = (
('fifo', 'foo.bar#3', 'foo.bar#1', True),
('lifo', 'foo.bar#4', 'foo.bar#4', True),
('unknown', 'foo.bar#5', None, False),
)
for (policy, created, deleted, api_is_called) in parametrizes:
state['monitors']['foo.bar']['policy'] = policy
state['scheduled']['foo.bar'].append(created)
appmonitor.reevaluate('/cellapi.sock', alerter, state, zkclient,
{})
if api_is_called:
restclient.post.assert_called_with(
['/cellapi.sock'],
'/instance/_bulk/delete',
payload={'instances': [deleted]},
headers={'X-Treadmill-Trusted-Agent': 'monitor'})
state['scheduled']['foo.bar'].remove(deleted)
restclient.post.reset_mock()
else:
restclient.post.assert_not_called()
def test_reevaluate_notfound(self):
"""Test state reevaluation."""
state = {
'scheduled': {
'foo.bar': [],
},
'monitors': {
'foo.bar': {
'count': 1,
'available': 0,
'rate': 1.0,
'last_update': 100,
},
},
'delayed': {},
}
time.time.return_value = 101
restclient.post.side_effect = restclient.NotFoundError('xxx')
appmonitor.reevaluate('/cellapi.sock', state)
self.assertTrue(restclient.post.called)
self.assertIn('foo.bar', state['delayed'])
self.assertEqual(state['delayed']['foo.bar'], float(101 + 300))
restclient.post.reset_mock()
time.time.return_value = 102
appmonitor.reevaluate('/cellapi.sock', state)
self.assertFalse(restclient.post.called)
self.assertIn('foo.bar', state['delayed'])
# Delay did not increase
self.assertEqual(state['delayed']['foo.bar'], float(101 + 300))
# After time reached, call will happen, fail, and delay will be
# extended.
restclient.post.reset_mock()
time.time.return_value = 500
appmonitor.reevaluate('/cellapi.sock', state)
self.assertTrue(restclient.post.called)
self.assertIn('foo.bar', state['delayed'])
self.assertEqual(state['delayed']['foo.bar'], float(500 + 300))
# More time pass, and this time call will succeed - application will
# be removed from delay dict.
restclient.post.reset_mock()
restclient.post.return_value = ()
restclient.post.side_effect = None
time.time.return_value = 500 + 300 + 1
appmonitor.reevaluate('/cellapi.sock', state)
self.assertTrue(restclient.post.called)
self.assertNotIn('foo.bar', state['delayed'])
def test_reevaluate(self):
"""Test state reevaluation."""
state = {
'scheduled': {
'foo.bar': ['foo.bar#1', 'foo.bar.#2'],
'foo.baz': ['foo.baz#3', 'foo.baz.#4'],
},
'monitors': {
'foo.bar': 2,
'foo.baz': 2,
}
}
instance_api = mock.MagicMock()
appmonitor.reevaluate(instance_api, state)
self.assertFalse(instance_api.create.called)
self.assertFalse(instance_api.delete.called)
state['monitors']['foo.bar'] = 3
appmonitor.reevaluate(instance_api, state)
instance_api.create.assert_called_with('foo.bar', {}, count=1)
state['scheduled']['foo.baz'].append('foo.baz#5')
appmonitor.reevaluate(instance_api, state)
instance_api.delete.assert_called_with('foo.baz#3')
def test_reevaluate(self):
"""Test state reevaluation."""
zkclient = mock.Mock()
alerter = mock.Mock()
state = {
'scheduled': {
'foo.bar': ['foo.bar#1', 'foo.bar.#2'],
'foo.baz': ['foo.baz#3', 'foo.baz.#4'],
},
'monitors': {
'foo.bar': {
'count': 2,
'available': 4,
'rate': 1.0,
'last_update': 100,
},
'foo.baz': {
'count': 2,
'available': 2,
'rate': 1.0,
'last_update': 100,
},
},
'suspended': {},
}
time.time.return_value = 101
appmonitor.reevaluate('/cellapi.sock', alerter, state, zkclient, {})
self.assertFalse(restclient.post.called)
self.assertFalse(alerter.called)
state['scheduled']['foo.baz'].append('foo.baz#5')
appmonitor.reevaluate('/cellapi.sock', alerter, state, zkclient, {})
restclient.post.assert_called_with(
['/cellapi.sock'],
'/instance/_bulk/delete',
payload={'instances': ['foo.baz#3']},
headers={'X-Treadmill-Trusted-Agent': 'monitor'})
self.assertFalse(alerter.called)
self.assertEqual(101, state['monitors']['foo.bar']['last_update'])
self.assertEqual(101, state['monitors']['foo.baz']['last_update'])
# Remove instance from mock state.
state['scheduled']['foo.baz'] = [
elt for elt in state['scheduled']['foo.baz'] if elt != 'foo.baz#3'
]
# Two missing.
restclient.post.reset_mock()
time.time.return_value = 102
state['scheduled']['foo.bar'] = []
appmonitor.reevaluate('/cellapi.sock', alerter, state, zkclient, {})
self.assertEqual(102, state['monitors']['foo.bar']['last_update'])
self.assertEqual(2.0, state['monitors']['foo.bar']['available'])
# Instance match count, after 1 sec with rate of 1/s, available will
# be 3.0
restclient.post.reset_mock()
time.time.return_value = 103
state['scheduled']['foo.bar'] = ['foo.bar#5', 'foo.bar#6']
appmonitor.reevaluate('/cellapi.sock', alerter, state, zkclient, {})
self.assertEqual(103, state['monitors']['foo.bar']['last_update'])
self.assertEqual(3.0, state['monitors']['foo.bar']['available'])
# Need to create two instance, 3 available.
restclient.post.reset_mock()
state['scheduled']['foo.bar'] = []
appmonitor.reevaluate('/cellapi.sock', alerter, state, zkclient, {})
restclient.post.assert_called_with(
['/cellapi.sock'],
'/instance/foo.bar?count=2',
payload={},
headers={'X-Treadmill-Trusted-Agent': 'monitor'})
self.assertFalse(alerter.called)
self.assertEqual(1.0, state['monitors']['foo.bar']['available'])
last_waited = appmonitor.reevaluate('/cellapi.sock', alerter, state,
zkclient, {})
restclient.post.assert_called_with(
['/cellapi.sock'],
'/instance/foo.bar?count=1',
payload={},
headers={'X-Treadmill-Trusted-Agent': 'monitor'})
self.assertFalse(alerter.called)
self.assertEqual(0.0, state['monitors']['foo.bar']['available'])
self.assertEqual(last_waited, {})
# No available, create not called.
restclient.post.reset_mock()
state['scheduled']['foo.bar'] = []
last_waited = appmonitor.reevaluate('/cellapi.sock', alerter, state,
zkclient, {})
self.assertFalse(restclient.post.called)
alerter.assert_called_with('foo.bar',
'Monitor suspended: Rate limited')
self.assertEqual(last_waited, {'foo.bar': 104})
time.time.return_value = 104
last_waited = appmonitor.reevaluate('/cellapi.sock', alerter, state,
zkclient, last_waited)
restclient.post.assert_called_with(
['/cellapi.sock'],
'/instance/foo.bar?count=1',
payload={},
headers={'X-Treadmill-Trusted-Agent': 'monitor'})
alerter.assert_called_with('foo.bar',
'Monitor active again',
status='clear')
self.assertEqual(0.0, state['monitors']['foo.bar']['available'])
self.assertEqual(last_waited, {})
def test_reevaluate_notfound(self):
"""Test state reevaluation."""
zkclient = mock.Mock()
alerter = mock.Mock()
state = {
'scheduled': {
'foo.bar': [],
},
'monitors': {
'foo.bar': {
'count': 1,
'available': 0,
'rate': 1.0,
'last_update': 100,
},
},
'suspended': {},
}
time.time.return_value = 101
restclient.post.side_effect = restclient.NotFoundError('xxx')
appmonitor.reevaluate('/cellapi.sock', alerter, state, zkclient, {})
self.assertTrue(restclient.post.called)
self.assertIn('foo.bar', state['suspended'])
self.assertEqual(state['suspended']['foo.bar'], float(101 + 300))
zkutils.update.assert_called_with(zkclient, '/app-monitors',
{'foo.bar': float(101 + 300)})
alerter.assert_called_with('foo.bar',
'Monitor suspended: App not configured')
restclient.post.reset_mock()
time.time.return_value = 102
appmonitor.reevaluate('/cellapi.sock', alerter, state, zkclient, {})
self.assertFalse(restclient.post.called)
self.assertIn('foo.bar', state['suspended'])
alerter.assert_called_with('foo.bar',
'Monitor suspended: App not configured')
# Delay did not increase
self.assertEqual(state['suspended']['foo.bar'], float(101 + 300))
# After time reached, call will happen, fail, and delay will be
# extended.
alerter.reset_mock()
restclient.post.reset_mock()
time.time.return_value = 500
last_waited = appmonitor.reevaluate('/cellapi.sock', alerter, state,
zkclient, {})
self.assertTrue(restclient.post.called)
alerter.assert_called_with('foo.bar',
'Monitor suspended: App not configured')
self.assertIn('foo.bar', state['suspended'])
self.assertEqual(state['suspended']['foo.bar'], float(500 + 300))
zkutils.update.assert_called_with(zkclient, '/app-monitors',
{'foo.bar': float(500 + 300)})
self.assertEqual(last_waited, {'foo.bar': 800.0})
# More time pass, and this time call will succeed - application will
# be removed from delay dict.
alerter.reset_mock()
restclient.post.reset_mock()
restclient.post.return_value = ()
restclient.post.side_effect = None
time.time.return_value = 500 + 300 + 1
last_waited = appmonitor.reevaluate('/cellapi.sock', alerter, state,
zkclient, last_waited)
self.assertTrue(restclient.post.called)
self.assertNotIn('foo.bar', state['suspended'])
alerter.assert_called_with('foo.bar',
'Monitor active again',
status='clear')
zkutils.update.assert_called_with(zkclient, '/app-monitors', {})
self.assertEqual(last_waited, {})
def test_reevaluate(self):
"""Test state reevaluation."""
state = {
'scheduled': {
'foo.bar': ['foo.bar#1', 'foo.bar.#2'],
'foo.baz': ['foo.baz#3', 'foo.baz.#4'],
},
'monitors': {
'foo.bar': {
'count': 2,
'available': 4,
'rate': 1.0,
'last_update': 100,
},
'foo.baz': {
'count': 2,
'available': 2,
'rate': 1.0,
'last_update': 100,
},
}
}
time.time.return_value = 101
instance_api = mock.MagicMock()
appmonitor.reevaluate(instance_api, state)
self.assertFalse(instance_api.create.called)
self.assertFalse(instance_api.delete.called)
state['scheduled']['foo.baz'].append('foo.baz#5')
appmonitor.reevaluate(instance_api, state)
instance_api.delete.assert_called_with('foo.baz#3')
self.assertEquals(101, state['monitors']['foo.bar']['last_update'])
self.assertEquals(101, state['monitors']['foo.baz']['last_update'])
# Two missing.
time.time.return_value = 102
state['scheduled']['foo.bar'] = []
appmonitor.reevaluate(instance_api, state)
self.assertEquals(102, state['monitors']['foo.bar']['last_update'])
self.assertEquals(2.0, state['monitors']['foo.bar']['available'])
# Instance match count, after 1 sec with rate of 1/s, available will
# be 3.0
time.time.return_value = 103
state['scheduled']['foo.bar'] = ['foo.bar#5', 'foo.bar#6']
appmonitor.reevaluate(instance_api, state)
self.assertEquals(103, state['monitors']['foo.bar']['last_update'])
self.assertEquals(3.0, state['monitors']['foo.bar']['available'])
# Need to create two instance, 3 available.
instance_api.create.reset_mock()
state['scheduled']['foo.bar'] = []
appmonitor.reevaluate(instance_api, state)
instance_api.create.assert_called_with('foo.bar', {}, count=2)
self.assertEquals(1.0, state['monitors']['foo.bar']['available'])
instance_api.create.reset_mock()
appmonitor.reevaluate(instance_api, state)
instance_api.create.assert_called_with('foo.bar', {}, count=1)
self.assertEquals(0.0, state['monitors']['foo.bar']['available'])
# No available, create not called.
instance_api.create.reset_mock()
appmonitor.reevaluate(instance_api, state)
self.assertFalse(instance_api.create.called)
time.time.return_value = 104
appmonitor.reevaluate(instance_api, state)
instance_api.create.assert_called_with('foo.bar', {}, count=1)
self.assertEquals(0.0, state['monitors']['foo.bar']['available'])
