def test_policy(self): """Test scale policy.""" zkclient = mock.Mock() alerter = mock.Mock() state = { 'scheduled': { 'foo.bar': ['foo.bar#1', 'foo.bar#2'], }, 'monitors': { 'foo.bar': { 'count': 2, 'available': 2, 'rate': 1.0, 'last_update': 100, }, }, 'suspended': {}, } time.time.return_value = 101 parametrizes = ( ('fifo', 'foo.bar#3', 'foo.bar#1', True), ('lifo', 'foo.bar#4', 'foo.bar#4', True), ('unknown', 'foo.bar#5', None, False), ) for (policy, created, deleted, api_is_called) in parametrizes: state['monitors']['foo.bar']['policy'] = policy state['scheduled']['foo.bar'].append(created) appmonitor.reevaluate('/cellapi.sock', alerter, state, zkclient, {}) if api_is_called: restclient.post.assert_called_with( ['/cellapi.sock'], '/instance/_bulk/delete', payload={'instances': [deleted]}, headers={'X-Treadmill-Trusted-Agent': 'monitor'}) state['scheduled']['foo.bar'].remove(deleted) restclient.post.reset_mock() else: restclient.post.assert_not_called()
def test_reevaluate_notfound(self): """Test state reevaluation.""" state = { 'scheduled': { 'foo.bar': [], }, 'monitors': { 'foo.bar': { 'count': 1, 'available': 0, 'rate': 1.0, 'last_update': 100, }, }, 'delayed': {}, } time.time.return_value = 101 restclient.post.side_effect = restclient.NotFoundError('xxx') appmonitor.reevaluate('/cellapi.sock', state) self.assertTrue(restclient.post.called) self.assertIn('foo.bar', state['delayed']) self.assertEqual(state['delayed']['foo.bar'], float(101 + 300)) restclient.post.reset_mock() time.time.return_value = 102 appmonitor.reevaluate('/cellapi.sock', state) self.assertFalse(restclient.post.called) self.assertIn('foo.bar', state['delayed']) # Delay did not increase self.assertEqual(state['delayed']['foo.bar'], float(101 + 300)) # After time reached, call will happen, fail, and delay will be # extended. restclient.post.reset_mock() time.time.return_value = 500 appmonitor.reevaluate('/cellapi.sock', state) self.assertTrue(restclient.post.called) self.assertIn('foo.bar', state['delayed']) self.assertEqual(state['delayed']['foo.bar'], float(500 + 300)) # More time pass, and this time call will succeed - application will # be removed from delay dict. restclient.post.reset_mock() restclient.post.return_value = () restclient.post.side_effect = None time.time.return_value = 500 + 300 + 1 appmonitor.reevaluate('/cellapi.sock', state) self.assertTrue(restclient.post.called) self.assertNotIn('foo.bar', state['delayed'])
def test_reevaluate(self): """Test state reevaluation.""" state = { 'scheduled': { 'foo.bar': ['foo.bar#1', 'foo.bar.#2'], 'foo.baz': ['foo.baz#3', 'foo.baz.#4'], }, 'monitors': { 'foo.bar': 2, 'foo.baz': 2, } } instance_api = mock.MagicMock() appmonitor.reevaluate(instance_api, state) self.assertFalse(instance_api.create.called) self.assertFalse(instance_api.delete.called) state['monitors']['foo.bar'] = 3 appmonitor.reevaluate(instance_api, state) instance_api.create.assert_called_with('foo.bar', {}, count=1) state['scheduled']['foo.baz'].append('foo.baz#5') appmonitor.reevaluate(instance_api, state) instance_api.delete.assert_called_with('foo.baz#3')
def test_reevaluate(self): """Test state reevaluation.""" zkclient = mock.Mock() alerter = mock.Mock() state = { 'scheduled': { 'foo.bar': ['foo.bar#1', 'foo.bar.#2'], 'foo.baz': ['foo.baz#3', 'foo.baz.#4'], }, 'monitors': { 'foo.bar': { 'count': 2, 'available': 4, 'rate': 1.0, 'last_update': 100, }, 'foo.baz': { 'count': 2, 'available': 2, 'rate': 1.0, 'last_update': 100, }, }, 'suspended': {}, } time.time.return_value = 101 appmonitor.reevaluate('/cellapi.sock', alerter, state, zkclient, {}) self.assertFalse(restclient.post.called) self.assertFalse(alerter.called) state['scheduled']['foo.baz'].append('foo.baz#5') appmonitor.reevaluate('/cellapi.sock', alerter, state, zkclient, {}) restclient.post.assert_called_with( ['/cellapi.sock'], '/instance/_bulk/delete', payload={'instances': ['foo.baz#3']}, headers={'X-Treadmill-Trusted-Agent': 'monitor'}) self.assertFalse(alerter.called) self.assertEqual(101, state['monitors']['foo.bar']['last_update']) self.assertEqual(101, state['monitors']['foo.baz']['last_update']) # Remove instance from mock state. state['scheduled']['foo.baz'] = [ elt for elt in state['scheduled']['foo.baz'] if elt != 'foo.baz#3' ] # Two missing. restclient.post.reset_mock() time.time.return_value = 102 state['scheduled']['foo.bar'] = [] appmonitor.reevaluate('/cellapi.sock', alerter, state, zkclient, {}) self.assertEqual(102, state['monitors']['foo.bar']['last_update']) self.assertEqual(2.0, state['monitors']['foo.bar']['available']) # Instance match count, after 1 sec with rate of 1/s, available will # be 3.0 restclient.post.reset_mock() time.time.return_value = 103 state['scheduled']['foo.bar'] = ['foo.bar#5', 'foo.bar#6'] appmonitor.reevaluate('/cellapi.sock', alerter, state, zkclient, {}) self.assertEqual(103, state['monitors']['foo.bar']['last_update']) self.assertEqual(3.0, state['monitors']['foo.bar']['available']) # Need to create two instance, 3 available. restclient.post.reset_mock() state['scheduled']['foo.bar'] = [] appmonitor.reevaluate('/cellapi.sock', alerter, state, zkclient, {}) restclient.post.assert_called_with( ['/cellapi.sock'], '/instance/foo.bar?count=2', payload={}, headers={'X-Treadmill-Trusted-Agent': 'monitor'}) self.assertFalse(alerter.called) self.assertEqual(1.0, state['monitors']['foo.bar']['available']) last_waited = appmonitor.reevaluate('/cellapi.sock', alerter, state, zkclient, {}) restclient.post.assert_called_with( ['/cellapi.sock'], '/instance/foo.bar?count=1', payload={}, headers={'X-Treadmill-Trusted-Agent': 'monitor'}) self.assertFalse(alerter.called) self.assertEqual(0.0, state['monitors']['foo.bar']['available']) self.assertEqual(last_waited, {}) # No available, create not called. restclient.post.reset_mock() state['scheduled']['foo.bar'] = [] last_waited = appmonitor.reevaluate('/cellapi.sock', alerter, state, zkclient, {}) self.assertFalse(restclient.post.called) alerter.assert_called_with('foo.bar', 'Monitor suspended: Rate limited') self.assertEqual(last_waited, {'foo.bar': 104}) time.time.return_value = 104 last_waited = appmonitor.reevaluate('/cellapi.sock', alerter, state, zkclient, last_waited) restclient.post.assert_called_with( ['/cellapi.sock'], '/instance/foo.bar?count=1', payload={}, headers={'X-Treadmill-Trusted-Agent': 'monitor'}) alerter.assert_called_with('foo.bar', 'Monitor active again', status='clear') self.assertEqual(0.0, state['monitors']['foo.bar']['available']) self.assertEqual(last_waited, {})
def test_reevaluate_notfound(self): """Test state reevaluation.""" zkclient = mock.Mock() alerter = mock.Mock() state = { 'scheduled': { 'foo.bar': [], }, 'monitors': { 'foo.bar': { 'count': 1, 'available': 0, 'rate': 1.0, 'last_update': 100, }, }, 'suspended': {}, } time.time.return_value = 101 restclient.post.side_effect = restclient.NotFoundError('xxx') appmonitor.reevaluate('/cellapi.sock', alerter, state, zkclient, {}) self.assertTrue(restclient.post.called) self.assertIn('foo.bar', state['suspended']) self.assertEqual(state['suspended']['foo.bar'], float(101 + 300)) zkutils.update.assert_called_with(zkclient, '/app-monitors', {'foo.bar': float(101 + 300)}) alerter.assert_called_with('foo.bar', 'Monitor suspended: App not configured') restclient.post.reset_mock() time.time.return_value = 102 appmonitor.reevaluate('/cellapi.sock', alerter, state, zkclient, {}) self.assertFalse(restclient.post.called) self.assertIn('foo.bar', state['suspended']) alerter.assert_called_with('foo.bar', 'Monitor suspended: App not configured') # Delay did not increase self.assertEqual(state['suspended']['foo.bar'], float(101 + 300)) # After time reached, call will happen, fail, and delay will be # extended. alerter.reset_mock() restclient.post.reset_mock() time.time.return_value = 500 last_waited = appmonitor.reevaluate('/cellapi.sock', alerter, state, zkclient, {}) self.assertTrue(restclient.post.called) alerter.assert_called_with('foo.bar', 'Monitor suspended: App not configured') self.assertIn('foo.bar', state['suspended']) self.assertEqual(state['suspended']['foo.bar'], float(500 + 300)) zkutils.update.assert_called_with(zkclient, '/app-monitors', {'foo.bar': float(500 + 300)}) self.assertEqual(last_waited, {'foo.bar': 800.0}) # More time pass, and this time call will succeed - application will # be removed from delay dict. alerter.reset_mock() restclient.post.reset_mock() restclient.post.return_value = () restclient.post.side_effect = None time.time.return_value = 500 + 300 + 1 last_waited = appmonitor.reevaluate('/cellapi.sock', alerter, state, zkclient, last_waited) self.assertTrue(restclient.post.called) self.assertNotIn('foo.bar', state['suspended']) alerter.assert_called_with('foo.bar', 'Monitor active again', status='clear') zkutils.update.assert_called_with(zkclient, '/app-monitors', {}) self.assertEqual(last_waited, {})
def test_reevaluate(self): """Test state reevaluation.""" state = { 'scheduled': { 'foo.bar': ['foo.bar#1', 'foo.bar.#2'], 'foo.baz': ['foo.baz#3', 'foo.baz.#4'], }, 'monitors': { 'foo.bar': { 'count': 2, 'available': 4, 'rate': 1.0, 'last_update': 100, }, 'foo.baz': { 'count': 2, 'available': 2, 'rate': 1.0, 'last_update': 100, }, } } time.time.return_value = 101 instance_api = mock.MagicMock() appmonitor.reevaluate(instance_api, state) self.assertFalse(instance_api.create.called) self.assertFalse(instance_api.delete.called) state['scheduled']['foo.baz'].append('foo.baz#5') appmonitor.reevaluate(instance_api, state) instance_api.delete.assert_called_with('foo.baz#3') self.assertEquals(101, state['monitors']['foo.bar']['last_update']) self.assertEquals(101, state['monitors']['foo.baz']['last_update']) # Two missing. time.time.return_value = 102 state['scheduled']['foo.bar'] = [] appmonitor.reevaluate(instance_api, state) self.assertEquals(102, state['monitors']['foo.bar']['last_update']) self.assertEquals(2.0, state['monitors']['foo.bar']['available']) # Instance match count, after 1 sec with rate of 1/s, available will # be 3.0 time.time.return_value = 103 state['scheduled']['foo.bar'] = ['foo.bar#5', 'foo.bar#6'] appmonitor.reevaluate(instance_api, state) self.assertEquals(103, state['monitors']['foo.bar']['last_update']) self.assertEquals(3.0, state['monitors']['foo.bar']['available']) # Need to create two instance, 3 available. instance_api.create.reset_mock() state['scheduled']['foo.bar'] = [] appmonitor.reevaluate(instance_api, state) instance_api.create.assert_called_with('foo.bar', {}, count=2) self.assertEquals(1.0, state['monitors']['foo.bar']['available']) instance_api.create.reset_mock() appmonitor.reevaluate(instance_api, state) instance_api.create.assert_called_with('foo.bar', {}, count=1) self.assertEquals(0.0, state['monitors']['foo.bar']['available']) # No available, create not called. instance_api.create.reset_mock() appmonitor.reevaluate(instance_api, state) self.assertFalse(instance_api.create.called) time.time.return_value = 104 appmonitor.reevaluate(instance_api, state) instance_api.create.assert_called_with('foo.bar', {}, count=1) self.assertEquals(0.0, state['monitors']['foo.bar']['available'])