def before(self): self.status = 'online' self.system_info = SystemInfo().format() self.worker_status = { 'worker_status': { 'system_info': self.system_info, 'status': self.status } } self.bad_status = 'bad_status' self.bad_system_info = SystemInfo() self.bad_worker_status = { 'worker_status': { 'system_info': self.system_info, 'status': self.bad_status } } self.req = MagicMock() self.resp = MagicMock() self.registration = WorkerRegistration('worker').format() self.worker_dict = Worker(**self.registration).format() self.worker_not_found = None self.db_handler = MagicMock() self.resource = WorkerStatusResource(self.db_handler) self.worker_id = '51375fc4eea50d53066292b6' self.test_route = '/v1/worker/{worker_id}/status' self.api.add_route(self.test_route, self.resource)
class WhenTestingWorkerObject(unittest.TestCase): def setUp(self): self.system_info = SystemInfo() self.test_worker = Worker(_id='010101', worker_token='9876543210', hostname='worker01', callback='172.22.15.25:8080/v1/config/', ip_address_v4='172.23.1.100', ip_address_v6='::1', personality='worker', status='new', system_info=self.system_info.format()) self.test_worker_lite = Worker(hostname='worker01', callback='172.22.15.25:8080/v1/config/', ip_address_v4='172.23.1.100', ip_address_v6='::1', personality='worker', status='new', system_info=self.system_info.format()) self.worker_status = self.test_worker.get_status() def test_new_worker_format_for_save(self): self.assertTrue('_id' in self.test_worker.format_for_save()) def test_get_status(self): self.assertEqual(self.worker_status['hostname'], 'worker01') self.assertEqual(self.worker_status['personality'], 'worker') self.assertEqual(self.worker_status['status'], 'new') self.assertEqual(self.worker_status['system_info'], self.system_info.format())
def test_new_system_info_empty_obj(self): system_info = SystemInfo() system_dict = system_info.format() self.assertTrue('cpu_cores' in system_dict) self.assertTrue('os_type' in system_dict) self.assertTrue('memory_mb' in system_dict) self.assertTrue('architecture' in system_dict) self.assertTrue('load_average' in system_dict) self.assertTrue('disk_usage' in system_dict)
def test_new_system_info_empty_obj(self): system_info = SystemInfo() system_dict = system_info.format() self.assertTrue('cpu_cores' in system_dict) self.assertTrue('os_type' in system_dict) self.assertTrue('memory_mb' in system_dict) self.assertTrue('architecture' in system_dict) self.assertTrue('load_average' in system_dict) self.assertTrue('disk_usage' in system_dict) self.assertTrue('timestamp' in system_dict)
class WhenTestingWorkerObject(unittest.TestCase): def setUp(self): self.system_info = SystemInfo() self.test_worker = Worker(_id='010101', worker_id='0123456789', worker_token='9876543210', hostname='worker01', callback='172.22.15.25:8080/v1/config/', ip_address_v4='172.23.1.100', ip_address_v6='::1', personality='correlation', status='new', system_info=self.system_info.format()) self.test_worker_lite = Worker(hostname='worker01', callback='172.22.15.25:8080/v1/config/', ip_address_v4='172.23.1.100', ip_address_v6='::1', personality='correlation', status='new', system_info=self.system_info.format()) self.worker_route = self.test_worker.get_route_info() self.worker_status = self.test_worker.get_status() def test_new_worker_no_ids(self): worker_dict = self.test_worker_lite.format() self.assertTrue('worker_id' in worker_dict) self.assertTrue('worker_token' in worker_dict) def test_new_worker_format_for_save(self): self.assertTrue('_id' in self.test_worker.format_for_save()) def test_new_worker_get_registration_identity(self): self.assertTrue('personality_module' in self.test_worker.get_registration_identity()) def test_new_worker_routes(self): self.assertEqual(self.worker_route['worker_id'], '0123456789') self.assertEqual(self.worker_route['ip_address_v4'], '172.23.1.100') self.assertEqual(self.worker_route['ip_address_v6'], '::1') def test_get_status(self): self.assertEqual(self.worker_status['hostname'], 'worker01') self.assertEqual(self.worker_status['worker_id'], '0123456789') self.assertEqual(self.worker_status['personality'], 'correlation') self.assertEqual(self.worker_status['status'], 'new') self.assertEqual(self.worker_status['system_info'], self.system_info.format()) self.assertEqual(self.worker_route['ip_address_v4'], '172.23.1.100') self.assertEqual(self.worker_route['ip_address_v6'], '::1')
def on_put(self, req, resp, hostname, validated_body): """ updates a worker's status or creates a new worker entry if not found """ #load validated json payload in body body = validated_body['worker_status'] #find the worker in db worker = worker_util.find_worker(hostname) if worker is None: #instantiate new worker object new_worker = Worker(**body) #persist the new worker worker_util.create_worker(new_worker) resp.status = falcon.HTTP_202 return if 'status' in body: worker.status = body['status'] if 'system_info' in body: worker.system_info = SystemInfo(**body['system_info']) worker_util.save_worker(worker) resp.status = falcon.HTTP_200
def test_when_load_average_is_negative_then_should_return_http_400(self): with patch('meniscus.data.model.worker_util.find_worker', MagicMock(return_value=self.worker)): system_info = SystemInfo() system_info.load_average = {"1": -2, "15": -2, "5": -2} self.simulate_request( self.test_route, method='PUT', headers={'content-type': 'application/json'}, body=jsonutils.dumps({ 'worker_status': { 'system_info': system_info.format(), 'status': self.status } })) self.assertEqual(falcon.HTTP_400, self.srmock.status)
def test_when_load_average_is_negative_then_should_return_http_400(self): with patch('meniscus.data.model.worker_util.find_worker', MagicMock(return_value=self.worker)): system_info = SystemInfo() system_info.load_average = {"1": -2, "15": -2, "5": -2} self.simulate_request( self.test_route, method='PUT', headers={'content-type': 'application/json'}, body=jsonutils.dumps({ 'worker_status': { 'hostname': self.hostname, 'system_info': system_info.format(), 'status': self.status } })) self.assertEqual(falcon.HTTP_400, self.srmock.status)
def before(self): self.status = 'online' self.hostname = 'worker01' self.personality = 'worker' self.ip4 = "192.168.100.101", self.ip6 = "::1", self.system_info = SystemInfo().format() self.bad_status = 'bad_status' self.bad_system_info = SystemInfo() self.bad_worker_status = { 'worker_status': { 'hostname': self.hostname, 'system_info': self.system_info, 'status': self.bad_status, 'personality': 'worker', 'ip_address_v4': '192.168.100.101', 'ip_address_v6': '::1' } } self.worker = { 'worker_status': { 'hostname': self.hostname, 'system_info': self.system_info, 'status': self.status, 'personality': 'worker', 'ip_address_v4': '192.168.100.101', 'ip_address_v6': '::1' } } self.returned_worker = Worker(**{"hostname": "worker01", "ip_address_v4": "192.168.100.101", "ip_address_v6": "::1", "personality": "worker", "status": "online", "system_info": self.system_info}) self.req = MagicMock() self.resp = MagicMock() self.resource = WorkerStatusResource() self.test_route = '/worker/{hostname}/status' self.api.add_route(self.test_route, self.resource)
def setUp(self): self.system_info = SystemInfo().format() self.worker = Worker( **{ "hostname": "worker-01", "ip_address_v4": "192.168.100.101", "ip_address_v6": "::1", "personality": "worker", "status": "online", "system_info": self.system_info })
def setUp(self): self.system_info = SystemInfo() self.test_worker = Worker(_id='010101', worker_token='9876543210', hostname='worker01', callback='172.22.15.25:8080/v1/config/', ip_address_v4='172.23.1.100', ip_address_v6='::1', personality='worker', status='new', system_info=self.system_info.format()) self.test_worker_lite = Worker(hostname='worker01', callback='172.22.15.25:8080/v1/config/', ip_address_v4='172.23.1.100', ip_address_v6='::1', personality='worker', status='new', system_info=self.system_info.format()) self.worker_status = self.test_worker.get_status()
def setUp(self): self.system_info = SystemInfo( cpu_cores='4', os_type='Darwin-11.4.2-x86-64bit', memory_mb='1024', timestamp='2013-07-15 19:26:53.076426', architecture='x86_64', load_average={ "1": 0.24755859375, "5": 1.0751953125, "15": 0.9365234375 }, disk_usage={"/dev/sda1": { "total": 313764528, "used": 112512436 }})
def setUp(self): self.conf = MagicMock() self.conf.status_update.worker_status_interval = 60 self.get_config = MagicMock(return_value=self.conf) self.config = WorkerConfiguration( personality='worker', hostname='worker01', coordinator_uri='http://192.168.1.2/v1') self.system_info = SystemInfo().format() self.request_uri = "{0}/worker/{1}/status".format( self.config.coordinator_uri, self.config.hostname) self.worker_status = { 'worker_status': Worker(personality='worker').format() } self.worker_status['worker_status']['system_info'] = self.system_info self.req_body = jsonutils.dumps(self.worker_status) self.get_config = MagicMock(return_value=self.config) self.resp = requests.Response() self.http_request = MagicMock(return_value=self.resp)
def publish_worker_stats(): """ Publishes worker stats to the Coordinator(s) at set times """ try: cache = ConfigCache() config = cache.get_config() request_uri = "{0}/worker/{1}/status".format(config.coordinator_uri, config.worker_id) req_body = { 'worker_status': { 'status': 'online', 'system_info': SystemInfo().format() } } http_request(url=request_uri, json_payload=jsonutils.dumps(req_body), http_verb='PUT') except Exception as ex: _LOG.info(ex.message)
def on_put(self, req, resp, worker_id, validated_body): """ updates a worker's status """ #load validated json payload in body body = validated_body['worker_status'] #find the worker in db worker_dict = self.db.find_one('worker', {'worker_id': worker_id}) if not worker_dict: _worker_not_found() worker = Worker(**worker_dict) if 'status' in body: worker.status = body['status'] if 'system_info' in body: worker.system_info = SystemInfo(**body['system_info']) self.db.update('worker', worker.format_for_save()) resp.status = falcon.HTTP_200
class WhenTestingCoordinatorFlow(unittest.TestCase): def setUp(self): self.db_handler = MagicMock() self.worker_id = "0123456789" self.req = MagicMock() self.resp = MagicMock() self.body_bad_header = {'worker': WorkerRegistration('correlation').format()} self.body_bad_personality = {'worker_registration': WorkerRegistration( 'bad_personality').format()} self.new_status = 'offline' self.new_bad_status = 'bad_status' self.system_info = SystemInfo() self.worker_dict = {"worker_id": self.worker_id, "hostname": "worker-01", "callback": "192.168.100.101:8080/v1/callback/", "ip_address_v4": "192.168.100.101", "ip_address_v6": "::1", "personality": "worker.storage", "status": "draining", "system_info": { "os_type": "Darwin-11.4.2-x86_64-i386-64bit", "memory_mb": "1024", "architecture": "", "cpu_cores": "4", "load_average": "0.234353456", "disk_usage": { "/dev/sda1": { "total": 313764528, "used": 112512436 } } } } self.worker = Worker(_id='010101', worker_id=self.worker_id, worker_token='9876543210', hostname='worker01', callback='172.22.15.25:8080/v1/config/', ip_address_v4='172.23.1.100', ip_address_v6='::1', personality='correlation', status='online', system_info=self.system_info.format()) self.worker_list = [ {"hostname": "worker-01", "callback": "192.168.100.101:8080/v1/callback/", "ip_address_v4": "192.168.100.101", "ip_address_v6": "::1", "personality": "storage", "status": "draining", "system_info": { "os_type": "Darwin-11.4.2-x86_64-i386-64bit", "memory_mb": "1024", "architecture": "", "cpu_cores": "4", "load_average": "0.234353456", "disk_usage": { "/dev/sda1": { "total": 313764528, "used": 112512436 }}}}, {"hostname": "worker-01", "callback": "192.168.100.101:8080/v1/callback/", "ip_address_v4": "192.168.100.101", "ip_address_v6": "::1", "personality": "storage", "status": "online", "system_info": { "os_type": "Darwin-11.4.2-x86_64-i386-64bit", "memory_mb": "1024", "architecture": "", "cpu_cores": "4", "load_average": "0.234353456", "disk_usage": { "/dev/sda1": { "total": 313764528, "used": 112512436 }}}}, {'hostname': "worker-01", "callback": "192.168.100.101:8080/v1/callback/", "ip_address_v4": "192.168.100.101", "ip_address_v6": "::1", "personality": "normalization", "status": "draining", "system_info": { "os_type": "Darwin-11.4.2-x86_64-i386-64bit", "memory_mb": "1024", "architecture": "", "cpu_cores": "4", "load_average": "0.234353456", "disk_usage": { "/dev/sda1": { "total": 313764528, "used": 112512436 }}}}] def test_add_worker(self): db_handler = MagicMock() db_handler.put.return_value = self.worker.format() coordinator_flow.add_worker(db_handler, self.worker_id) def test_find_worker(self): db_handler = MagicMock() db_handler.find_one.return_value = self.worker.format_for_save() self.db_handler.find_one.return_value = self.worker.format() self.assertIsInstance(coordinator_flow.find_worker(self.db_handler, self.worker_id), Worker) def test_find_worker_empty(self): db_handler = MagicMock() db_handler.find_one.return_value = None with self.assertRaises(falcon.HTTPError): coordinator_flow.find_worker(db_handler, self.worker_id)
class WhenTestingWatchlistFlow(unittest.TestCase): def setUp(self): self.broadcaster_list = [ Worker(**WorkerRegistration(personality='broadcaster', status='online').format()).format(), Worker(**WorkerRegistration(personality='broadcaster', status='online').format()).format(), Worker(**WorkerRegistration(personality='broadcaster', status='online').format()).format(), ] self.broadcaster_uri_list = [worker['ip_address_v4'] for worker in self.broadcaster_list] self.target_list = [ Worker(**WorkerRegistration(personality='coordinator', status='online').format()).format(), Worker(**WorkerRegistration(personality='normalizer', status='online').format()).format(), Worker(**WorkerRegistration(personality='coordinator', status='online').format()).format(), ] self.target_uri_list = {"broadcast": {"type": "ROUTES", "targets": [worker['ip_address_v4'] for worker in self.target_list]}} self.damaged_worker = Worker(**WorkerRegistration( personality='broadcaster',).format()) self.db_handler = MagicMock() self.worker_id = "0123456789" self.req = MagicMock() self.resp = MagicMock() self.body_bad_header = {'worker': WorkerRegistration('correlation').format()} self.body_bad_personality = {'worker_registration': WorkerRegistration( 'bad_personality').format()} self.new_status = 'offline' self.new_bad_status = 'bad_status' self.system_info = SystemInfo() self.watchlist_dict = {"_id": "010101", "watch_count": 5, "worker_id": "0123456789", "last_changed": "2013-04-09T15:11:19.818Z" } self.watchlist_dict_over = {"_id": "010101", "watch_count": 7, "worker_id": "123456789", "last_changed": "2013-04-09T15:11:19.818Z" } self.worker = Worker(_id='010101', worker_id=self.worker_id, worker_token='9876543210', hostname='worker01', callback='172.22.15.25:8080/v1/config/', ip_address_v4='172.23.1.100', ip_address_v6='::1', personality='correlation', status='offline', system_info=self.system_info.format()) self.worker_online = Worker(_id='010101', worker_id=self.worker_id, worker_token='9876543210', hostname='worker01', callback='172.22.15.25:8080/v1/config/', ip_address_v4='172.23.1.100', ip_address_v6='::1', personality='correlation', status='online', system_info=self.system_info.format()) def test_add_watchlist_item(self): watchlist_flow._add_watchlist_item(self.db_handler, self.worker_id) def test_update_watchlist_item(self): watchlist_flow._update_watchlist_item(self.db_handler, self.watchlist_dict) def test_delete_expired_watchlist_item(self): watchlist_flow._delete_expired_watchlist_items(self.db_handler) def test_get_broadcaster_list_true(self): db_handler = MagicMock() db_handler.find.return_value = self.broadcaster_list self.assertEqual(watchlist_flow._get_broadcaster_list(db_handler), self.broadcaster_uri_list) def test_get_broadcaster_list_empty(self): db_handler = MagicMock() db_handler.find = MagicMock(return_value=[]) self.assertFalse( watchlist_flow._get_broadcaster_list(db_handler)) def test_get_broadcast_targets_list_empty(self): db_handler = MagicMock() db_handler.find - MagicMock(return_value=[]) self.assertFalse( watchlist_flow._get_broadcast_targets(db_handler, self.damaged_worker)) def test_get_broadcast_targets_list_true(self): db_handler = MagicMock() db_handler.find.return_value = self.target_list self.assertEqual( watchlist_flow._get_broadcast_targets(db_handler, self.damaged_worker), self.target_uri_list) def test_send_target_list_to_broadcaster_false_empty_target_list(self): db_handler = MagicMock() watchlist_flow._get_broadcast_targets = MagicMock( return_value=[]) watchlist_flow._get_broadcaster_list = MagicMock( return_value=self.broadcaster_uri_list) result = watchlist_flow._send_target_list_to_broadcaster( db_handler, self.damaged_worker) self.assertFalse(result) def test_send_target_list_to_broadcaster_no_connection_made(self): db_handler = MagicMock() resp = MagicMock() resp.status_code = httplib.BAD_REQUEST http_request = MagicMock(return_value=resp) watchlist_flow._get_broadcast_targets = MagicMock( return_value=self.target_uri_list) watchlist_flow._get_broadcaster_list = MagicMock( return_value=self.broadcaster_uri_list) with patch('meniscus.api.coordinator.' 'watchlist_flow.http_request', http_request): result = watchlist_flow._send_target_list_to_broadcaster( db_handler, self.damaged_worker) self.assertFalse(result) def test_send_target_list_to_broadcaster_exception_raised(self): db_handler = MagicMock() http_request = MagicMock( side_effect=requests.RequestException) watchlist_flow._get_broadcast_targets = MagicMock( return_value=self.target_uri_list) watchlist_flow._get_broadcaster_list = MagicMock( return_value=self.broadcaster_uri_list) with patch('meniscus.api.coordinator.' 'watchlist_flow.http_request', http_request): with self.assertRaises( coordinator_errors.BroadcasterCommunicationError): watchlist_flow._send_target_list_to_broadcaster( db_handler, self.damaged_worker) def test_send_target_list_to_broadcaster_connection_made(self): db_handler = MagicMock() resp = MagicMock() resp.status_code = httplib.OK http_request = MagicMock(return_value=resp) watchlist_flow._get_broadcast_targets = MagicMock( return_value=self.target_uri_list) watchlist_flow._get_broadcaster_list = MagicMock( return_value=self.broadcaster_uri_list) with patch('meniscus.api.coordinator.' 'watchlist_flow.http_request', http_request): result = watchlist_flow._send_target_list_to_broadcaster( db_handler, self.damaged_worker) self.assertTrue(result) def test_broadcast_config_change(self): watchlist_flow._send_target_list_to_broadcaster(self.db_handler, self.worker) def test_process_watchlist_item(self): watchlist_flow.process_watchlist_item(self.db_handler, self.worker_id) def test_process_watchlist_item_new_item(self): self.db_handler.find_one.return_value = None self.assertIs(watchlist_flow.process_watchlist_item( self.db_handler, self.worker_id), None) def test_process_watchlist_item_count_at_threshold(self): find_worker = MagicMock(return_value=self.worker) with patch('meniscus.api.coordinator.coordinator_flow.find_worker', find_worker): self.db_handler.find_one.return_value = self.watchlist_dict watchlist_flow.process_watchlist_item(self.db_handler, self.worker_id) find_worker.assert_called_once_with(self.db_handler, self.worker_id) def test_process_watchlist_item_count_at_threshold(self): find_worker = MagicMock(return_value=self.worker_online) with patch('meniscus.api.coordinator.coordinator_flow.find_worker', find_worker): self.db_handler.find_one.return_value = self.watchlist_dict watchlist_flow.process_watchlist_item(self.db_handler, self.worker_id) find_worker.assert_called_once_with(self.db_handler, self.worker_id)
def setUp(self): self.broadcaster_list = [ Worker(**WorkerRegistration(personality='broadcaster', status='online').format()).format(), Worker(**WorkerRegistration(personality='broadcaster', status='online').format()).format(), Worker(**WorkerRegistration(personality='broadcaster', status='online').format()).format(), ] self.broadcaster_uri_list = [worker['ip_address_v4'] for worker in self.broadcaster_list] self.target_list = [ Worker(**WorkerRegistration(personality='coordinator', status='online').format()).format(), Worker(**WorkerRegistration(personality='normalizer', status='online').format()).format(), Worker(**WorkerRegistration(personality='coordinator', status='online').format()).format(), ] self.target_uri_list = {"broadcast": {"type": "ROUTES", "targets": [worker['ip_address_v4'] for worker in self.target_list]}} self.damaged_worker = Worker(**WorkerRegistration( personality='broadcaster',).format()) self.db_handler = MagicMock() self.worker_id = "0123456789" self.req = MagicMock() self.resp = MagicMock() self.body_bad_header = {'worker': WorkerRegistration('correlation').format()} self.body_bad_personality = {'worker_registration': WorkerRegistration( 'bad_personality').format()} self.new_status = 'offline' self.new_bad_status = 'bad_status' self.system_info = SystemInfo() self.watchlist_dict = {"_id": "010101", "watch_count": 5, "worker_id": "0123456789", "last_changed": "2013-04-09T15:11:19.818Z" } self.watchlist_dict_over = {"_id": "010101", "watch_count": 7, "worker_id": "123456789", "last_changed": "2013-04-09T15:11:19.818Z" } self.worker = Worker(_id='010101', worker_id=self.worker_id, worker_token='9876543210', hostname='worker01', callback='172.22.15.25:8080/v1/config/', ip_address_v4='172.23.1.100', ip_address_v6='::1', personality='correlation', status='offline', system_info=self.system_info.format()) self.worker_online = Worker(_id='010101', worker_id=self.worker_id, worker_token='9876543210', hostname='worker01', callback='172.22.15.25:8080/v1/config/', ip_address_v4='172.23.1.100', ip_address_v6='::1', personality='correlation', status='online', system_info=self.system_info.format())
def setUp(self): self.db_handler = MagicMock() self.worker_id = "0123456789" self.req = MagicMock() self.resp = MagicMock() self.body_bad_header = {'worker': WorkerRegistration('correlation').format()} self.body_bad_personality = {'worker_registration': WorkerRegistration( 'bad_personality').format()} self.new_status = 'offline' self.new_bad_status = 'bad_status' self.system_info = SystemInfo() self.worker_dict = {"worker_id": self.worker_id, "hostname": "worker-01", "callback": "192.168.100.101:8080/v1/callback/", "ip_address_v4": "192.168.100.101", "ip_address_v6": "::1", "personality": "worker.storage", "status": "draining", "system_info": { "os_type": "Darwin-11.4.2-x86_64-i386-64bit", "memory_mb": "1024", "architecture": "", "cpu_cores": "4", "load_average": "0.234353456", "disk_usage": { "/dev/sda1": { "total": 313764528, "used": 112512436 } } } } self.worker = Worker(_id='010101', worker_id=self.worker_id, worker_token='9876543210', hostname='worker01', callback='172.22.15.25:8080/v1/config/', ip_address_v4='172.23.1.100', ip_address_v6='::1', personality='correlation', status='online', system_info=self.system_info.format()) self.worker_list = [ {"hostname": "worker-01", "callback": "192.168.100.101:8080/v1/callback/", "ip_address_v4": "192.168.100.101", "ip_address_v6": "::1", "personality": "storage", "status": "draining", "system_info": { "os_type": "Darwin-11.4.2-x86_64-i386-64bit", "memory_mb": "1024", "architecture": "", "cpu_cores": "4", "load_average": "0.234353456", "disk_usage": { "/dev/sda1": { "total": 313764528, "used": 112512436 }}}}, {"hostname": "worker-01", "callback": "192.168.100.101:8080/v1/callback/", "ip_address_v4": "192.168.100.101", "ip_address_v6": "::1", "personality": "storage", "status": "online", "system_info": { "os_type": "Darwin-11.4.2-x86_64-i386-64bit", "memory_mb": "1024", "architecture": "", "cpu_cores": "4", "load_average": "0.234353456", "disk_usage": { "/dev/sda1": { "total": 313764528, "used": 112512436 }}}}, {'hostname': "worker-01", "callback": "192.168.100.101:8080/v1/callback/", "ip_address_v4": "192.168.100.101", "ip_address_v6": "::1", "personality": "normalization", "status": "draining", "system_info": { "os_type": "Darwin-11.4.2-x86_64-i386-64bit", "memory_mb": "1024", "architecture": "", "cpu_cores": "4", "load_average": "0.234353456", "disk_usage": { "/dev/sda1": { "total": 313764528, "used": 112512436 }}}}]