def _broadcast_route_messages(self, message_interval): cache = BroadcastCache() while True: sleep(message_interval) uri_targets = cache.get_targets(ROUTES) # Get route messages from cache and notify the workers if uri_targets: type_header = { "TYPE": ROUTES } for uri in uri_targets: try: http_request(uri, add_headers=type_header, http_verb='HEAD') except requests.RequestException: # Swallow exception for non-responsive worker pass cache.delete_message(ROUTES) if self.run_once: break
def test_should_raise_value_error(self): HTTPretty.register_uri(HTTPretty.PATCH, self.url, body=self.json_payload, content_type="application/json") with self.assertRaises(ValueError): http_request(self.url, json_payload=self.json_payload, http_verb='PATCH')
def publish_worker_stats(): """ Publishes worker stats to the Coordinator(s) at set times """ try: cache = ConfigCache() config = cache.get_config() request_uri = "{0}/worker/{1}/status".format(config.coordinator_uri, config.worker_id) req_body = {"worker_status": {"status": "online", "system_info": SystemInfo().format()}} http_request(url=request_uri, json_payload=jsonutils.dumps(req_body), http_verb="PUT") except Exception as ex: _LOG.info(ex.message)
def _send_target_list_to_broadcaster(db, worker): """ send broadcast list to broadcaster """ # list of broadcasters broadcasters = _get_broadcaster_list(db) if not broadcasters: ## todo log no online broadcasters return False # list of broadcast targets broadcast_targets = _get_broadcast_targets(db, worker) if not broadcast_targets: ## todo: log failed to find target return False for broadcaster_uri in broadcasters: resp = None try: ## todo refactor callback address to not include /v1/callback/" resp = http_request( '{0}:8080/v1/broadcast'.format(broadcaster_uri), jsonutils.dumps(broadcast_targets), http_verb='PUT') except requests.RequestException: raise coordinator_errors.BroadcasterCommunicationError if resp.status_code == httplib.OK: return True else: ## todo Log broadcaster connection failure return False
def _get_tenant_from_coordinator(self): """ This method calls to the coordinator to retrieve tenant """ config_cache = ConfigCache() config = config_cache.get_config() token_header = { MESSAGE_TOKEN: self.message_token, "WORKER-ID": config.worker_id, "WORKER-TOKEN": config.worker_token } request_uri = "{0}/tenant/{1}".format( config.coordinator_uri, self.tenant_id) try: resp = http_request(request_uri, token_header, http_verb='GET') except requests.RequestException: raise errors.CoordinatorCommunicationError if resp.status_code == httplib.OK: response_body = resp.json() tenant = load_tenant_from_dict(response_body['tenant']) return tenant elif resp.status_code == httplib.NOT_FOUND: raise errors.ResourceNotFoundError('Unable to locate tenant.') else: raise errors.CoordinatorCommunicationError
def _register_with_coordinator(self, coordinator_uri, personality, registration, auth_header): """ register with the coordinator and persist the configuration to cache """ try: resp = http_request(coordinator_uri + '/pairing', auth_header, jsonutils.dumps(registration), http_verb='POST') except requests.RequestException: _LOG.exception( 'Pairing Process: error posting worker registration') return False _LOG.debug('resp.status_code: {0}'.format(resp.status_code)) if resp.status_code == httplib.ACCEPTED: body = resp.json()['worker_identity'] config = WorkerConfiguration(personality, body['personality_module'], body['worker_token'], body['worker_id'], coordinator_uri) config_cache = ConfigCache() config_cache.set_config(config) return True
def _validate_token_with_coordinator(self): """ This method calls to the coordinator to validate the message token """ config_cache = ConfigCache() config = config_cache.get_config() token_header = { MESSAGE_TOKEN: self.message_token, "WORKER-ID": config.worker_id, "WORKER-TOKEN": config.worker_token } request_uri = "{0}/tenant/{1}/token".format(config.coordinator_uri, self.tenant_id) try: resp = http_request(request_uri, token_header, http_verb='HEAD') except requests.RequestException as ex: _LOG.exception(ex.message) raise errors.CoordinatorCommunicationError if resp.status_code != httplib.OK: raise errors.MessageAuthenticationError( 'Message not authenticated, check your tenant id ' 'and or message token for validity') return True
def get_routes_from_coordinator(): """ get the associated routes for the worker and store them in cache """ config_cache = ConfigCache() config = config_cache.get_config() token_header = {"WORKER-ID": config.worker_id, "WORKER-TOKEN": config.worker_token} request_uri = "{0}/worker/{1}/routes".format( config.coordinator_uri, config.worker_id) try: resp = http_request(request_uri, token_header, http_verb='GET') except requests.RequestException: return False #if the coordinator issues a response, cache the worker routes #and return true if resp.status_code == httplib.OK: routes = resp.json()['routes'] config_cache.set_routes(routes) return True
def _validate_token_with_coordinator(self): """ This method calls to the coordinator to validate the message token """ config_cache = ConfigCache() config = config_cache.get_config() token_header = { MESSAGE_TOKEN: self.message_token, "WORKER-ID": config.worker_id, "WORKER-TOKEN": config.worker_token } request_uri = "{0}/tenant/{1}/token".format( config.coordinator_uri, self.tenant_id) try: resp = http_request(request_uri, token_header, http_verb='HEAD') except requests.RequestException as ex: _LOG.exception(ex.message) raise errors.CoordinatorCommunicationError if resp.status_code != httplib.OK: raise errors.MessageAuthenticationError( 'Message not authenticated, check your tenant id ' 'and or message token for validity') return True
def _validate_token_with_coordinator(tenant_id, message_token, message): """ Call coordinator to validate the message token. If token is validated, persist the token in the local cache for future lookups, and hand off message to retrieve tenant information. """ config = _get_config_from_cache() try: resp = http_request('{0}/tenant/{1}/token'.format( config.coordinator_uri, tenant_id), { MESSAGE_TOKEN: message_token, 'hostname': config.hostname }, http_verb='HEAD') except requests.RequestException as ex: _LOG.exception(ex.message) raise errors.CoordinatorCommunicationError if resp.status_code != httplib.OK: raise errors.MessageAuthenticationError( 'Message not authenticated, check your tenant id ' 'and or message token for validity') # hand off the message to validate the tenant with the coordinator _get_tenant_from_coordinator(tenant_id, message_token, message)
def _validate_token_with_coordinator(tenant_id, message_token, message): """ Call coordinator to validate the message token. If token is validated, persist the token in the local cache for future lookups, and hand off message to retrieve tenant information. """ config = _get_config_from_cache() try: resp = http_request( '{0}/tenant/{1}/token'.format(config.coordinator_uri, tenant_id), {MESSAGE_TOKEN: message_token, 'hostname': config.hostname}, http_verb='HEAD') except requests.RequestException as ex: _LOG.exception(ex.message) raise errors.CoordinatorCommunicationError if resp.status_code != httplib.OK: raise errors.MessageAuthenticationError( 'Message not authenticated, check your tenant id ' 'and or message token for validity') # hand off the message to validate the tenant with the coordinator _get_tenant_from_coordinator(tenant_id, message_token, message)
def _register_with_coordinator( self, coordinator_uri, personality, registration, auth_header): """ register with the coordinator and persist the configuration to cache """ try: resp = http_request(coordinator_uri + '/pairing', auth_header, jsonutils.dumps( registration), http_verb='POST') except requests.RequestException: _LOG.exception( 'Pairing Process: error posting worker registration') return False _LOG.debug('resp.status_code: {0}'.format(resp.status_code)) if resp.status_code == httplib.ACCEPTED: body = resp.json()['worker_identity'] config = WorkerConfiguration( personality, body['personality_module'], body['worker_token'], body['worker_id'], coordinator_uri) config_cache = ConfigCache() config_cache.set_config(config) return True
def _get_tenant_from_coordinator(self): """ This method calls to the coordinator to retrieve tenant """ config_cache = ConfigCache() config = config_cache.get_config() token_header = { MESSAGE_TOKEN: self.message_token, "WORKER-ID": config.worker_id, "WORKER-TOKEN": config.worker_token } request_uri = "{0}/tenant/{1}".format(config.coordinator_uri, self.tenant_id) try: resp = http_request(request_uri, token_header, http_verb='GET') except requests.RequestException as ex: _LOG.exception(ex.message) raise errors.CoordinatorCommunicationError if resp.status_code == httplib.OK: response_body = resp.json() tenant = load_tenant_from_dict(response_body['tenant']) return tenant elif resp.status_code == httplib.NOT_FOUND: message = 'unable to locate tenant.' _LOG.debug(message) raise errors.ResourceNotFoundError(message) else: raise errors.CoordinatorCommunicationError
def publish_worker_stats(): """ Publishes worker stats to the Coordinator(s) at set times """ try: cache = ConfigCache() config = cache.get_config() request_uri = "{0}/worker/{1}/status".format( config.coordinator_uri, config.hostname) req_body = { 'worker_status': Worker(personality=config.personality).format() } http_request(url=request_uri, json_payload=jsonutils.dumps(req_body), http_verb='PUT') except Exception as ex: _LOG.info(ex.message)
def _blacklist_worker(self, service_domain, worker_id): self._active_worker_socket[service_domain] = None self._blacklist_cache.add_blacklist_worker(worker_id) config = self._config_cache.get_config() if config: token_header = { "WORKER-ID": config.worker_id, "WORKER-TOKEN": config.worker_token } request_uri = "{0}/worker/{1}".format( config.coordinator_uri, worker_id) try: http_request(request_uri, token_header, http_verb='PUT') except requests.RequestException: #Todo log failure to contact coordinator pass
def _send_stats(self, load_ave_interval, disk_usage_interval): """ send system usage data to the coordinator on specified intervals """ time_lapsed = 0 while True: sleep(load_ave_interval) time_lapsed += load_ave_interval cache = ConfigCache() config = cache.get_config() if config: token_header = { "WORKER-ID": config.worker_id, "WORKER-TOKEN": config.worker_token } request_uri = "{0}/worker/{1}/status".format( config.coordinator_uri, config.worker_id) req_body = {'load_average': sys_assist.get_load_average()} if time_lapsed == disk_usage_interval: time_lapsed = 0 req_body.update( {'disk_usage': sys_assist.get_disk_usage()}) try: http_request(request_uri, token_header, jsonutils.dumps(req_body), http_verb='PUT') except requests.RequestException: pass if self.run_once: break
def publish_worker_stats(): """ Publishes worker stats to the Coordinator(s) at set times """ try: cache = ConfigCache() config = cache.get_config() request_uri = "{0}/worker/{1}/status".format(config.coordinator_uri, config.worker_id) req_body = { 'worker_status': { 'status': 'online', 'system_info': SystemInfo().format() } } http_request(url=request_uri, json_payload=jsonutils.dumps(req_body), http_verb='PUT') except Exception as ex: _LOG.info(ex.message)
def test_should_return_http_200_on_all_http_verbs(self): httpretty_verbs = { 'POST': HTTPretty.POST, 'GET': HTTPretty.GET, 'DELETE': HTTPretty.DELETE, 'PUT': HTTPretty.PUT, 'HEAD': HTTPretty.HEAD, } for http_verb in HTTP_VERBS: HTTPretty.register_uri(httpretty_verbs[http_verb], self.url, body=self.json_payload, content_type="application/json", status=200) self.assertTrue(http_request(self.url, json_payload=self.json_payload, http_verb=http_verb), falcon.HTTP_200)
def test_should_return_http_200_on_all_http_verbs(self): httpretty_verbs = { 'POST': HTTPretty.POST, 'GET': HTTPretty.GET, 'DELETE': HTTPretty.DELETE, 'PUT': HTTPretty.PUT, 'HEAD': HTTPretty.HEAD, } for http_verb in HTTP_VERBS: HTTPretty.register_uri(httpretty_verbs[http_verb], self.url, body=self.json_payload, content_type="application/json", status=200) self.assertTrue( http_request(self.url, json_payload=self.json_payload, http_verb=http_verb), falcon.HTTP_200)
def _get_tenant_from_coordinator(tenant_id, message_token, message): """ This method retrieves tenant data from the coordinator, and persists the tenant data in the local cache for future lookups. The message is then handed off to be packed with correlation data. """ config = _get_config_from_cache() try: resp = http_request('{0}/tenant/{1}'.format(config.coordinator_uri, tenant_id), { MESSAGE_TOKEN: message_token, 'hostname': config.hostname }, http_verb='GET') except requests.RequestException as ex: _LOG.exception(ex.message) raise errors.CoordinatorCommunicationError if resp.status_code == httplib.OK: response_body = resp.json() #load new tenant data from response body tenant = tenant_util.load_tenant_from_dict(response_body['tenant']) # update the cache with new tenant info _save_tenant_to_cache(tenant_id, tenant) # add correlation to message _add_correlation_info_to_message(tenant, message) elif resp.status_code == httplib.NOT_FOUND: error_message = 'unable to locate tenant.' _LOG.debug(error_message) raise errors.ResourceNotFoundError(error_message) else: #coordinator responds, but coordinator datasink could be unreachable raise errors.CoordinatorCommunicationError
def _get_tenant_from_coordinator(tenant_id, message_token, message): """ This method retrieves tenant data from the coordinator, and persists the tenant data in the local cache for future lookups. The message is then handed off to be packed with correlation data. """ config = _get_config_from_cache() try: resp = http_request( '{0}/tenant/{1}'.format(config.coordinator_uri, tenant_id), {MESSAGE_TOKEN: message_token, 'hostname': config.hostname}, http_verb='GET') except requests.RequestException as ex: _LOG.exception(ex.message) raise errors.CoordinatorCommunicationError if resp.status_code == httplib.OK: response_body = resp.json() #load new tenant data from response body tenant = tenant_util.load_tenant_from_dict(response_body['tenant']) # update the cache with new tenant info _save_tenant_to_cache(tenant_id, tenant) # add correlation to message _add_correlation_info_to_message(tenant, message) elif resp.status_code == httplib.NOT_FOUND: error_message = 'unable to locate tenant.' _LOG.debug(error_message) raise errors.ResourceNotFoundError(error_message) else: #coordinator responds, but coordinator datasink could be unreachable raise errors.CoordinatorCommunicationError
def _register_worker_online(self, status): """ register the worker with the coordinator with an online status """ cache = ConfigCache() config = cache.get_config() token_header = {"WORKER-TOKEN": config.worker_token} request_uri = "{0}/worker/{1}/status".format( config.coordinator_uri, config.worker_id) status = {"worker_status": status} try: resp = http_request(request_uri, token_header, jsonutils.dumps(status), http_verb='PUT') except requests.RequestException: return False if resp.status_code == httplib.OK: return True
def test_should_cause_a_request_exception(self): with patch.object(requests, 'get') as mock_method: with self.assertRaises(requests.RequestException): mock_method.side_effect = requests.RequestException http_request(self.url, json_payload=self.json_payload)