def _broadcast_route_messages(self, message_interval):
cache = BroadcastCache()
while True:
sleep(message_interval)
uri_targets = cache.get_targets(ROUTES)
# Get route messages from cache and notify the workers
if uri_targets:
type_header = {
"TYPE": ROUTES
}
for uri in uri_targets:
try:
http_request(uri,
add_headers=type_header,
http_verb='HEAD')
except requests.RequestException:
# Swallow exception for non-responsive worker
pass
cache.delete_message(ROUTES)
if self.run_once:
break
def test_should_raise_value_error(self):
HTTPretty.register_uri(HTTPretty.PATCH, self.url,
body=self.json_payload,
content_type="application/json")
with self.assertRaises(ValueError):
http_request(self.url, json_payload=self.json_payload,
http_verb='PATCH')
def test_should_raise_value_error(self):
HTTPretty.register_uri(HTTPretty.PATCH,
self.url,
body=self.json_payload,
content_type="application/json")
with self.assertRaises(ValueError):
http_request(self.url,
json_payload=self.json_payload,
http_verb='PATCH')
def publish_worker_stats():
"""
Publishes worker stats to the Coordinator(s) at set times
"""
try:
cache = ConfigCache()
config = cache.get_config()
request_uri = "{0}/worker/{1}/status".format(config.coordinator_uri, config.worker_id)
req_body = {"worker_status": {"status": "online", "system_info": SystemInfo().format()}}
http_request(url=request_uri, json_payload=jsonutils.dumps(req_body), http_verb="PUT")
except Exception as ex:
_LOG.info(ex.message)
def _send_target_list_to_broadcaster(db, worker):
"""
send broadcast list to broadcaster
"""
# list of broadcasters
broadcasters = _get_broadcaster_list(db)
if not broadcasters:
## todo log no online broadcasters
return False
# list of broadcast targets
broadcast_targets = _get_broadcast_targets(db, worker)
if not broadcast_targets:
## todo: log failed to find target
return False
for broadcaster_uri in broadcasters:
resp = None
try:
## todo refactor callback address to not include /v1/callback/"
resp = http_request(
'{0}:8080/v1/broadcast'.format(broadcaster_uri),
jsonutils.dumps(broadcast_targets), http_verb='PUT')
except requests.RequestException:
raise coordinator_errors.BroadcasterCommunicationError
if resp.status_code == httplib.OK:
return True
else:
## todo Log broadcaster connection failure
return False
def _get_tenant_from_coordinator(self):
"""
This method calls to the coordinator to retrieve tenant
"""
config_cache = ConfigCache()
config = config_cache.get_config()
token_header = {
MESSAGE_TOKEN: self.message_token,
"WORKER-ID": config.worker_id,
"WORKER-TOKEN": config.worker_token
}
request_uri = "{0}/tenant/{1}".format(
config.coordinator_uri, self.tenant_id)
try:
resp = http_request(request_uri, token_header,
http_verb='GET')
except requests.RequestException:
raise errors.CoordinatorCommunicationError
if resp.status_code == httplib.OK:
response_body = resp.json()
tenant = load_tenant_from_dict(response_body['tenant'])
return tenant
elif resp.status_code == httplib.NOT_FOUND:
raise errors.ResourceNotFoundError('Unable to locate tenant.')
else:
raise errors.CoordinatorCommunicationError
def _register_with_coordinator(self, coordinator_uri, personality,
registration, auth_header):
"""
register with the coordinator and persist the configuration to cache
"""
try:
resp = http_request(coordinator_uri + '/pairing',
auth_header,
jsonutils.dumps(registration),
http_verb='POST')
except requests.RequestException:
_LOG.exception(
'Pairing Process: error posting worker registration')
return False
_LOG.debug('resp.status_code: {0}'.format(resp.status_code))
if resp.status_code == httplib.ACCEPTED:
body = resp.json()['worker_identity']
config = WorkerConfiguration(personality,
body['personality_module'],
body['worker_token'],
body['worker_id'], coordinator_uri)
config_cache = ConfigCache()
config_cache.set_config(config)
return True
def _validate_token_with_coordinator(self):
"""
This method calls to the coordinator to validate the message token
"""
config_cache = ConfigCache()
config = config_cache.get_config()
token_header = {
MESSAGE_TOKEN: self.message_token,
"WORKER-ID": config.worker_id,
"WORKER-TOKEN": config.worker_token
}
request_uri = "{0}/tenant/{1}/token".format(config.coordinator_uri,
self.tenant_id)
try:
resp = http_request(request_uri, token_header, http_verb='HEAD')
except requests.RequestException as ex:
_LOG.exception(ex.message)
raise errors.CoordinatorCommunicationError
if resp.status_code != httplib.OK:
raise errors.MessageAuthenticationError(
'Message not authenticated, check your tenant id '
'and or message token for validity')
return True
def get_routes_from_coordinator():
"""
get the associated routes for the worker and store them in cache
"""
config_cache = ConfigCache()
config = config_cache.get_config()
token_header = {"WORKER-ID": config.worker_id,
"WORKER-TOKEN": config.worker_token}
request_uri = "{0}/worker/{1}/routes".format(
config.coordinator_uri, config.worker_id)
try:
resp = http_request(request_uri, token_header, http_verb='GET')
except requests.RequestException:
return False
#if the coordinator issues a response, cache the worker routes
#and return true
if resp.status_code == httplib.OK:
routes = resp.json()['routes']
config_cache.set_routes(routes)
return True
def _validate_token_with_coordinator(self):
"""
This method calls to the coordinator to validate the message token
"""
config_cache = ConfigCache()
config = config_cache.get_config()
token_header = {
MESSAGE_TOKEN: self.message_token,
"WORKER-ID": config.worker_id,
"WORKER-TOKEN": config.worker_token
}
request_uri = "{0}/tenant/{1}/token".format(
config.coordinator_uri, self.tenant_id)
try:
resp = http_request(request_uri, token_header,
http_verb='HEAD')
except requests.RequestException as ex:
_LOG.exception(ex.message)
raise errors.CoordinatorCommunicationError
if resp.status_code != httplib.OK:
raise errors.MessageAuthenticationError(
'Message not authenticated, check your tenant id '
'and or message token for validity')
return True
def _validate_token_with_coordinator(tenant_id, message_token, message):
"""
Call coordinator to validate the message token. If token is validated,
persist the token in the local cache for future lookups, and hand off
message to retrieve tenant information.
"""
config = _get_config_from_cache()
try:
resp = http_request('{0}/tenant/{1}/token'.format(
config.coordinator_uri, tenant_id), {
MESSAGE_TOKEN: message_token,
'hostname': config.hostname
},
http_verb='HEAD')
except requests.RequestException as ex:
_LOG.exception(ex.message)
raise errors.CoordinatorCommunicationError
if resp.status_code != httplib.OK:
raise errors.MessageAuthenticationError(
'Message not authenticated, check your tenant id '
'and or message token for validity')
# hand off the message to validate the tenant with the coordinator
_get_tenant_from_coordinator(tenant_id, message_token, message)
def _validate_token_with_coordinator(tenant_id, message_token, message):
"""
Call coordinator to validate the message token. If token is validated,
persist the token in the local cache for future lookups, and hand off
message to retrieve tenant information.
"""
config = _get_config_from_cache()
try:
resp = http_request(
'{0}/tenant/{1}/token'.format(config.coordinator_uri, tenant_id),
{MESSAGE_TOKEN: message_token, 'hostname': config.hostname},
http_verb='HEAD')
except requests.RequestException as ex:
_LOG.exception(ex.message)
raise errors.CoordinatorCommunicationError
if resp.status_code != httplib.OK:
raise errors.MessageAuthenticationError(
'Message not authenticated, check your tenant id '
'and or message token for validity')
# hand off the message to validate the tenant with the coordinator
_get_tenant_from_coordinator(tenant_id, message_token, message)
def _register_with_coordinator(
self, coordinator_uri, personality, registration, auth_header):
"""
register with the coordinator and persist the configuration to cache
"""
try:
resp = http_request(coordinator_uri + '/pairing', auth_header,
jsonutils.dumps(
registration), http_verb='POST')
except requests.RequestException:
_LOG.exception(
'Pairing Process: error posting worker registration')
return False
_LOG.debug('resp.status_code: {0}'.format(resp.status_code))
if resp.status_code == httplib.ACCEPTED:
body = resp.json()['worker_identity']
config = WorkerConfiguration(
personality, body['personality_module'], body['worker_token'],
body['worker_id'], coordinator_uri)
config_cache = ConfigCache()
config_cache.set_config(config)
return True
def _get_tenant_from_coordinator(self):
"""
This method calls to the coordinator to retrieve tenant
"""
config_cache = ConfigCache()
config = config_cache.get_config()
token_header = {
MESSAGE_TOKEN: self.message_token,
"WORKER-ID": config.worker_id,
"WORKER-TOKEN": config.worker_token
}
request_uri = "{0}/tenant/{1}".format(config.coordinator_uri,
self.tenant_id)
try:
resp = http_request(request_uri, token_header, http_verb='GET')
except requests.RequestException as ex:
_LOG.exception(ex.message)
raise errors.CoordinatorCommunicationError
if resp.status_code == httplib.OK:
response_body = resp.json()
tenant = load_tenant_from_dict(response_body['tenant'])
return tenant
elif resp.status_code == httplib.NOT_FOUND:
message = 'unable to locate tenant.'
_LOG.debug(message)
raise errors.ResourceNotFoundError(message)
else:
raise errors.CoordinatorCommunicationError
def publish_worker_stats():
"""
Publishes worker stats to the Coordinator(s) at set times
"""
try:
cache = ConfigCache()
config = cache.get_config()
request_uri = "{0}/worker/{1}/status".format(
config.coordinator_uri, config.hostname)
req_body = {
'worker_status': Worker(personality=config.personality).format()
}
http_request(url=request_uri, json_payload=jsonutils.dumps(req_body),
http_verb='PUT')
except Exception as ex:
_LOG.info(ex.message)
def _blacklist_worker(self, service_domain, worker_id):
self._active_worker_socket[service_domain] = None
self._blacklist_cache.add_blacklist_worker(worker_id)
config = self._config_cache.get_config()
if config:
token_header = {
"WORKER-ID": config.worker_id,
"WORKER-TOKEN": config.worker_token
}
request_uri = "{0}/worker/{1}".format(
config.coordinator_uri, worker_id)
try:
http_request(request_uri, token_header, http_verb='PUT')
except requests.RequestException:
#Todo log failure to contact coordinator
pass
def _send_stats(self, load_ave_interval, disk_usage_interval):
"""
send system usage data to the coordinator on specified intervals
"""
time_lapsed = 0
while True:
sleep(load_ave_interval)
time_lapsed += load_ave_interval
cache = ConfigCache()
config = cache.get_config()
if config:
token_header = {
"WORKER-ID": config.worker_id,
"WORKER-TOKEN": config.worker_token
}
request_uri = "{0}/worker/{1}/status".format(
config.coordinator_uri, config.worker_id)
req_body = {'load_average': sys_assist.get_load_average()}
if time_lapsed == disk_usage_interval:
time_lapsed = 0
req_body.update(
{'disk_usage': sys_assist.get_disk_usage()})
try:
http_request(request_uri, token_header,
jsonutils.dumps(req_body),
http_verb='PUT')
except requests.RequestException:
pass
if self.run_once:
break
def publish_worker_stats():
"""
Publishes worker stats to the Coordinator(s) at set times
"""
try:
cache = ConfigCache()
config = cache.get_config()
request_uri = "{0}/worker/{1}/status".format(config.coordinator_uri,
config.worker_id)
req_body = {
'worker_status': {
'status': 'online',
'system_info': SystemInfo().format()
}
}
http_request(url=request_uri,
json_payload=jsonutils.dumps(req_body),
http_verb='PUT')
except Exception as ex:
_LOG.info(ex.message)
def test_should_return_http_200_on_all_http_verbs(self):
httpretty_verbs = {
'POST': HTTPretty.POST,
'GET': HTTPretty.GET,
'DELETE': HTTPretty.DELETE,
'PUT': HTTPretty.PUT,
'HEAD': HTTPretty.HEAD,
}
for http_verb in HTTP_VERBS:
HTTPretty.register_uri(httpretty_verbs[http_verb],
self.url,
body=self.json_payload,
content_type="application/json",
status=200)
self.assertTrue(http_request(self.url,
json_payload=self.json_payload,
http_verb=http_verb),
falcon.HTTP_200)
def test_should_return_http_200_on_all_http_verbs(self):
httpretty_verbs = {
'POST': HTTPretty.POST,
'GET': HTTPretty.GET,
'DELETE': HTTPretty.DELETE,
'PUT': HTTPretty.PUT,
'HEAD': HTTPretty.HEAD,
}
for http_verb in HTTP_VERBS:
HTTPretty.register_uri(httpretty_verbs[http_verb],
self.url,
body=self.json_payload,
content_type="application/json",
status=200)
self.assertTrue(
http_request(self.url,
json_payload=self.json_payload,
http_verb=http_verb), falcon.HTTP_200)
def _get_tenant_from_coordinator(tenant_id, message_token, message):
"""
This method retrieves tenant data from the coordinator, and persists the
tenant data in the local cache for future lookups. The message is then
handed off to be packed with correlation data.
"""
config = _get_config_from_cache()
try:
resp = http_request('{0}/tenant/{1}'.format(config.coordinator_uri,
tenant_id),
{
MESSAGE_TOKEN: message_token,
'hostname': config.hostname
},
http_verb='GET')
except requests.RequestException as ex:
_LOG.exception(ex.message)
raise errors.CoordinatorCommunicationError
if resp.status_code == httplib.OK:
response_body = resp.json()
#load new tenant data from response body
tenant = tenant_util.load_tenant_from_dict(response_body['tenant'])
# update the cache with new tenant info
_save_tenant_to_cache(tenant_id, tenant)
# add correlation to message
_add_correlation_info_to_message(tenant, message)
elif resp.status_code == httplib.NOT_FOUND:
error_message = 'unable to locate tenant.'
_LOG.debug(error_message)
raise errors.ResourceNotFoundError(error_message)
else:
#coordinator responds, but coordinator datasink could be unreachable
raise errors.CoordinatorCommunicationError
def _get_tenant_from_coordinator(tenant_id, message_token, message):
"""
This method retrieves tenant data from the coordinator, and persists the
tenant data in the local cache for future lookups. The message is then
handed off to be packed with correlation data.
"""
config = _get_config_from_cache()
try:
resp = http_request(
'{0}/tenant/{1}'.format(config.coordinator_uri, tenant_id),
{MESSAGE_TOKEN: message_token, 'hostname': config.hostname},
http_verb='GET')
except requests.RequestException as ex:
_LOG.exception(ex.message)
raise errors.CoordinatorCommunicationError
if resp.status_code == httplib.OK:
response_body = resp.json()
#load new tenant data from response body
tenant = tenant_util.load_tenant_from_dict(response_body['tenant'])
# update the cache with new tenant info
_save_tenant_to_cache(tenant_id, tenant)
# add correlation to message
_add_correlation_info_to_message(tenant, message)
elif resp.status_code == httplib.NOT_FOUND:
error_message = 'unable to locate tenant.'
_LOG.debug(error_message)
raise errors.ResourceNotFoundError(error_message)
else:
#coordinator responds, but coordinator datasink could be unreachable
raise errors.CoordinatorCommunicationError
def _register_worker_online(self, status):
"""
register the worker with the coordinator with an online status
"""
cache = ConfigCache()
config = cache.get_config()
token_header = {"WORKER-TOKEN": config.worker_token}
request_uri = "{0}/worker/{1}/status".format(
config.coordinator_uri, config.worker_id)
status = {"worker_status": status}
try:
resp = http_request(request_uri, token_header,
jsonutils.dumps(status), http_verb='PUT')
except requests.RequestException:
return False
if resp.status_code == httplib.OK:
return True
def test_should_cause_a_request_exception(self):
with patch.object(requests, 'get') as mock_method:
with self.assertRaises(requests.RequestException):
mock_method.side_effect = requests.RequestException
http_request(self.url, json_payload=self.json_payload)
def test_should_cause_a_request_exception(self):
with patch.object(requests, 'get') as mock_method:
with self.assertRaises(requests.RequestException):
mock_method.side_effect = requests.RequestException
http_request(self.url, json_payload=self.json_payload)
