class ServiceMethodCaller(object):
"""
This class maintains a pool of service clients.
"""
DEFAULT_POOL_SIZE = 5
CLIENTS_PER_SERVICE_CONFIG = 5
MOCK = False # this is for tests
def __init__(self, service_registry_redis_config, services,
logger=None):
self._registry_redis_config = service_registry_redis_config
self._registry = RedisServiceRegistry(**(self._registry_redis_config
or {}))
self.logger = logger
if self.MOCK:
return
self._managed_services = {}
for service in services:
if isinstance(service, tuple) or isinstance(service, list):
service_name = service[0]
try:
pool_size = int(service[1])
except IndexError:
pool_size = self.DEFAULT_POOL_SIZE
else:
service_name = service
pool_size = self.DEFAULT_POOL_SIZE
self._managed_services[service_name] = [pool_size]
for service_name, value in self._managed_services.items():
self._managed_services[service_name].append(
self._create_service_pool(service_name, value[0])
)
self.log('debug', 'created service method caller')
def _create_service_pool(self, service_name, pool_size):
resources = []
service_configs = self._registry.discover_service(service_name,
num=pool_size)
for config in service_configs:
for i in range(self.CLIENTS_PER_SERVICE_CONFIG):
self.log('debug', 'creating %d client resource for service '
'config: %s' % (i + 1, config))
resource = ServiceClientResource(service_name, config,
self.logger)
resources.append(resource)
self.log('debug', 'created a pool of clients: %r' % resources)
return ResourcePool(resources)
def log(self, level, message):
try:
if not hasattr(self, 'logger'):
return
logger = self.logger
if logger is None:
return
if not hasattr(logger, level):
return
logger_method = getattr(logger, level)
if not logger_method:
return
logger_method(message)
except:
pass
def __call__(self, method, service, request, response_class=None,
timeout=DEFAULT_TIME_OUT, max_tries=DEFAULT_MAX_TRIES,
sleep_before_retry=DEFAULT_SLEEP_BEFORE_RETRY):
"""
calls function 'method' on service 'service'
:param method:
:param service:
:param request:
:param response_class:
:return:
"""
if self.MOCK:
return
if service not in self._managed_services:
raise UnknownServiceError('service: %s unknown' % service)
try:
pool = self._managed_services[service][1]
with pool.acquire(timeout=RESOURCE_ACQUIRING_TIMEOUT) as resource:
client = resource.client
self.log('debug', 'using client: %r' % client)
if hasattr(request, 'SerializeToString'):
request.header.request_guid = str(uuid.uuid4())
self.log('info', 'calling %s method on %s service '
'with request guid: %s' %
(method, service,
request.header.request_guid))
request_message = request.SerializeToString()
else:
request_message = str(request)
response = client.request(method, request_message,
response_class=response_class,
timeout=timeout, max_tries=max_tries,
sleep_before_retry=sleep_before_retry)
if hasattr(request, 'SerializeToString'):
response_type = 'good' if response.header.success else 'bad'
self.log('info', 'received %s response for %s method from %s '
'service for request guid: %s in %s '
'microseconds' %
(response_type, method, service,
response.header.request_guid,
response.header.response_time))
else:
self.log('info', 'received response: %s' % response)
return response
except Queue.Empty:
self.log('error', 'no client to call method: %s on service: %s '
% (method, service))
raise ClientResourceNotAvailableError()
except Exception as exception:
import traceback
self.log('error', 'Error while calling method: %s on '
'service: %s. traceback: %s' %
(method, service, traceback.format_exc()))
raise exception
class ServiceMethodCaller(object):
"""
This class maintains a pool of service clients.
"""
DEFAULT_POOL_SIZE = 5
CLIENTS_PER_SERVICE_CONFIG = 5
MOCK = False # this is for tests
def __init__(self, service_registry_redis_config, services, logger=None):
self._registry_redis_config = service_registry_redis_config
self._registry = RedisServiceRegistry(
**(self._registry_redis_config or {}))
self.logger = logger
if self.MOCK:
return
self._managed_services = {}
for service in services:
if isinstance(service, tuple) or isinstance(service, list):
service_name = service[0]
try:
pool_size = int(service[1])
except IndexError:
pool_size = self.DEFAULT_POOL_SIZE
else:
service_name = service
pool_size = self.DEFAULT_POOL_SIZE
self._managed_services[service_name] = [pool_size]
for service_name, value in self._managed_services.items():
self._managed_services[service_name].append(
self._create_service_pool(service_name, value[0]))
self.log('debug', 'created service method caller')
def _create_service_pool(self, service_name, pool_size):
resources = []
service_configs = self._registry.discover_service(service_name,
num=pool_size)
for config in service_configs:
for i in range(self.CLIENTS_PER_SERVICE_CONFIG):
self.log(
'debug', 'creating %d client resource for service '
'config: %s' % (i + 1, config))
resource = ServiceClientResource(service_name, config,
self.logger)
resources.append(resource)
self.log('debug', 'created a pool of clients: %r' % resources)
return ResourcePool(resources)
def log(self, level, message):
try:
if not hasattr(self, 'logger'):
return
logger = self.logger
if logger is None:
return
if not hasattr(logger, level):
return
logger_method = getattr(logger, level)
if not logger_method:
return
logger_method(message)
except:
pass
def __call__(self,
method,
service,
request,
response_class=None,
timeout=DEFAULT_TIME_OUT,
max_tries=DEFAULT_MAX_TRIES,
sleep_before_retry=DEFAULT_SLEEP_BEFORE_RETRY):
"""
calls function 'method' on service 'service'
:param method:
:param service:
:param request:
:param response_class:
:return:
"""
if self.MOCK:
return
if service not in self._managed_services:
raise UnknownServiceError('service: %s unknown' % service)
try:
pool = self._managed_services[service][1]
with pool.acquire(timeout=RESOURCE_ACQUIRING_TIMEOUT) as resource:
client = resource.client
self.log('debug', 'using client: %r' % client)
if hasattr(request, 'SerializeToString'):
request.header.request_guid = str(uuid.uuid4())
self.log(
'info', 'calling %s method on %s service '
'with request guid: %s' %
(method, service, request.header.request_guid))
request_message = request.SerializeToString()
else:
request_message = str(request)
response = client.request(
method,
request_message,
response_class=response_class,
timeout=timeout,
max_tries=max_tries,
sleep_before_retry=sleep_before_retry)
if hasattr(request, 'SerializeToString'):
response_type = 'good' if response.header.success else 'bad'
self.log(
'info', 'received %s response for %s method from %s '
'service for request guid: %s in %s '
'microseconds' % (response_type, method, service,
response.header.request_guid,
response.header.response_time))
else:
self.log('info', 'received response: %s' % response)
return response
except Queue.Empty:
self.log(
'error', 'no client to call method: %s on service: %s ' %
(method, service))
raise ClientResourceNotAvailableError()
except Exception as exception:
import traceback
self.log(
'error', 'Error while calling method: %s on '
'service: %s. traceback: %s' %
(method, service, traceback.format_exc()))
raise exception
class ServiceClient(object):
"""
Base class to represent a client for a service
"""
def __init__(self,
service_name,
registry_redis_config=None,
service_config=None,
timeout=DEFAULT_TIME_OUT,
sleep_before_retry=DEFAULT_SLEEP_BEFORE_RETRY,
max_tries=DEFAULT_MAX_TRIES,
heartbeat_frequency=DEFAULT_HEARTBEAT_FREQUENCY,
start_heartbeat_thread=True,
logger=None):
self.logger = logger
self._timeout = timeout
self._sleep_before_retry = sleep_before_retry
self._max_tries = max_tries
self._heartbeat_frequency = heartbeat_frequency
self._service_name = service_name
if service_config:
self._service_config = service_config
else:
self._registry = RedisServiceRegistry(
**(registry_redis_config or {}))
self._service_config = self._registry.discover_service(
self._service_name)[0]
self.start_time = current_timestamp()
self.shutdown_time = None
self.guid = str(uuid.uuid4())
self._context = zmq.Context()
self._socket = socket_from_service_config(self._context,
self._service_config,
self._timeout)
self.alive = True
self.killed_by_error = None
if start_heartbeat_thread:
self._heartbeat_stop_event = threading.Event()
heartbeat = ClientHeartbeat(self, self._service_config,
self._timeout,
self._heartbeat_frequency,
self._max_tries)
self._heartbeat_thread = threading.Thread(
target=heartbeat,
name='%s-client-heartbeat-%s' %
(self._service_name, current_timestamp()),
args=(self._heartbeat_stop_event, ))
self._heartbeat_thread.start()
else:
self._heartbeat_thread = None
def __repr__(self):
return 'ServiceClient(guid=%s, service_name=%s, service_guid=%s)' % \
(self.guid, self._service_name, self._service_config['guid'])
def log(self, level, message):
try:
if not hasattr(self, 'logger'):
return
logger = self.logger
if logger is None:
return
if not hasattr(logger, level):
return
logger_method = getattr(logger, level)
if not logger_method:
return
logger_method(message)
except:
pass
def shutdown(self):
if self._heartbeat_thread is None:
return
if self.shutdown_time is not None:
return
self.alive = False
self._heartbeat_stop_event.set()
self._heartbeat_thread.join()
self.log('error', 'stopped heartbeat thread')
self._socket.close()
self._socket = None
self.shutdown_time = current_timestamp()
def _setup_socket(self, reuse=True, timeout=DEFAULT_TIME_OUT):
if reuse and self._socket is not None:
return
self._socket = socket_from_service_config(self._context,
self._service_config,
timeout)
def ping(self):
return self.request('heartbeat', 'ping')
def healthcheck(self):
return self.request('healthcheck', 'health')
def description(self):
return self.request('description', 'description')
def stop(self):
return self.request('stop', 'stop')
def request(self,
function_name,
request_message,
response_class=None,
timeout=DEFAULT_TIME_OUT,
max_tries=DEFAULT_MAX_TRIES,
sleep_before_retry=DEFAULT_SLEEP_BEFORE_RETRY):
try_num = 0
sleep_duration = None
error = None
self._setup_socket(timeout=timeout)
while self.alive and try_num < max_tries:
if function_name not in self._service_config['functions']:
raise ServiceFunctionNotAvailableError(
'%r: function: %s not available for service: %s' %
(self, function_name, self._service_name))
if sleep_duration:
self.log('debug',
'Will try again in %s milliseconds' % sleep_duration)
time.sleep(sleep_duration / 1000.0)
self._setup_socket(reuse=False, timeout=timeout)
try:
self._socket.send_multipart(
[str(function_name), request_message])
response_string = self._socket.recv()
if response_class is None:
return response_string
else:
response = response_class()
response.ParseFromString(response_string)
return response
except zmq.error.Again:
error = ServiceClientTimeoutError(self._service_name,
function_name, timeout,
max_tries,
sleep_before_retry)
self.log(
'debug', '%r can not complete function: %s of '
'service: %s in %s milliseconds' %
(self, function_name, self._service_name, timeout))
self._socket.close()
self._socket = None
sleep_duration = pow(2, try_num) * sleep_before_retry
try_num += 1
except zmq.error.ZMQError as exception:
error = ServiceClientError(exception)
self.log(
'error', 'ZMQError in %r while requesting '
'function: %s of '
'service: %s. Error: %r' %
(self, function_name, self._service_name, exception))
break
except Exception as exception:
error = ServiceClientError(exception)
self.log(
'error', 'Error in %r while requesting '
'function: %s of service: %s. Error: %r' %
(self, function_name, self._service_name, exception))
break
if not self.alive:
self.log('error', '%r is no longer alive, shutting it '
'down.' % self)
self.shutdown()
self.log('debug', 'heartbeat of %r shutdown' % self)
if error:
self.killed_by_error = error
if isinstance(error, ServiceClientTimeoutError):
self.log(
'error', '%r can not complete function: %s of '
'service: %s in %s tries. Something must be '
'wrong.' %
(self, function_name, self._service_name, max_tries))
raise error
raise ServiceClientError('%r Should never get here' % self)
class ServiceClient(object):
"""
Base class to represent a client for a service
"""
def __init__(self, service_name,
registry_redis_config=None,
service_config=None,
timeout=DEFAULT_TIME_OUT,
sleep_before_retry=DEFAULT_SLEEP_BEFORE_RETRY,
max_tries=DEFAULT_MAX_TRIES,
heartbeat_frequency=DEFAULT_HEARTBEAT_FREQUENCY,
start_heartbeat_thread=True,
logger=None):
self.logger = logger
self._timeout = timeout
self._sleep_before_retry = sleep_before_retry
self._max_tries = max_tries
self._heartbeat_frequency = heartbeat_frequency
self._service_name = service_name
if service_config:
self._service_config = service_config
else:
self._registry = RedisServiceRegistry(**(registry_redis_config or
{}))
self._service_config = self._registry.discover_service(
self._service_name)[0]
self.start_time = current_timestamp()
self.shutdown_time = None
self.guid = str(uuid.uuid4())
self._context = zmq.Context()
self._socket = socket_from_service_config(self._context,
self._service_config,
self._timeout)
self.alive = True
self.killed_by_error = None
if start_heartbeat_thread:
self._heartbeat_stop_event = threading.Event()
heartbeat = ClientHeartbeat(self, self._service_config,
self._timeout, self._heartbeat_frequency,
self._max_tries)
self._heartbeat_thread = threading.Thread(
target=heartbeat,
name='%s-client-heartbeat-%s' % (self._service_name,
current_timestamp()),
args=(self._heartbeat_stop_event, )
)
self._heartbeat_thread.start()
else:
self._heartbeat_thread = None
def __repr__(self):
return 'ServiceClient(guid=%s, service_name=%s, service_guid=%s)' % \
(self.guid, self._service_name, self._service_config['guid'])
def log(self, level, message):
try:
if not hasattr(self, 'logger'):
return
logger = self.logger
if logger is None:
return
if not hasattr(logger, level):
return
logger_method = getattr(logger, level)
if not logger_method:
return
logger_method(message)
except:
pass
def shutdown(self):
if self._heartbeat_thread is None:
return
if self.shutdown_time is not None:
return
self.alive = False
self._heartbeat_stop_event.set()
self._heartbeat_thread.join()
self.log('error', 'stopped heartbeat thread')
self._socket.close()
self._socket = None
self.shutdown_time = current_timestamp()
def _setup_socket(self, reuse=True, timeout=DEFAULT_TIME_OUT):
if reuse and self._socket is not None:
return
self._socket = socket_from_service_config(self._context,
self._service_config, timeout)
def ping(self):
return self.request('heartbeat', 'ping')
def healthcheck(self):
return self.request('healthcheck', 'health')
def description(self):
return self.request('description', 'description')
def stop(self):
return self.request('stop', 'stop')
def request(self, function_name, request_message, response_class=None,
timeout=DEFAULT_TIME_OUT, max_tries=DEFAULT_MAX_TRIES,
sleep_before_retry=DEFAULT_SLEEP_BEFORE_RETRY):
try_num = 0
sleep_duration = None
error = None
self._setup_socket(timeout=timeout)
while self.alive and try_num < max_tries:
if function_name not in self._service_config['functions']:
raise ServiceFunctionNotAvailableError(
'%r: function: %s not available for service: %s' % (
self, function_name, self._service_name
))
if sleep_duration:
self.log('debug', 'Will try again in %s milliseconds' %
sleep_duration)
time.sleep(sleep_duration/1000.0)
self._setup_socket(reuse=False, timeout=timeout)
try:
self._socket.send_multipart(
[str(function_name), request_message]
)
response_string = self._socket.recv()
if response_class is None:
return response_string
else:
response = response_class()
response.ParseFromString(response_string)
return response
except zmq.error.Again:
error = ServiceClientTimeoutError(self._service_name,
function_name, timeout,
max_tries,
sleep_before_retry)
self.log('debug', '%r can not complete function: %s of '
'service: %s in %s milliseconds' %
(self, function_name, self._service_name, timeout))
self._socket.close()
self._socket = None
sleep_duration = pow(2, try_num) * sleep_before_retry
try_num += 1
except zmq.error.ZMQError as exception:
error = ServiceClientError(exception)
self.log('error', 'ZMQError in %r while requesting '
'function: %s of '
'service: %s. Error: %r' %
(self, function_name, self._service_name, exception))
break
except Exception as exception:
error = ServiceClientError(exception)
self.log('error', 'Error in %r while requesting '
'function: %s of service: %s. Error: %r' %
(self, function_name, self._service_name, exception))
break
if not self.alive:
self.log('error', '%r is no longer alive, shutting it '
'down.' % self)
self.shutdown()
self.log('debug', 'heartbeat of %r shutdown' % self)
if error:
self.killed_by_error = error
if isinstance(error, ServiceClientTimeoutError):
self.log('error', '%r can not complete function: %s of '
'service: %s in %s tries. Something must be '
'wrong.' % (self, function_name,
self._service_name, max_tries))
raise error
raise ServiceClientError('%r Should never get here' % self)