class ServiceMethodCaller(object): """ This class maintains a pool of service clients. """ DEFAULT_POOL_SIZE = 5 CLIENTS_PER_SERVICE_CONFIG = 5 MOCK = False # this is for tests def __init__(self, service_registry_redis_config, services, logger=None): self._registry_redis_config = service_registry_redis_config self._registry = RedisServiceRegistry(**(self._registry_redis_config or {})) self.logger = logger if self.MOCK: return self._managed_services = {} for service in services: if isinstance(service, tuple) or isinstance(service, list): service_name = service[0] try: pool_size = int(service[1]) except IndexError: pool_size = self.DEFAULT_POOL_SIZE else: service_name = service pool_size = self.DEFAULT_POOL_SIZE self._managed_services[service_name] = [pool_size] for service_name, value in self._managed_services.items(): self._managed_services[service_name].append( self._create_service_pool(service_name, value[0]) ) self.log('debug', 'created service method caller') def _create_service_pool(self, service_name, pool_size): resources = [] service_configs = self._registry.discover_service(service_name, num=pool_size) for config in service_configs: for i in range(self.CLIENTS_PER_SERVICE_CONFIG): self.log('debug', 'creating %d client resource for service ' 'config: %s' % (i + 1, config)) resource = ServiceClientResource(service_name, config, self.logger) resources.append(resource) self.log('debug', 'created a pool of clients: %r' % resources) return ResourcePool(resources) def log(self, level, message): try: if not hasattr(self, 'logger'): return logger = self.logger if logger is None: return if not hasattr(logger, level): return logger_method = getattr(logger, level) if not logger_method: return logger_method(message) except: pass def __call__(self, method, service, request, response_class=None, timeout=DEFAULT_TIME_OUT, max_tries=DEFAULT_MAX_TRIES, sleep_before_retry=DEFAULT_SLEEP_BEFORE_RETRY): """ calls function 'method' on service 'service' :param method: :param service: :param request: :param response_class: :return: """ if self.MOCK: return if service not in self._managed_services: raise UnknownServiceError('service: %s unknown' % service) try: pool = self._managed_services[service][1] with pool.acquire(timeout=RESOURCE_ACQUIRING_TIMEOUT) as resource: client = resource.client self.log('debug', 'using client: %r' % client) if hasattr(request, 'SerializeToString'): request.header.request_guid = str(uuid.uuid4()) self.log('info', 'calling %s method on %s service ' 'with request guid: %s' % (method, service, request.header.request_guid)) request_message = request.SerializeToString() else: request_message = str(request) response = client.request(method, request_message, response_class=response_class, timeout=timeout, max_tries=max_tries, sleep_before_retry=sleep_before_retry) if hasattr(request, 'SerializeToString'): response_type = 'good' if response.header.success else 'bad' self.log('info', 'received %s response for %s method from %s ' 'service for request guid: %s in %s ' 'microseconds' % (response_type, method, service, response.header.request_guid, response.header.response_time)) else: self.log('info', 'received response: %s' % response) return response except Queue.Empty: self.log('error', 'no client to call method: %s on service: %s ' % (method, service)) raise ClientResourceNotAvailableError() except Exception as exception: import traceback self.log('error', 'Error while calling method: %s on ' 'service: %s. traceback: %s' % (method, service, traceback.format_exc())) raise exception
class ServiceMethodCaller(object): """ This class maintains a pool of service clients. """ DEFAULT_POOL_SIZE = 5 CLIENTS_PER_SERVICE_CONFIG = 5 MOCK = False # this is for tests def __init__(self, service_registry_redis_config, services, logger=None): self._registry_redis_config = service_registry_redis_config self._registry = RedisServiceRegistry( **(self._registry_redis_config or {})) self.logger = logger if self.MOCK: return self._managed_services = {} for service in services: if isinstance(service, tuple) or isinstance(service, list): service_name = service[0] try: pool_size = int(service[1]) except IndexError: pool_size = self.DEFAULT_POOL_SIZE else: service_name = service pool_size = self.DEFAULT_POOL_SIZE self._managed_services[service_name] = [pool_size] for service_name, value in self._managed_services.items(): self._managed_services[service_name].append( self._create_service_pool(service_name, value[0])) self.log('debug', 'created service method caller') def _create_service_pool(self, service_name, pool_size): resources = [] service_configs = self._registry.discover_service(service_name, num=pool_size) for config in service_configs: for i in range(self.CLIENTS_PER_SERVICE_CONFIG): self.log( 'debug', 'creating %d client resource for service ' 'config: %s' % (i + 1, config)) resource = ServiceClientResource(service_name, config, self.logger) resources.append(resource) self.log('debug', 'created a pool of clients: %r' % resources) return ResourcePool(resources) def log(self, level, message): try: if not hasattr(self, 'logger'): return logger = self.logger if logger is None: return if not hasattr(logger, level): return logger_method = getattr(logger, level) if not logger_method: return logger_method(message) except: pass def __call__(self, method, service, request, response_class=None, timeout=DEFAULT_TIME_OUT, max_tries=DEFAULT_MAX_TRIES, sleep_before_retry=DEFAULT_SLEEP_BEFORE_RETRY): """ calls function 'method' on service 'service' :param method: :param service: :param request: :param response_class: :return: """ if self.MOCK: return if service not in self._managed_services: raise UnknownServiceError('service: %s unknown' % service) try: pool = self._managed_services[service][1] with pool.acquire(timeout=RESOURCE_ACQUIRING_TIMEOUT) as resource: client = resource.client self.log('debug', 'using client: %r' % client) if hasattr(request, 'SerializeToString'): request.header.request_guid = str(uuid.uuid4()) self.log( 'info', 'calling %s method on %s service ' 'with request guid: %s' % (method, service, request.header.request_guid)) request_message = request.SerializeToString() else: request_message = str(request) response = client.request( method, request_message, response_class=response_class, timeout=timeout, max_tries=max_tries, sleep_before_retry=sleep_before_retry) if hasattr(request, 'SerializeToString'): response_type = 'good' if response.header.success else 'bad' self.log( 'info', 'received %s response for %s method from %s ' 'service for request guid: %s in %s ' 'microseconds' % (response_type, method, service, response.header.request_guid, response.header.response_time)) else: self.log('info', 'received response: %s' % response) return response except Queue.Empty: self.log( 'error', 'no client to call method: %s on service: %s ' % (method, service)) raise ClientResourceNotAvailableError() except Exception as exception: import traceback self.log( 'error', 'Error while calling method: %s on ' 'service: %s. traceback: %s' % (method, service, traceback.format_exc())) raise exception
class ServiceClient(object): """ Base class to represent a client for a service """ def __init__(self, service_name, registry_redis_config=None, service_config=None, timeout=DEFAULT_TIME_OUT, sleep_before_retry=DEFAULT_SLEEP_BEFORE_RETRY, max_tries=DEFAULT_MAX_TRIES, heartbeat_frequency=DEFAULT_HEARTBEAT_FREQUENCY, start_heartbeat_thread=True, logger=None): self.logger = logger self._timeout = timeout self._sleep_before_retry = sleep_before_retry self._max_tries = max_tries self._heartbeat_frequency = heartbeat_frequency self._service_name = service_name if service_config: self._service_config = service_config else: self._registry = RedisServiceRegistry( **(registry_redis_config or {})) self._service_config = self._registry.discover_service( self._service_name)[0] self.start_time = current_timestamp() self.shutdown_time = None self.guid = str(uuid.uuid4()) self._context = zmq.Context() self._socket = socket_from_service_config(self._context, self._service_config, self._timeout) self.alive = True self.killed_by_error = None if start_heartbeat_thread: self._heartbeat_stop_event = threading.Event() heartbeat = ClientHeartbeat(self, self._service_config, self._timeout, self._heartbeat_frequency, self._max_tries) self._heartbeat_thread = threading.Thread( target=heartbeat, name='%s-client-heartbeat-%s' % (self._service_name, current_timestamp()), args=(self._heartbeat_stop_event, )) self._heartbeat_thread.start() else: self._heartbeat_thread = None def __repr__(self): return 'ServiceClient(guid=%s, service_name=%s, service_guid=%s)' % \ (self.guid, self._service_name, self._service_config['guid']) def log(self, level, message): try: if not hasattr(self, 'logger'): return logger = self.logger if logger is None: return if not hasattr(logger, level): return logger_method = getattr(logger, level) if not logger_method: return logger_method(message) except: pass def shutdown(self): if self._heartbeat_thread is None: return if self.shutdown_time is not None: return self.alive = False self._heartbeat_stop_event.set() self._heartbeat_thread.join() self.log('error', 'stopped heartbeat thread') self._socket.close() self._socket = None self.shutdown_time = current_timestamp() def _setup_socket(self, reuse=True, timeout=DEFAULT_TIME_OUT): if reuse and self._socket is not None: return self._socket = socket_from_service_config(self._context, self._service_config, timeout) def ping(self): return self.request('heartbeat', 'ping') def healthcheck(self): return self.request('healthcheck', 'health') def description(self): return self.request('description', 'description') def stop(self): return self.request('stop', 'stop') def request(self, function_name, request_message, response_class=None, timeout=DEFAULT_TIME_OUT, max_tries=DEFAULT_MAX_TRIES, sleep_before_retry=DEFAULT_SLEEP_BEFORE_RETRY): try_num = 0 sleep_duration = None error = None self._setup_socket(timeout=timeout) while self.alive and try_num < max_tries: if function_name not in self._service_config['functions']: raise ServiceFunctionNotAvailableError( '%r: function: %s not available for service: %s' % (self, function_name, self._service_name)) if sleep_duration: self.log('debug', 'Will try again in %s milliseconds' % sleep_duration) time.sleep(sleep_duration / 1000.0) self._setup_socket(reuse=False, timeout=timeout) try: self._socket.send_multipart( [str(function_name), request_message]) response_string = self._socket.recv() if response_class is None: return response_string else: response = response_class() response.ParseFromString(response_string) return response except zmq.error.Again: error = ServiceClientTimeoutError(self._service_name, function_name, timeout, max_tries, sleep_before_retry) self.log( 'debug', '%r can not complete function: %s of ' 'service: %s in %s milliseconds' % (self, function_name, self._service_name, timeout)) self._socket.close() self._socket = None sleep_duration = pow(2, try_num) * sleep_before_retry try_num += 1 except zmq.error.ZMQError as exception: error = ServiceClientError(exception) self.log( 'error', 'ZMQError in %r while requesting ' 'function: %s of ' 'service: %s. Error: %r' % (self, function_name, self._service_name, exception)) break except Exception as exception: error = ServiceClientError(exception) self.log( 'error', 'Error in %r while requesting ' 'function: %s of service: %s. Error: %r' % (self, function_name, self._service_name, exception)) break if not self.alive: self.log('error', '%r is no longer alive, shutting it ' 'down.' % self) self.shutdown() self.log('debug', 'heartbeat of %r shutdown' % self) if error: self.killed_by_error = error if isinstance(error, ServiceClientTimeoutError): self.log( 'error', '%r can not complete function: %s of ' 'service: %s in %s tries. Something must be ' 'wrong.' % (self, function_name, self._service_name, max_tries)) raise error raise ServiceClientError('%r Should never get here' % self)
class ServiceClient(object): """ Base class to represent a client for a service """ def __init__(self, service_name, registry_redis_config=None, service_config=None, timeout=DEFAULT_TIME_OUT, sleep_before_retry=DEFAULT_SLEEP_BEFORE_RETRY, max_tries=DEFAULT_MAX_TRIES, heartbeat_frequency=DEFAULT_HEARTBEAT_FREQUENCY, start_heartbeat_thread=True, logger=None): self.logger = logger self._timeout = timeout self._sleep_before_retry = sleep_before_retry self._max_tries = max_tries self._heartbeat_frequency = heartbeat_frequency self._service_name = service_name if service_config: self._service_config = service_config else: self._registry = RedisServiceRegistry(**(registry_redis_config or {})) self._service_config = self._registry.discover_service( self._service_name)[0] self.start_time = current_timestamp() self.shutdown_time = None self.guid = str(uuid.uuid4()) self._context = zmq.Context() self._socket = socket_from_service_config(self._context, self._service_config, self._timeout) self.alive = True self.killed_by_error = None if start_heartbeat_thread: self._heartbeat_stop_event = threading.Event() heartbeat = ClientHeartbeat(self, self._service_config, self._timeout, self._heartbeat_frequency, self._max_tries) self._heartbeat_thread = threading.Thread( target=heartbeat, name='%s-client-heartbeat-%s' % (self._service_name, current_timestamp()), args=(self._heartbeat_stop_event, ) ) self._heartbeat_thread.start() else: self._heartbeat_thread = None def __repr__(self): return 'ServiceClient(guid=%s, service_name=%s, service_guid=%s)' % \ (self.guid, self._service_name, self._service_config['guid']) def log(self, level, message): try: if not hasattr(self, 'logger'): return logger = self.logger if logger is None: return if not hasattr(logger, level): return logger_method = getattr(logger, level) if not logger_method: return logger_method(message) except: pass def shutdown(self): if self._heartbeat_thread is None: return if self.shutdown_time is not None: return self.alive = False self._heartbeat_stop_event.set() self._heartbeat_thread.join() self.log('error', 'stopped heartbeat thread') self._socket.close() self._socket = None self.shutdown_time = current_timestamp() def _setup_socket(self, reuse=True, timeout=DEFAULT_TIME_OUT): if reuse and self._socket is not None: return self._socket = socket_from_service_config(self._context, self._service_config, timeout) def ping(self): return self.request('heartbeat', 'ping') def healthcheck(self): return self.request('healthcheck', 'health') def description(self): return self.request('description', 'description') def stop(self): return self.request('stop', 'stop') def request(self, function_name, request_message, response_class=None, timeout=DEFAULT_TIME_OUT, max_tries=DEFAULT_MAX_TRIES, sleep_before_retry=DEFAULT_SLEEP_BEFORE_RETRY): try_num = 0 sleep_duration = None error = None self._setup_socket(timeout=timeout) while self.alive and try_num < max_tries: if function_name not in self._service_config['functions']: raise ServiceFunctionNotAvailableError( '%r: function: %s not available for service: %s' % ( self, function_name, self._service_name )) if sleep_duration: self.log('debug', 'Will try again in %s milliseconds' % sleep_duration) time.sleep(sleep_duration/1000.0) self._setup_socket(reuse=False, timeout=timeout) try: self._socket.send_multipart( [str(function_name), request_message] ) response_string = self._socket.recv() if response_class is None: return response_string else: response = response_class() response.ParseFromString(response_string) return response except zmq.error.Again: error = ServiceClientTimeoutError(self._service_name, function_name, timeout, max_tries, sleep_before_retry) self.log('debug', '%r can not complete function: %s of ' 'service: %s in %s milliseconds' % (self, function_name, self._service_name, timeout)) self._socket.close() self._socket = None sleep_duration = pow(2, try_num) * sleep_before_retry try_num += 1 except zmq.error.ZMQError as exception: error = ServiceClientError(exception) self.log('error', 'ZMQError in %r while requesting ' 'function: %s of ' 'service: %s. Error: %r' % (self, function_name, self._service_name, exception)) break except Exception as exception: error = ServiceClientError(exception) self.log('error', 'Error in %r while requesting ' 'function: %s of service: %s. Error: %r' % (self, function_name, self._service_name, exception)) break if not self.alive: self.log('error', '%r is no longer alive, shutting it ' 'down.' % self) self.shutdown() self.log('debug', 'heartbeat of %r shutdown' % self) if error: self.killed_by_error = error if isinstance(error, ServiceClientTimeoutError): self.log('error', '%r can not complete function: %s of ' 'service: %s in %s tries. Something must be ' 'wrong.' % (self, function_name, self._service_name, max_tries)) raise error raise ServiceClientError('%r Should never get here' % self)