def __init__(self, service_registry_redis_config, services, logger=None): self._registry_redis_config = service_registry_redis_config self._registry = RedisServiceRegistry( **(self._registry_redis_config or {})) self.logger = logger if self.MOCK: return self._managed_services = {} for service in services: if isinstance(service, tuple) or isinstance(service, list): service_name = service[0] try: pool_size = int(service[1]) except IndexError: pool_size = self.DEFAULT_POOL_SIZE else: service_name = service pool_size = self.DEFAULT_POOL_SIZE self._managed_services[service_name] = [pool_size] for service_name, value in self._managed_services.items(): self._managed_services[service_name].append( self._create_service_pool(service_name, value[0])) self.log('debug', 'created service method caller')
def __init__(self, service_name, registry_redis_config=None, service_config=None, timeout=DEFAULT_TIME_OUT, sleep_before_retry=DEFAULT_SLEEP_BEFORE_RETRY, max_tries=DEFAULT_MAX_TRIES, heartbeat_frequency=DEFAULT_HEARTBEAT_FREQUENCY, start_heartbeat_thread=True, logger=None): self.logger = logger self._timeout = timeout self._sleep_before_retry = sleep_before_retry self._max_tries = max_tries self._heartbeat_frequency = heartbeat_frequency self._service_name = service_name if service_config: self._service_config = service_config else: self._registry = RedisServiceRegistry( **(registry_redis_config or {})) self._service_config = self._registry.discover_service( self._service_name)[0] self.start_time = current_timestamp() self.shutdown_time = None self.guid = str(uuid.uuid4()) self._context = zmq.Context() self._socket = socket_from_service_config(self._context, self._service_config, self._timeout) self.alive = True self.killed_by_error = None if start_heartbeat_thread: self._heartbeat_stop_event = threading.Event() heartbeat = ClientHeartbeat(self, self._service_config, self._timeout, self._heartbeat_frequency, self._max_tries) self._heartbeat_thread = threading.Thread( target=heartbeat, name='%s-client-heartbeat-%s' % (self._service_name, current_timestamp()), args=(self._heartbeat_stop_event, )) self._heartbeat_thread.start() else: self._heartbeat_thread = None
def __init__(self, service_registry_redis_config, services, logger=None): self._registry_redis_config = service_registry_redis_config self._registry = RedisServiceRegistry(**(self._registry_redis_config or {})) self.logger = logger if self.MOCK: return self._managed_services = {} for service in services: if isinstance(service, tuple) or isinstance(service, list): service_name = service[0] try: pool_size = int(service[1]) except IndexError: pool_size = self.DEFAULT_POOL_SIZE else: service_name = service pool_size = self.DEFAULT_POOL_SIZE self._managed_services[service_name] = [pool_size] for service_name, value in self._managed_services.items(): self._managed_services[service_name].append( self._create_service_pool(service_name, value[0]) ) self.log('debug', 'created service method caller')
def __init__(self, service_name, registry_redis_config=None, service_config=None, timeout=DEFAULT_TIME_OUT, sleep_before_retry=DEFAULT_SLEEP_BEFORE_RETRY, max_tries=DEFAULT_MAX_TRIES, heartbeat_frequency=DEFAULT_HEARTBEAT_FREQUENCY, start_heartbeat_thread=True, logger=None): self.logger = logger self._timeout = timeout self._sleep_before_retry = sleep_before_retry self._max_tries = max_tries self._heartbeat_frequency = heartbeat_frequency self._service_name = service_name if service_config: self._service_config = service_config else: self._registry = RedisServiceRegistry(**(registry_redis_config or {})) self._service_config = self._registry.discover_service( self._service_name)[0] self.start_time = current_timestamp() self.shutdown_time = None self.guid = str(uuid.uuid4()) self._context = zmq.Context() self._socket = socket_from_service_config(self._context, self._service_config, self._timeout) self.alive = True self.killed_by_error = None if start_heartbeat_thread: self._heartbeat_stop_event = threading.Event() heartbeat = ClientHeartbeat(self, self._service_config, self._timeout, self._heartbeat_frequency, self._max_tries) self._heartbeat_thread = threading.Thread( target=heartbeat, name='%s-client-heartbeat-%s' % (self._service_name, current_timestamp()), args=(self._heartbeat_stop_event, ) ) self._heartbeat_thread.start() else: self._heartbeat_thread = None
class Service(object): """ """ config = None VALID_CONN_METHOD = {"bind", "connect"} VALID_SCK_TYPES = {"REQ", "REP", "PUB", "SUB", "PUSH", "PULL"} MESSAGE_HANDLERS = {} CONFIG_REDIS_SECTION = "config_redis" BASE_TCP_ADDR = 'tcp://%s:%d' DEFAULT_FUNCTIONS = ["heartbeat", "healthcheck", "description", "stop"] DEFAULT_FUNCTION_MESSAGE_HANDLERS = { "heartbeat": HeartbeatHandler, "healthcheck": HealthCheckHandler, "description": DescriptionHandler, "stop": StopServiceHandler, "default": DefaultMessageHandler } EC2_INSTANCE_HOSTNAME_URL = \ "http://169.254.169.254/latest/meta-data/public-hostname" EC2_METADATA_REQUEST_TIMEOUT = 1 PID_DIR = "/home/ec2-user/publishing_services" FUNCTIONS_DECK_LENGTH = 10 def __repr__(self): return "%s(name=%s, host=%s, guid=%s, pid=%s, description=%s, " \ "functions=[%s])" % \ (self.__class__.__name__, self.name, self.host, self.guid, self.pid, self.description, ", ".join(self.functions)) def log(self, level, message): try: if not hasattr(self, 'logger'): return logger = self.logger if logger is None: return if not hasattr(logger, level): return logger_method = getattr(logger, level) if not logger_method: return logger_method(message) except: pass def determine_host(self): try: r = requests.get(self.EC2_INSTANCE_HOSTNAME_URL, timeout=self.EC2_METADATA_REQUEST_TIMEOUT) if r.status_code == requests.codes.OKAY: return r.content.strip() except Exception as exception: self.log('debug', 'looks like not running on an EC2 instance. ' 'Trying to determine host from config file. ' 'Error: %r' % exception) try: return self.config.get("global", "host") except Exception as exception: self.log('error', 'can not determine hostname, returning ' 'localhost. error: %r' % exception) return "localhost" def determine_guid(self): return hashlib.md5( ", ".join([ "%s service" % self.name, self.host, "%s" % self.pid, "%s" % self.start_time, "%s" % random.randint(1, 1000000000) ]) ).hexdigest() def _set_config(self, config_file=None): if not config_file: raise RuntimeError('A config file must be specified') logging.config.fileConfig(config_file) self.logger = logging.getLogger(self.__class__.__name__) self.config = ConfigParser.SafeConfigParser() self.config.read(config_file) self.description = self.config.get("global", "description") self.name = self.config.get("global", "name") self.env = self.config.get("global", "env") self.version = self.config.get("global", "version") self.start_time = current_timestamp() self.pid = os.getpid() self.proc = psutil.Process(self.pid) self.host = self.determine_host() self.guid = self.determine_guid() self.function_deque = collections.deque( maxlen=self.FUNCTIONS_DECK_LENGTH) self.stats = { 'num_messages': 0, 'num_success': 0, 'num_error': 0, 'avg_response_time': 0, 'min_response_time': 0, 'max_response_time': 0, 'last_response_time': 0 } self.pid_dir_path = self.PID_DIR try: self.pid_dir_path = self.config.get("global", "pid_dir") except ConfigParser.NoOptionError: pass self.pid_dir_path = "%s/%s" % (self.pid_dir_path, self.name) self.pid_file = "%s/%s" % (self.pid_dir_path, self.pid) self._registry = RedisServiceRegistry( **redis_config_from_config_file( self.config, "redis_service_registry", RedisServiceRegistry.DEFAULT_REDIS_CONFIG ) ) for k, v in self.DEFAULT_FUNCTION_MESSAGE_HANDLERS.items(): if k not in self.MESSAGE_HANDLERS: self.MESSAGE_HANDLERS[k] = v self.functions = self.MESSAGE_HANDLERS.keys() self._message_handlers = {} try: self._setup_sockets() self._setup_message_handlers() self._registry.register_service({ 'name': self.name, 'env': self.env, 'guid': self.guid, 'pid': self.pid, 'host': self.host, 'port': self.port, 'socket_type': self.socket_type, 'connect_method': self.connect_method, 'functions': json.dumps(self.functions), 'start_time': json.dumps(self.start_time), 'alive': json.dumps(True) }) except Exception as exception: import traceback self.log('error', 'Error while registering service: %s' % traceback.format_exc()) self._registry.deregister_service(self.name, self.guid, self.host) self.log('error', "%s while initializing %s service. Error: " "%r" % (exception.__class__.__name__, self.name, exception)) raise exception def _setup_sockets(self): self._poller = zmq.Poller() self._ports = {} self._context = zmq.Context() self.port, self.socket = self._get_socket_for_service() self._poller.register(self.socket, zmq.POLLIN) def _get_socket_for_service(self): port = self._registry.next_available_port(self.name, self.guid, self.host) self.socket_type = "REP" self.connect_method = "bind" try: self.socket_type = self.config.get("global", "socket_type").upper() self.connect_method = self.config.get("global", "connect_method").lower() except ConfigParser.NoSectionError: pass except ConfigParser.NoOptionError: pass if self.socket_type not in self.VALID_SCK_TYPES: raise RuntimeError( "Socket type %s not in set [%s] of valid socket types" % (self.connect_method, ", ".join(self.VALID_SCK_TYPES))) if self.connect_method not in self.VALID_CONN_METHOD: raise RuntimeError( "Connect method %s not in set [%s] of valid methods" % (self.connect_method, ", ".join(self.VALID_CONN_METHOD))) connect_string = self.BASE_TCP_ADDR % ( "*" if self.connect_method == "bind" else self.host, port ) socket = zmq_socket_from_socket_type(self._context, self.socket_type) getattr(socket, self.connect_method)(connect_string) return port, socket def _setup_message_handlers(self): for function, handler_class in self.MESSAGE_HANDLERS.items(): self._message_handlers[function] = handler_class( self, function, self.socket, logger=self.logger ) self.logger.debug("Registered handler: service: %s, function: %s, " "handler: %s", self.name, function, handler_class.__name__) def _run(self): while True: try: # self.logger.debug("poller: %s", self._poller) socks = dict(self._poller.poll()) except KeyboardInterrupt as e: raise e except Exception as e: self.logger.error(e) raise e if self.socket in socks: function, request = self.socket.recv_multipart() function = function if function in self._message_handlers \ else 'default' self.function_deque.appendleft(function) if function != 'heartbeat': self.logger.debug("Received RPC for function: %s", function) self.stats['num_messages'] += 1 response_start_time = current_timestamp() try: response = self._message_handlers[function].handle(request) self.stats['num_success'] += 1 except Exception: response = 'empty response' self.stats['num_error'] += 1 import traceback self.log('error', 'Error while processing request for ' 'function: %s. Traceback: %s' % (function, traceback.format_exc())) response_processing_time = current_timestamp() - \ response_start_time self.stats['last_response_time'] = response_processing_time self.stats['max_response_time'] = max( self.stats['max_response_time'], response_processing_time ) if self.stats['min_response_time'] == 0: self.stats['min_response_time'] = response_processing_time else: self.stats['min_response_time'] = min( self.stats['min_response_time'], response_processing_time ) self.stats['avg_response_time'] = ( response_processing_time + ((self.stats['num_messages'] - 1) * self.stats[ 'avg_response_time']) )/float(self.stats['num_messages']) self.socket.send(response) if function == 'stop': raise StopServiceError() def run(self): if not self.config: raise RuntimeError('A config file must be specified') try: self._run() except StopServiceError: self.logger.debug("Stopping :%s service in response to STOP " "message.") except KeyboardInterrupt as e: self.log('debug', 'KeyboardInterrupt while running %s service' % self.name) except Exception as exception: self.logger.error("Service crashed due to %s. args: [%s], message: " "%s", exception.__class__.__name__, ", ".join( exception.args), str(exception)) finally: self._registry.deregister_service(self.name, self.guid, self.host) try: os.remove(self.pid_file) except: pass self.logger.debug("Deregistered %s service", self.name) self.logger.debug("%s service stopped", self.name) @classmethod def get_cmd_line_parser(cls): argparser = argparse.ArgumentParser( description="User management service") argparser.add_argument("-c", "--config_file", required=True, help="config file") return argparser def run_service(self): set_time_zone() parser = self.get_cmd_line_parser() args = parser.parse_args() self._set_config(args.config_file) if not os.path.exists(self.pid_dir_path): os.makedirs(self.pid_dir_path) f = open(self.pid_file, 'w') f.write(json.dumps({ 'name': self.name, 'env': self.env, 'pid': self.pid, 'guid': self.guid, 'host': self.host, 'port': self.port, 'socket_type': self.socket_type, 'connect_method': self.connect_method, 'functions': self.functions, 'start_time': self.start_time, 'cmdline': self.proc.cmdline() }, indent=4)) f.close() print 'Running %r' % self self.run()
def _set_config(self, config_file=None): if not config_file: raise RuntimeError('A config file must be specified') logging.config.fileConfig(config_file) self.logger = logging.getLogger(self.__class__.__name__) self.config = ConfigParser.SafeConfigParser() self.config.read(config_file) self.description = self.config.get("global", "description") self.name = self.config.get("global", "name") self.env = self.config.get("global", "env") self.version = self.config.get("global", "version") self.start_time = current_timestamp() self.pid = os.getpid() self.proc = psutil.Process(self.pid) self.host = self.determine_host() self.guid = self.determine_guid() self.function_deque = collections.deque( maxlen=self.FUNCTIONS_DECK_LENGTH) self.stats = { 'num_messages': 0, 'num_success': 0, 'num_error': 0, 'avg_response_time': 0, 'min_response_time': 0, 'max_response_time': 0, 'last_response_time': 0 } self.pid_dir_path = self.PID_DIR try: self.pid_dir_path = self.config.get("global", "pid_dir") except ConfigParser.NoOptionError: pass self.pid_dir_path = "%s/%s" % (self.pid_dir_path, self.name) self.pid_file = "%s/%s" % (self.pid_dir_path, self.pid) self._registry = RedisServiceRegistry( **redis_config_from_config_file( self.config, "redis_service_registry", RedisServiceRegistry.DEFAULT_REDIS_CONFIG ) ) for k, v in self.DEFAULT_FUNCTION_MESSAGE_HANDLERS.items(): if k not in self.MESSAGE_HANDLERS: self.MESSAGE_HANDLERS[k] = v self.functions = self.MESSAGE_HANDLERS.keys() self._message_handlers = {} try: self._setup_sockets() self._setup_message_handlers() self._registry.register_service({ 'name': self.name, 'env': self.env, 'guid': self.guid, 'pid': self.pid, 'host': self.host, 'port': self.port, 'socket_type': self.socket_type, 'connect_method': self.connect_method, 'functions': json.dumps(self.functions), 'start_time': json.dumps(self.start_time), 'alive': json.dumps(True) }) except Exception as exception: import traceback self.log('error', 'Error while registering service: %s' % traceback.format_exc()) self._registry.deregister_service(self.name, self.guid, self.host) self.log('error', "%s while initializing %s service. Error: " "%r" % (exception.__class__.__name__, self.name, exception)) raise exception
class ServiceMethodCaller(object): """ This class maintains a pool of service clients. """ DEFAULT_POOL_SIZE = 5 CLIENTS_PER_SERVICE_CONFIG = 5 MOCK = False # this is for tests def __init__(self, service_registry_redis_config, services, logger=None): self._registry_redis_config = service_registry_redis_config self._registry = RedisServiceRegistry(**(self._registry_redis_config or {})) self.logger = logger if self.MOCK: return self._managed_services = {} for service in services: if isinstance(service, tuple) or isinstance(service, list): service_name = service[0] try: pool_size = int(service[1]) except IndexError: pool_size = self.DEFAULT_POOL_SIZE else: service_name = service pool_size = self.DEFAULT_POOL_SIZE self._managed_services[service_name] = [pool_size] for service_name, value in self._managed_services.items(): self._managed_services[service_name].append( self._create_service_pool(service_name, value[0]) ) self.log('debug', 'created service method caller') def _create_service_pool(self, service_name, pool_size): resources = [] service_configs = self._registry.discover_service(service_name, num=pool_size) for config in service_configs: for i in range(self.CLIENTS_PER_SERVICE_CONFIG): self.log('debug', 'creating %d client resource for service ' 'config: %s' % (i + 1, config)) resource = ServiceClientResource(service_name, config, self.logger) resources.append(resource) self.log('debug', 'created a pool of clients: %r' % resources) return ResourcePool(resources) def log(self, level, message): try: if not hasattr(self, 'logger'): return logger = self.logger if logger is None: return if not hasattr(logger, level): return logger_method = getattr(logger, level) if not logger_method: return logger_method(message) except: pass def __call__(self, method, service, request, response_class=None, timeout=DEFAULT_TIME_OUT, max_tries=DEFAULT_MAX_TRIES, sleep_before_retry=DEFAULT_SLEEP_BEFORE_RETRY): """ calls function 'method' on service 'service' :param method: :param service: :param request: :param response_class: :return: """ if self.MOCK: return if service not in self._managed_services: raise UnknownServiceError('service: %s unknown' % service) try: pool = self._managed_services[service][1] with pool.acquire(timeout=RESOURCE_ACQUIRING_TIMEOUT) as resource: client = resource.client self.log('debug', 'using client: %r' % client) if hasattr(request, 'SerializeToString'): request.header.request_guid = str(uuid.uuid4()) self.log('info', 'calling %s method on %s service ' 'with request guid: %s' % (method, service, request.header.request_guid)) request_message = request.SerializeToString() else: request_message = str(request) response = client.request(method, request_message, response_class=response_class, timeout=timeout, max_tries=max_tries, sleep_before_retry=sleep_before_retry) if hasattr(request, 'SerializeToString'): response_type = 'good' if response.header.success else 'bad' self.log('info', 'received %s response for %s method from %s ' 'service for request guid: %s in %s ' 'microseconds' % (response_type, method, service, response.header.request_guid, response.header.response_time)) else: self.log('info', 'received response: %s' % response) return response except Queue.Empty: self.log('error', 'no client to call method: %s on service: %s ' % (method, service)) raise ClientResourceNotAvailableError() except Exception as exception: import traceback self.log('error', 'Error while calling method: %s on ' 'service: %s. traceback: %s' % (method, service, traceback.format_exc())) raise exception
class ServiceClient(object): """ Base class to represent a client for a service """ def __init__(self, service_name, registry_redis_config=None, service_config=None, timeout=DEFAULT_TIME_OUT, sleep_before_retry=DEFAULT_SLEEP_BEFORE_RETRY, max_tries=DEFAULT_MAX_TRIES, heartbeat_frequency=DEFAULT_HEARTBEAT_FREQUENCY, start_heartbeat_thread=True, logger=None): self.logger = logger self._timeout = timeout self._sleep_before_retry = sleep_before_retry self._max_tries = max_tries self._heartbeat_frequency = heartbeat_frequency self._service_name = service_name if service_config: self._service_config = service_config else: self._registry = RedisServiceRegistry( **(registry_redis_config or {})) self._service_config = self._registry.discover_service( self._service_name)[0] self.start_time = current_timestamp() self.shutdown_time = None self.guid = str(uuid.uuid4()) self._context = zmq.Context() self._socket = socket_from_service_config(self._context, self._service_config, self._timeout) self.alive = True self.killed_by_error = None if start_heartbeat_thread: self._heartbeat_stop_event = threading.Event() heartbeat = ClientHeartbeat(self, self._service_config, self._timeout, self._heartbeat_frequency, self._max_tries) self._heartbeat_thread = threading.Thread( target=heartbeat, name='%s-client-heartbeat-%s' % (self._service_name, current_timestamp()), args=(self._heartbeat_stop_event, )) self._heartbeat_thread.start() else: self._heartbeat_thread = None def __repr__(self): return 'ServiceClient(guid=%s, service_name=%s, service_guid=%s)' % \ (self.guid, self._service_name, self._service_config['guid']) def log(self, level, message): try: if not hasattr(self, 'logger'): return logger = self.logger if logger is None: return if not hasattr(logger, level): return logger_method = getattr(logger, level) if not logger_method: return logger_method(message) except: pass def shutdown(self): if self._heartbeat_thread is None: return if self.shutdown_time is not None: return self.alive = False self._heartbeat_stop_event.set() self._heartbeat_thread.join() self.log('error', 'stopped heartbeat thread') self._socket.close() self._socket = None self.shutdown_time = current_timestamp() def _setup_socket(self, reuse=True, timeout=DEFAULT_TIME_OUT): if reuse and self._socket is not None: return self._socket = socket_from_service_config(self._context, self._service_config, timeout) def ping(self): return self.request('heartbeat', 'ping') def healthcheck(self): return self.request('healthcheck', 'health') def description(self): return self.request('description', 'description') def stop(self): return self.request('stop', 'stop') def request(self, function_name, request_message, response_class=None, timeout=DEFAULT_TIME_OUT, max_tries=DEFAULT_MAX_TRIES, sleep_before_retry=DEFAULT_SLEEP_BEFORE_RETRY): try_num = 0 sleep_duration = None error = None self._setup_socket(timeout=timeout) while self.alive and try_num < max_tries: if function_name not in self._service_config['functions']: raise ServiceFunctionNotAvailableError( '%r: function: %s not available for service: %s' % (self, function_name, self._service_name)) if sleep_duration: self.log('debug', 'Will try again in %s milliseconds' % sleep_duration) time.sleep(sleep_duration / 1000.0) self._setup_socket(reuse=False, timeout=timeout) try: self._socket.send_multipart( [str(function_name), request_message]) response_string = self._socket.recv() if response_class is None: return response_string else: response = response_class() response.ParseFromString(response_string) return response except zmq.error.Again: error = ServiceClientTimeoutError(self._service_name, function_name, timeout, max_tries, sleep_before_retry) self.log( 'debug', '%r can not complete function: %s of ' 'service: %s in %s milliseconds' % (self, function_name, self._service_name, timeout)) self._socket.close() self._socket = None sleep_duration = pow(2, try_num) * sleep_before_retry try_num += 1 except zmq.error.ZMQError as exception: error = ServiceClientError(exception) self.log( 'error', 'ZMQError in %r while requesting ' 'function: %s of ' 'service: %s. Error: %r' % (self, function_name, self._service_name, exception)) break except Exception as exception: error = ServiceClientError(exception) self.log( 'error', 'Error in %r while requesting ' 'function: %s of service: %s. Error: %r' % (self, function_name, self._service_name, exception)) break if not self.alive: self.log('error', '%r is no longer alive, shutting it ' 'down.' % self) self.shutdown() self.log('debug', 'heartbeat of %r shutdown' % self) if error: self.killed_by_error = error if isinstance(error, ServiceClientTimeoutError): self.log( 'error', '%r can not complete function: %s of ' 'service: %s in %s tries. Something must be ' 'wrong.' % (self, function_name, self._service_name, max_tries)) raise error raise ServiceClientError('%r Should never get here' % self)
class ServiceMethodCaller(object): """ This class maintains a pool of service clients. """ DEFAULT_POOL_SIZE = 5 CLIENTS_PER_SERVICE_CONFIG = 5 MOCK = False # this is for tests def __init__(self, service_registry_redis_config, services, logger=None): self._registry_redis_config = service_registry_redis_config self._registry = RedisServiceRegistry( **(self._registry_redis_config or {})) self.logger = logger if self.MOCK: return self._managed_services = {} for service in services: if isinstance(service, tuple) or isinstance(service, list): service_name = service[0] try: pool_size = int(service[1]) except IndexError: pool_size = self.DEFAULT_POOL_SIZE else: service_name = service pool_size = self.DEFAULT_POOL_SIZE self._managed_services[service_name] = [pool_size] for service_name, value in self._managed_services.items(): self._managed_services[service_name].append( self._create_service_pool(service_name, value[0])) self.log('debug', 'created service method caller') def _create_service_pool(self, service_name, pool_size): resources = [] service_configs = self._registry.discover_service(service_name, num=pool_size) for config in service_configs: for i in range(self.CLIENTS_PER_SERVICE_CONFIG): self.log( 'debug', 'creating %d client resource for service ' 'config: %s' % (i + 1, config)) resource = ServiceClientResource(service_name, config, self.logger) resources.append(resource) self.log('debug', 'created a pool of clients: %r' % resources) return ResourcePool(resources) def log(self, level, message): try: if not hasattr(self, 'logger'): return logger = self.logger if logger is None: return if not hasattr(logger, level): return logger_method = getattr(logger, level) if not logger_method: return logger_method(message) except: pass def __call__(self, method, service, request, response_class=None, timeout=DEFAULT_TIME_OUT, max_tries=DEFAULT_MAX_TRIES, sleep_before_retry=DEFAULT_SLEEP_BEFORE_RETRY): """ calls function 'method' on service 'service' :param method: :param service: :param request: :param response_class: :return: """ if self.MOCK: return if service not in self._managed_services: raise UnknownServiceError('service: %s unknown' % service) try: pool = self._managed_services[service][1] with pool.acquire(timeout=RESOURCE_ACQUIRING_TIMEOUT) as resource: client = resource.client self.log('debug', 'using client: %r' % client) if hasattr(request, 'SerializeToString'): request.header.request_guid = str(uuid.uuid4()) self.log( 'info', 'calling %s method on %s service ' 'with request guid: %s' % (method, service, request.header.request_guid)) request_message = request.SerializeToString() else: request_message = str(request) response = client.request( method, request_message, response_class=response_class, timeout=timeout, max_tries=max_tries, sleep_before_retry=sleep_before_retry) if hasattr(request, 'SerializeToString'): response_type = 'good' if response.header.success else 'bad' self.log( 'info', 'received %s response for %s method from %s ' 'service for request guid: %s in %s ' 'microseconds' % (response_type, method, service, response.header.request_guid, response.header.response_time)) else: self.log('info', 'received response: %s' % response) return response except Queue.Empty: self.log( 'error', 'no client to call method: %s on service: %s ' % (method, service)) raise ClientResourceNotAvailableError() except Exception as exception: import traceback self.log( 'error', 'Error while calling method: %s on ' 'service: %s. traceback: %s' % (method, service, traceback.format_exc())) raise exception
class ServiceClient(object): """ Base class to represent a client for a service """ def __init__(self, service_name, registry_redis_config=None, service_config=None, timeout=DEFAULT_TIME_OUT, sleep_before_retry=DEFAULT_SLEEP_BEFORE_RETRY, max_tries=DEFAULT_MAX_TRIES, heartbeat_frequency=DEFAULT_HEARTBEAT_FREQUENCY, start_heartbeat_thread=True, logger=None): self.logger = logger self._timeout = timeout self._sleep_before_retry = sleep_before_retry self._max_tries = max_tries self._heartbeat_frequency = heartbeat_frequency self._service_name = service_name if service_config: self._service_config = service_config else: self._registry = RedisServiceRegistry(**(registry_redis_config or {})) self._service_config = self._registry.discover_service( self._service_name)[0] self.start_time = current_timestamp() self.shutdown_time = None self.guid = str(uuid.uuid4()) self._context = zmq.Context() self._socket = socket_from_service_config(self._context, self._service_config, self._timeout) self.alive = True self.killed_by_error = None if start_heartbeat_thread: self._heartbeat_stop_event = threading.Event() heartbeat = ClientHeartbeat(self, self._service_config, self._timeout, self._heartbeat_frequency, self._max_tries) self._heartbeat_thread = threading.Thread( target=heartbeat, name='%s-client-heartbeat-%s' % (self._service_name, current_timestamp()), args=(self._heartbeat_stop_event, ) ) self._heartbeat_thread.start() else: self._heartbeat_thread = None def __repr__(self): return 'ServiceClient(guid=%s, service_name=%s, service_guid=%s)' % \ (self.guid, self._service_name, self._service_config['guid']) def log(self, level, message): try: if not hasattr(self, 'logger'): return logger = self.logger if logger is None: return if not hasattr(logger, level): return logger_method = getattr(logger, level) if not logger_method: return logger_method(message) except: pass def shutdown(self): if self._heartbeat_thread is None: return if self.shutdown_time is not None: return self.alive = False self._heartbeat_stop_event.set() self._heartbeat_thread.join() self.log('error', 'stopped heartbeat thread') self._socket.close() self._socket = None self.shutdown_time = current_timestamp() def _setup_socket(self, reuse=True, timeout=DEFAULT_TIME_OUT): if reuse and self._socket is not None: return self._socket = socket_from_service_config(self._context, self._service_config, timeout) def ping(self): return self.request('heartbeat', 'ping') def healthcheck(self): return self.request('healthcheck', 'health') def description(self): return self.request('description', 'description') def stop(self): return self.request('stop', 'stop') def request(self, function_name, request_message, response_class=None, timeout=DEFAULT_TIME_OUT, max_tries=DEFAULT_MAX_TRIES, sleep_before_retry=DEFAULT_SLEEP_BEFORE_RETRY): try_num = 0 sleep_duration = None error = None self._setup_socket(timeout=timeout) while self.alive and try_num < max_tries: if function_name not in self._service_config['functions']: raise ServiceFunctionNotAvailableError( '%r: function: %s not available for service: %s' % ( self, function_name, self._service_name )) if sleep_duration: self.log('debug', 'Will try again in %s milliseconds' % sleep_duration) time.sleep(sleep_duration/1000.0) self._setup_socket(reuse=False, timeout=timeout) try: self._socket.send_multipart( [str(function_name), request_message] ) response_string = self._socket.recv() if response_class is None: return response_string else: response = response_class() response.ParseFromString(response_string) return response except zmq.error.Again: error = ServiceClientTimeoutError(self._service_name, function_name, timeout, max_tries, sleep_before_retry) self.log('debug', '%r can not complete function: %s of ' 'service: %s in %s milliseconds' % (self, function_name, self._service_name, timeout)) self._socket.close() self._socket = None sleep_duration = pow(2, try_num) * sleep_before_retry try_num += 1 except zmq.error.ZMQError as exception: error = ServiceClientError(exception) self.log('error', 'ZMQError in %r while requesting ' 'function: %s of ' 'service: %s. Error: %r' % (self, function_name, self._service_name, exception)) break except Exception as exception: error = ServiceClientError(exception) self.log('error', 'Error in %r while requesting ' 'function: %s of service: %s. Error: %r' % (self, function_name, self._service_name, exception)) break if not self.alive: self.log('error', '%r is no longer alive, shutting it ' 'down.' % self) self.shutdown() self.log('debug', 'heartbeat of %r shutdown' % self) if error: self.killed_by_error = error if isinstance(error, ServiceClientTimeoutError): self.log('error', '%r can not complete function: %s of ' 'service: %s in %s tries. Something must be ' 'wrong.' % (self, function_name, self._service_name, max_tries)) raise error raise ServiceClientError('%r Should never get here' % self)