def _run(self): while True: try: # self.logger.debug("poller: %s", self._poller) socks = dict(self._poller.poll()) except KeyboardInterrupt as e: raise e except Exception as e: self.logger.error(e) raise e if self.socket in socks: function, request = self.socket.recv_multipart() function = function if function in self._message_handlers \ else 'default' self.function_deque.appendleft(function) if function != 'heartbeat': self.logger.debug("Received RPC for function: %s", function) self.stats['num_messages'] += 1 response_start_time = current_timestamp() try: response = self._message_handlers[function].handle(request) self.stats['num_success'] += 1 except Exception: response = 'empty response' self.stats['num_error'] += 1 import traceback self.log('error', 'Error while processing request for ' 'function: %s. Traceback: %s' % (function, traceback.format_exc())) response_processing_time = current_timestamp() - \ response_start_time self.stats['last_response_time'] = response_processing_time self.stats['max_response_time'] = max( self.stats['max_response_time'], response_processing_time ) if self.stats['min_response_time'] == 0: self.stats['min_response_time'] = response_processing_time else: self.stats['min_response_time'] = min( self.stats['min_response_time'], response_processing_time ) self.stats['avg_response_time'] = ( response_processing_time + ((self.stats['num_messages'] - 1) * self.stats[ 'avg_response_time']) )/float(self.stats['num_messages']) self.socket.send(response) if function == 'stop': raise StopServiceError()
def __init__(self, service_name, registry_redis_config=None, service_config=None, timeout=DEFAULT_TIME_OUT, sleep_before_retry=DEFAULT_SLEEP_BEFORE_RETRY, max_tries=DEFAULT_MAX_TRIES, heartbeat_frequency=DEFAULT_HEARTBEAT_FREQUENCY, start_heartbeat_thread=True, logger=None): self.logger = logger self._timeout = timeout self._sleep_before_retry = sleep_before_retry self._max_tries = max_tries self._heartbeat_frequency = heartbeat_frequency self._service_name = service_name if service_config: self._service_config = service_config else: self._registry = RedisServiceRegistry( **(registry_redis_config or {})) self._service_config = self._registry.discover_service( self._service_name)[0] self.start_time = current_timestamp() self.shutdown_time = None self.guid = str(uuid.uuid4()) self._context = zmq.Context() self._socket = socket_from_service_config(self._context, self._service_config, self._timeout) self.alive = True self.killed_by_error = None if start_heartbeat_thread: self._heartbeat_stop_event = threading.Event() heartbeat = ClientHeartbeat(self, self._service_config, self._timeout, self._heartbeat_frequency, self._max_tries) self._heartbeat_thread = threading.Thread( target=heartbeat, name='%s-client-heartbeat-%s' % (self._service_name, current_timestamp()), args=(self._heartbeat_stop_event, )) self._heartbeat_thread.start() else: self._heartbeat_thread = None
def __init__(self, service_name, registry_redis_config=None, service_config=None, timeout=DEFAULT_TIME_OUT, sleep_before_retry=DEFAULT_SLEEP_BEFORE_RETRY, max_tries=DEFAULT_MAX_TRIES, heartbeat_frequency=DEFAULT_HEARTBEAT_FREQUENCY, start_heartbeat_thread=True, logger=None): self.logger = logger self._timeout = timeout self._sleep_before_retry = sleep_before_retry self._max_tries = max_tries self._heartbeat_frequency = heartbeat_frequency self._service_name = service_name if service_config: self._service_config = service_config else: self._registry = RedisServiceRegistry(**(registry_redis_config or {})) self._service_config = self._registry.discover_service( self._service_name)[0] self.start_time = current_timestamp() self.shutdown_time = None self.guid = str(uuid.uuid4()) self._context = zmq.Context() self._socket = socket_from_service_config(self._context, self._service_config, self._timeout) self.alive = True self.killed_by_error = None if start_heartbeat_thread: self._heartbeat_stop_event = threading.Event() heartbeat = ClientHeartbeat(self, self._service_config, self._timeout, self._heartbeat_frequency, self._max_tries) self._heartbeat_thread = threading.Thread( target=heartbeat, name='%s-client-heartbeat-%s' % (self._service_name, current_timestamp()), args=(self._heartbeat_stop_event, ) ) self._heartbeat_thread.start() else: self._heartbeat_thread = None
def handle(self, message): request_start_time = current_timestamp() response = self.response_class() request_guid = None request_client = None try: try: request = self.request_class() request.ParseFromString(message) request_guid = request.header.request_guid request_client = request.header.client response.header.request_guid = request_guid except Exception as exception: raise BadServiceRequestError(exception) else: self._validate_request(request) self.log( 'info', '%s of %s service got request guid %s, ' 'from client: %s' % (self.__class__.__name__, self._service.name, request.header.request_guid, request.header.client)) self._handle(request, response) response.header.success = True self.log( 'debug', 'successfully processed request guid: %s, ' 'from client: %s' % (request.header.request_guid, request.header.client)) except Exception as exception: import traceback self.log( 'error', 'Error while handling request. Type: %s, ' 'Error: %r. Traceback: %s' % (exception.__class__.__name__, exception, traceback.format_exc())) self._response_from_exception(exception, response) finally: response.header.response_time = current_timestamp() - \ request_start_time self.log( 'info', '%s of %s service took %s microseconds to ' 'respond to request guid: %s, from client: %s' % (self.__class__.__name__, self._service.name, response.header.response_time, request_guid, request_client)) return response.SerializeToString()
def handle(self, message): request_start_time = current_timestamp() response = self.response_class() request_guid = None request_client = None try: try: request = self.request_class() request.ParseFromString(message) request_guid = request.header.request_guid request_client = request.header.client response.header.request_guid = request_guid except Exception as exception: raise BadServiceRequestError(exception) else: self._validate_request(request) self.log('info', '%s of %s service got request guid %s, ' 'from client: %s' % (self.__class__.__name__, self._service.name, request.header.request_guid, request.header.client)) self._handle(request, response) response.header.success = True self.log('debug', 'successfully processed request guid: %s, ' 'from client: %s' % (request.header.request_guid, request.header.client)) except Exception as exception: import traceback self.log('error', 'Error while handling request. Type: %s, ' 'Error: %r. Traceback: %s' % (exception.__class__.__name__, exception, traceback.format_exc())) self._response_from_exception(exception, response) finally: response.header.response_time = current_timestamp() - \ request_start_time self.log('info', '%s of %s service took %s microseconds to ' 'respond to request guid: %s, from client: %s' % (self.__class__.__name__, self._service.name, response.header.response_time, request_guid, request_client)) return response.SerializeToString()
def shutdown(self): if self._heartbeat_thread is None: return if self.shutdown_time is not None: return self.alive = False self._heartbeat_stop_event.set() self._heartbeat_thread.join() self.log('error', 'stopped heartbeat thread') self._socket.close() self._socket = None self.shutdown_time = current_timestamp()
def __call__(self, *args, **kwargs): stop_event = args[0] error = None try_num = 0 sleep_duration = None while not stop_event.is_set() and try_num < self._max_tries: error = None if sleep_duration: time.sleep(sleep_duration / 1000.0) try: # print '%r sending heartbeat' % self._client self._socket.send_multipart(['heartbeat', 'heartbeat']) self._socket.recv() sleep_duration = self._heartbeat_frequency except zmq.error.Again: error = ServiceClientTimeoutError() self._socket.close() self._socket = socket_from_service_config( self._context, self._service_config, self._timeout) sleep_duration = pow(2, try_num) * self._heartbeat_frequency try_num += 1 except zmq.error.ZMQError as exception: error = ServiceClientError(exception) break except Exception as exception: error = ServiceClientError(exception) break self._client.alive = False self._client.shutdown_time = current_timestamp() if error: self._client.killed_by_error = error self.log( 'error', 'heartbeat thread of client %r dying due ' 'to error: %r' % (self._client, error)) print '[%s] heartbeat of %r thread dying due to error: %r' % ( time.strftime('%Y-%m-%d %H-%M-%S', time.localtime()), self._client, error) else: self.log('debug', 'Stopping heartbeat thread of %r' % self._client) print '[%s] Stopping heartbeat thread of %r' % (time.strftime( '%Y-%m-%d %H-%M-%S', time.localtime()), self._client)
def __call__(self, *args, **kwargs): stop_event = args[0] error = None try_num = 0 sleep_duration = None while not stop_event.is_set() and try_num < self._max_tries: error = None if sleep_duration: time.sleep(sleep_duration/1000.0) try: # print '%r sending heartbeat' % self._client self._socket.send_multipart(['heartbeat', 'heartbeat']) self._socket.recv() sleep_duration = self._heartbeat_frequency except zmq.error.Again: error = ServiceClientTimeoutError() self._socket.close() self._socket = socket_from_service_config(self._context, self._service_config, self._timeout) sleep_duration = pow(2, try_num) * self._heartbeat_frequency try_num += 1 except zmq.error.ZMQError as exception: error = ServiceClientError(exception) break except Exception as exception: error = ServiceClientError(exception) break self._client.alive = False self._client.shutdown_time = current_timestamp() if error: self._client.killed_by_error = error self.log('error', 'heartbeat thread of client %r dying due ' 'to error: %r' % (self._client, error)) print '[%s] heartbeat of %r thread dying due to error: %r' % ( time.strftime('%Y-%m-%d %H-%M-%S', time.localtime()), self._client, error) else: self.log('debug', 'Stopping heartbeat thread of %r' % self._client) print '[%s] Stopping heartbeat thread of %r' % ( time.strftime('%Y-%m-%d %H-%M-%S', time.localtime()), self._client)
def _set_config(self, config_file=None): if not config_file: raise RuntimeError('A config file must be specified') logging.config.fileConfig(config_file) self.logger = logging.getLogger(self.__class__.__name__) self.config = ConfigParser.SafeConfigParser() self.config.read(config_file) self.description = self.config.get("global", "description") self.name = self.config.get("global", "name") self.env = self.config.get("global", "env") self.version = self.config.get("global", "version") self.start_time = current_timestamp() self.pid = os.getpid() self.proc = psutil.Process(self.pid) self.host = self.determine_host() self.guid = self.determine_guid() self.function_deque = collections.deque( maxlen=self.FUNCTIONS_DECK_LENGTH) self.stats = { 'num_messages': 0, 'num_success': 0, 'num_error': 0, 'avg_response_time': 0, 'min_response_time': 0, 'max_response_time': 0, 'last_response_time': 0 } self.pid_dir_path = self.PID_DIR try: self.pid_dir_path = self.config.get("global", "pid_dir") except ConfigParser.NoOptionError: pass self.pid_dir_path = "%s/%s" % (self.pid_dir_path, self.name) self.pid_file = "%s/%s" % (self.pid_dir_path, self.pid) self._registry = RedisServiceRegistry( **redis_config_from_config_file( self.config, "redis_service_registry", RedisServiceRegistry.DEFAULT_REDIS_CONFIG ) ) for k, v in self.DEFAULT_FUNCTION_MESSAGE_HANDLERS.items(): if k not in self.MESSAGE_HANDLERS: self.MESSAGE_HANDLERS[k] = v self.functions = self.MESSAGE_HANDLERS.keys() self._message_handlers = {} try: self._setup_sockets() self._setup_message_handlers() self._registry.register_service({ 'name': self.name, 'env': self.env, 'guid': self.guid, 'pid': self.pid, 'host': self.host, 'port': self.port, 'socket_type': self.socket_type, 'connect_method': self.connect_method, 'functions': json.dumps(self.functions), 'start_time': json.dumps(self.start_time), 'alive': json.dumps(True) }) except Exception as exception: import traceback self.log('error', 'Error while registering service: %s' % traceback.format_exc()) self._registry.deregister_service(self.name, self.guid, self.host) self.log('error', "%s while initializing %s service. Error: " "%r" % (exception.__class__.__name__, self.name, exception)) raise exception