def _run(self):
while True:
try:
# self.logger.debug("poller: %s", self._poller)
socks = dict(self._poller.poll())
except KeyboardInterrupt as e:
raise e
except Exception as e:
self.logger.error(e)
raise e
if self.socket in socks:
function, request = self.socket.recv_multipart()
function = function if function in self._message_handlers \
else 'default'
self.function_deque.appendleft(function)
if function != 'heartbeat':
self.logger.debug("Received RPC for function: %s", function)
self.stats['num_messages'] += 1
response_start_time = current_timestamp()
try:
response = self._message_handlers[function].handle(request)
self.stats['num_success'] += 1
except Exception:
response = 'empty response'
self.stats['num_error'] += 1
import traceback
self.log('error', 'Error while processing request for '
'function: %s. Traceback: %s' %
(function, traceback.format_exc()))
response_processing_time = current_timestamp() - \
response_start_time
self.stats['last_response_time'] = response_processing_time
self.stats['max_response_time'] = max(
self.stats['max_response_time'],
response_processing_time
)
if self.stats['min_response_time'] == 0:
self.stats['min_response_time'] = response_processing_time
else:
self.stats['min_response_time'] = min(
self.stats['min_response_time'],
response_processing_time
)
self.stats['avg_response_time'] = (
response_processing_time +
((self.stats['num_messages'] - 1) * self.stats[
'avg_response_time'])
)/float(self.stats['num_messages'])
self.socket.send(response)
if function == 'stop':
raise StopServiceError()
def __init__(self,
service_name,
registry_redis_config=None,
service_config=None,
timeout=DEFAULT_TIME_OUT,
sleep_before_retry=DEFAULT_SLEEP_BEFORE_RETRY,
max_tries=DEFAULT_MAX_TRIES,
heartbeat_frequency=DEFAULT_HEARTBEAT_FREQUENCY,
start_heartbeat_thread=True,
logger=None):
self.logger = logger
self._timeout = timeout
self._sleep_before_retry = sleep_before_retry
self._max_tries = max_tries
self._heartbeat_frequency = heartbeat_frequency
self._service_name = service_name
if service_config:
self._service_config = service_config
else:
self._registry = RedisServiceRegistry(
**(registry_redis_config or {}))
self._service_config = self._registry.discover_service(
self._service_name)[0]
self.start_time = current_timestamp()
self.shutdown_time = None
self.guid = str(uuid.uuid4())
self._context = zmq.Context()
self._socket = socket_from_service_config(self._context,
self._service_config,
self._timeout)
self.alive = True
self.killed_by_error = None
if start_heartbeat_thread:
self._heartbeat_stop_event = threading.Event()
heartbeat = ClientHeartbeat(self, self._service_config,
self._timeout,
self._heartbeat_frequency,
self._max_tries)
self._heartbeat_thread = threading.Thread(
target=heartbeat,
name='%s-client-heartbeat-%s' %
(self._service_name, current_timestamp()),
args=(self._heartbeat_stop_event, ))
self._heartbeat_thread.start()
else:
self._heartbeat_thread = None
def __init__(self, service_name,
registry_redis_config=None,
service_config=None,
timeout=DEFAULT_TIME_OUT,
sleep_before_retry=DEFAULT_SLEEP_BEFORE_RETRY,
max_tries=DEFAULT_MAX_TRIES,
heartbeat_frequency=DEFAULT_HEARTBEAT_FREQUENCY,
start_heartbeat_thread=True,
logger=None):
self.logger = logger
self._timeout = timeout
self._sleep_before_retry = sleep_before_retry
self._max_tries = max_tries
self._heartbeat_frequency = heartbeat_frequency
self._service_name = service_name
if service_config:
self._service_config = service_config
else:
self._registry = RedisServiceRegistry(**(registry_redis_config or
{}))
self._service_config = self._registry.discover_service(
self._service_name)[0]
self.start_time = current_timestamp()
self.shutdown_time = None
self.guid = str(uuid.uuid4())
self._context = zmq.Context()
self._socket = socket_from_service_config(self._context,
self._service_config,
self._timeout)
self.alive = True
self.killed_by_error = None
if start_heartbeat_thread:
self._heartbeat_stop_event = threading.Event()
heartbeat = ClientHeartbeat(self, self._service_config,
self._timeout, self._heartbeat_frequency,
self._max_tries)
self._heartbeat_thread = threading.Thread(
target=heartbeat,
name='%s-client-heartbeat-%s' % (self._service_name,
current_timestamp()),
args=(self._heartbeat_stop_event, )
)
self._heartbeat_thread.start()
else:
self._heartbeat_thread = None
def handle(self, message):
request_start_time = current_timestamp()
response = self.response_class()
request_guid = None
request_client = None
try:
try:
request = self.request_class()
request.ParseFromString(message)
request_guid = request.header.request_guid
request_client = request.header.client
response.header.request_guid = request_guid
except Exception as exception:
raise BadServiceRequestError(exception)
else:
self._validate_request(request)
self.log(
'info', '%s of %s service got request guid %s, '
'from client: %s' %
(self.__class__.__name__, self._service.name,
request.header.request_guid, request.header.client))
self._handle(request, response)
response.header.success = True
self.log(
'debug', 'successfully processed request guid: %s, '
'from client: %s' %
(request.header.request_guid, request.header.client))
except Exception as exception:
import traceback
self.log(
'error', 'Error while handling request. Type: %s, '
'Error: %r. Traceback: %s' %
(exception.__class__.__name__, exception,
traceback.format_exc()))
self._response_from_exception(exception, response)
finally:
response.header.response_time = current_timestamp() - \
request_start_time
self.log(
'info', '%s of %s service took %s microseconds to '
'respond to request guid: %s, from client: %s' %
(self.__class__.__name__, self._service.name,
response.header.response_time, request_guid, request_client))
return response.SerializeToString()
def handle(self, message):
request_start_time = current_timestamp()
response = self.response_class()
request_guid = None
request_client = None
try:
try:
request = self.request_class()
request.ParseFromString(message)
request_guid = request.header.request_guid
request_client = request.header.client
response.header.request_guid = request_guid
except Exception as exception:
raise BadServiceRequestError(exception)
else:
self._validate_request(request)
self.log('info', '%s of %s service got request guid %s, '
'from client: %s' %
(self.__class__.__name__, self._service.name,
request.header.request_guid,
request.header.client))
self._handle(request, response)
response.header.success = True
self.log('debug', 'successfully processed request guid: %s, '
'from client: %s' %
(request.header.request_guid,
request.header.client))
except Exception as exception:
import traceback
self.log('error', 'Error while handling request. Type: %s, '
'Error: %r. Traceback: %s' %
(exception.__class__.__name__, exception,
traceback.format_exc()))
self._response_from_exception(exception, response)
finally:
response.header.response_time = current_timestamp() - \
request_start_time
self.log('info', '%s of %s service took %s microseconds to '
'respond to request guid: %s, from client: %s' %
(self.__class__.__name__, self._service.name,
response.header.response_time, request_guid,
request_client))
return response.SerializeToString()
def shutdown(self):
if self._heartbeat_thread is None:
return
if self.shutdown_time is not None:
return
self.alive = False
self._heartbeat_stop_event.set()
self._heartbeat_thread.join()
self.log('error', 'stopped heartbeat thread')
self._socket.close()
self._socket = None
self.shutdown_time = current_timestamp()
def shutdown(self):
if self._heartbeat_thread is None:
return
if self.shutdown_time is not None:
return
self.alive = False
self._heartbeat_stop_event.set()
self._heartbeat_thread.join()
self.log('error', 'stopped heartbeat thread')
self._socket.close()
self._socket = None
self.shutdown_time = current_timestamp()
def __call__(self, *args, **kwargs):
stop_event = args[0]
error = None
try_num = 0
sleep_duration = None
while not stop_event.is_set() and try_num < self._max_tries:
error = None
if sleep_duration:
time.sleep(sleep_duration / 1000.0)
try:
# print '%r sending heartbeat' % self._client
self._socket.send_multipart(['heartbeat', 'heartbeat'])
self._socket.recv()
sleep_duration = self._heartbeat_frequency
except zmq.error.Again:
error = ServiceClientTimeoutError()
self._socket.close()
self._socket = socket_from_service_config(
self._context, self._service_config, self._timeout)
sleep_duration = pow(2, try_num) * self._heartbeat_frequency
try_num += 1
except zmq.error.ZMQError as exception:
error = ServiceClientError(exception)
break
except Exception as exception:
error = ServiceClientError(exception)
break
self._client.alive = False
self._client.shutdown_time = current_timestamp()
if error:
self._client.killed_by_error = error
self.log(
'error', 'heartbeat thread of client %r dying due '
'to error: %r' % (self._client, error))
print '[%s] heartbeat of %r thread dying due to error: %r' % (
time.strftime('%Y-%m-%d %H-%M-%S',
time.localtime()), self._client, error)
else:
self.log('debug', 'Stopping heartbeat thread of %r' % self._client)
print '[%s] Stopping heartbeat thread of %r' % (time.strftime(
'%Y-%m-%d %H-%M-%S', time.localtime()), self._client)
def __call__(self, *args, **kwargs):
stop_event = args[0]
error = None
try_num = 0
sleep_duration = None
while not stop_event.is_set() and try_num < self._max_tries:
error = None
if sleep_duration:
time.sleep(sleep_duration/1000.0)
try:
# print '%r sending heartbeat' % self._client
self._socket.send_multipart(['heartbeat', 'heartbeat'])
self._socket.recv()
sleep_duration = self._heartbeat_frequency
except zmq.error.Again:
error = ServiceClientTimeoutError()
self._socket.close()
self._socket = socket_from_service_config(self._context,
self._service_config,
self._timeout)
sleep_duration = pow(2, try_num) * self._heartbeat_frequency
try_num += 1
except zmq.error.ZMQError as exception:
error = ServiceClientError(exception)
break
except Exception as exception:
error = ServiceClientError(exception)
break
self._client.alive = False
self._client.shutdown_time = current_timestamp()
if error:
self._client.killed_by_error = error
self.log('error', 'heartbeat thread of client %r dying due '
'to error: %r' % (self._client, error))
print '[%s] heartbeat of %r thread dying due to error: %r' % (
time.strftime('%Y-%m-%d %H-%M-%S', time.localtime()),
self._client, error)
else:
self.log('debug', 'Stopping heartbeat thread of %r' % self._client)
print '[%s] Stopping heartbeat thread of %r' % (
time.strftime('%Y-%m-%d %H-%M-%S', time.localtime()), self._client)
def _set_config(self, config_file=None):
if not config_file:
raise RuntimeError('A config file must be specified')
logging.config.fileConfig(config_file)
self.logger = logging.getLogger(self.__class__.__name__)
self.config = ConfigParser.SafeConfigParser()
self.config.read(config_file)
self.description = self.config.get("global", "description")
self.name = self.config.get("global", "name")
self.env = self.config.get("global", "env")
self.version = self.config.get("global", "version")
self.start_time = current_timestamp()
self.pid = os.getpid()
self.proc = psutil.Process(self.pid)
self.host = self.determine_host()
self.guid = self.determine_guid()
self.function_deque = collections.deque(
maxlen=self.FUNCTIONS_DECK_LENGTH)
self.stats = {
'num_messages': 0,
'num_success': 0,
'num_error': 0,
'avg_response_time': 0,
'min_response_time': 0,
'max_response_time': 0,
'last_response_time': 0
}
self.pid_dir_path = self.PID_DIR
try:
self.pid_dir_path = self.config.get("global", "pid_dir")
except ConfigParser.NoOptionError:
pass
self.pid_dir_path = "%s/%s" % (self.pid_dir_path, self.name)
self.pid_file = "%s/%s" % (self.pid_dir_path, self.pid)
self._registry = RedisServiceRegistry(
**redis_config_from_config_file(
self.config, "redis_service_registry",
RedisServiceRegistry.DEFAULT_REDIS_CONFIG
)
)
for k, v in self.DEFAULT_FUNCTION_MESSAGE_HANDLERS.items():
if k not in self.MESSAGE_HANDLERS:
self.MESSAGE_HANDLERS[k] = v
self.functions = self.MESSAGE_HANDLERS.keys()
self._message_handlers = {}
try:
self._setup_sockets()
self._setup_message_handlers()
self._registry.register_service({
'name': self.name,
'env': self.env,
'guid': self.guid,
'pid': self.pid,
'host': self.host,
'port': self.port,
'socket_type': self.socket_type,
'connect_method': self.connect_method,
'functions': json.dumps(self.functions),
'start_time': json.dumps(self.start_time),
'alive': json.dumps(True)
})
except Exception as exception:
import traceback
self.log('error', 'Error while registering service: %s' %
traceback.format_exc())
self._registry.deregister_service(self.name, self.guid, self.host)
self.log('error', "%s while initializing %s service. Error: "
"%r" % (exception.__class__.__name__, self.name,
exception))
raise exception
