def __init__(self, service_registry_redis_config, services, logger=None):
self._registry_redis_config = service_registry_redis_config
self._registry = RedisServiceRegistry(
**(self._registry_redis_config or {}))
self.logger = logger
if self.MOCK:
return
self._managed_services = {}
for service in services:
if isinstance(service, tuple) or isinstance(service, list):
service_name = service[0]
try:
pool_size = int(service[1])
except IndexError:
pool_size = self.DEFAULT_POOL_SIZE
else:
service_name = service
pool_size = self.DEFAULT_POOL_SIZE
self._managed_services[service_name] = [pool_size]
for service_name, value in self._managed_services.items():
self._managed_services[service_name].append(
self._create_service_pool(service_name, value[0]))
self.log('debug', 'created service method caller')
def __init__(self,
service_name,
registry_redis_config=None,
service_config=None,
timeout=DEFAULT_TIME_OUT,
sleep_before_retry=DEFAULT_SLEEP_BEFORE_RETRY,
max_tries=DEFAULT_MAX_TRIES,
heartbeat_frequency=DEFAULT_HEARTBEAT_FREQUENCY,
start_heartbeat_thread=True,
logger=None):
self.logger = logger
self._timeout = timeout
self._sleep_before_retry = sleep_before_retry
self._max_tries = max_tries
self._heartbeat_frequency = heartbeat_frequency
self._service_name = service_name
if service_config:
self._service_config = service_config
else:
self._registry = RedisServiceRegistry(
**(registry_redis_config or {}))
self._service_config = self._registry.discover_service(
self._service_name)[0]
self.start_time = current_timestamp()
self.shutdown_time = None
self.guid = str(uuid.uuid4())
self._context = zmq.Context()
self._socket = socket_from_service_config(self._context,
self._service_config,
self._timeout)
self.alive = True
self.killed_by_error = None
if start_heartbeat_thread:
self._heartbeat_stop_event = threading.Event()
heartbeat = ClientHeartbeat(self, self._service_config,
self._timeout,
self._heartbeat_frequency,
self._max_tries)
self._heartbeat_thread = threading.Thread(
target=heartbeat,
name='%s-client-heartbeat-%s' %
(self._service_name, current_timestamp()),
args=(self._heartbeat_stop_event, ))
self._heartbeat_thread.start()
else:
self._heartbeat_thread = None
def __init__(self, service_registry_redis_config, services,
logger=None):
self._registry_redis_config = service_registry_redis_config
self._registry = RedisServiceRegistry(**(self._registry_redis_config
or {}))
self.logger = logger
if self.MOCK:
return
self._managed_services = {}
for service in services:
if isinstance(service, tuple) or isinstance(service, list):
service_name = service[0]
try:
pool_size = int(service[1])
except IndexError:
pool_size = self.DEFAULT_POOL_SIZE
else:
service_name = service
pool_size = self.DEFAULT_POOL_SIZE
self._managed_services[service_name] = [pool_size]
for service_name, value in self._managed_services.items():
self._managed_services[service_name].append(
self._create_service_pool(service_name, value[0])
)
self.log('debug', 'created service method caller')
def __init__(self, service_name,
registry_redis_config=None,
service_config=None,
timeout=DEFAULT_TIME_OUT,
sleep_before_retry=DEFAULT_SLEEP_BEFORE_RETRY,
max_tries=DEFAULT_MAX_TRIES,
heartbeat_frequency=DEFAULT_HEARTBEAT_FREQUENCY,
start_heartbeat_thread=True,
logger=None):
self.logger = logger
self._timeout = timeout
self._sleep_before_retry = sleep_before_retry
self._max_tries = max_tries
self._heartbeat_frequency = heartbeat_frequency
self._service_name = service_name
if service_config:
self._service_config = service_config
else:
self._registry = RedisServiceRegistry(**(registry_redis_config or
{}))
self._service_config = self._registry.discover_service(
self._service_name)[0]
self.start_time = current_timestamp()
self.shutdown_time = None
self.guid = str(uuid.uuid4())
self._context = zmq.Context()
self._socket = socket_from_service_config(self._context,
self._service_config,
self._timeout)
self.alive = True
self.killed_by_error = None
if start_heartbeat_thread:
self._heartbeat_stop_event = threading.Event()
heartbeat = ClientHeartbeat(self, self._service_config,
self._timeout, self._heartbeat_frequency,
self._max_tries)
self._heartbeat_thread = threading.Thread(
target=heartbeat,
name='%s-client-heartbeat-%s' % (self._service_name,
current_timestamp()),
args=(self._heartbeat_stop_event, )
)
self._heartbeat_thread.start()
else:
self._heartbeat_thread = None
class Service(object):
"""
"""
config = None
VALID_CONN_METHOD = {"bind", "connect"}
VALID_SCK_TYPES = {"REQ", "REP", "PUB", "SUB", "PUSH", "PULL"}
MESSAGE_HANDLERS = {}
CONFIG_REDIS_SECTION = "config_redis"
BASE_TCP_ADDR = 'tcp://%s:%d'
DEFAULT_FUNCTIONS = ["heartbeat", "healthcheck", "description", "stop"]
DEFAULT_FUNCTION_MESSAGE_HANDLERS = {
"heartbeat": HeartbeatHandler,
"healthcheck": HealthCheckHandler,
"description": DescriptionHandler,
"stop": StopServiceHandler,
"default": DefaultMessageHandler
}
EC2_INSTANCE_HOSTNAME_URL = \
"http://169.254.169.254/latest/meta-data/public-hostname"
EC2_METADATA_REQUEST_TIMEOUT = 1
PID_DIR = "/home/ec2-user/publishing_services"
FUNCTIONS_DECK_LENGTH = 10
def __repr__(self):
return "%s(name=%s, host=%s, guid=%s, pid=%s, description=%s, " \
"functions=[%s])" % \
(self.__class__.__name__, self.name, self.host, self.guid,
self.pid, self.description, ", ".join(self.functions))
def log(self, level, message):
try:
if not hasattr(self, 'logger'):
return
logger = self.logger
if logger is None:
return
if not hasattr(logger, level):
return
logger_method = getattr(logger, level)
if not logger_method:
return
logger_method(message)
except:
pass
def determine_host(self):
try:
r = requests.get(self.EC2_INSTANCE_HOSTNAME_URL,
timeout=self.EC2_METADATA_REQUEST_TIMEOUT)
if r.status_code == requests.codes.OKAY:
return r.content.strip()
except Exception as exception:
self.log('debug', 'looks like not running on an EC2 instance. '
'Trying to determine host from config file. '
'Error: %r' % exception)
try:
return self.config.get("global", "host")
except Exception as exception:
self.log('error', 'can not determine hostname, returning '
'localhost. error: %r' % exception)
return "localhost"
def determine_guid(self):
return hashlib.md5(
", ".join([
"%s service" % self.name,
self.host,
"%s" % self.pid,
"%s" % self.start_time,
"%s" % random.randint(1, 1000000000)
])
).hexdigest()
def _set_config(self, config_file=None):
if not config_file:
raise RuntimeError('A config file must be specified')
logging.config.fileConfig(config_file)
self.logger = logging.getLogger(self.__class__.__name__)
self.config = ConfigParser.SafeConfigParser()
self.config.read(config_file)
self.description = self.config.get("global", "description")
self.name = self.config.get("global", "name")
self.env = self.config.get("global", "env")
self.version = self.config.get("global", "version")
self.start_time = current_timestamp()
self.pid = os.getpid()
self.proc = psutil.Process(self.pid)
self.host = self.determine_host()
self.guid = self.determine_guid()
self.function_deque = collections.deque(
maxlen=self.FUNCTIONS_DECK_LENGTH)
self.stats = {
'num_messages': 0,
'num_success': 0,
'num_error': 0,
'avg_response_time': 0,
'min_response_time': 0,
'max_response_time': 0,
'last_response_time': 0
}
self.pid_dir_path = self.PID_DIR
try:
self.pid_dir_path = self.config.get("global", "pid_dir")
except ConfigParser.NoOptionError:
pass
self.pid_dir_path = "%s/%s" % (self.pid_dir_path, self.name)
self.pid_file = "%s/%s" % (self.pid_dir_path, self.pid)
self._registry = RedisServiceRegistry(
**redis_config_from_config_file(
self.config, "redis_service_registry",
RedisServiceRegistry.DEFAULT_REDIS_CONFIG
)
)
for k, v in self.DEFAULT_FUNCTION_MESSAGE_HANDLERS.items():
if k not in self.MESSAGE_HANDLERS:
self.MESSAGE_HANDLERS[k] = v
self.functions = self.MESSAGE_HANDLERS.keys()
self._message_handlers = {}
try:
self._setup_sockets()
self._setup_message_handlers()
self._registry.register_service({
'name': self.name,
'env': self.env,
'guid': self.guid,
'pid': self.pid,
'host': self.host,
'port': self.port,
'socket_type': self.socket_type,
'connect_method': self.connect_method,
'functions': json.dumps(self.functions),
'start_time': json.dumps(self.start_time),
'alive': json.dumps(True)
})
except Exception as exception:
import traceback
self.log('error', 'Error while registering service: %s' %
traceback.format_exc())
self._registry.deregister_service(self.name, self.guid, self.host)
self.log('error', "%s while initializing %s service. Error: "
"%r" % (exception.__class__.__name__, self.name,
exception))
raise exception
def _setup_sockets(self):
self._poller = zmq.Poller()
self._ports = {}
self._context = zmq.Context()
self.port, self.socket = self._get_socket_for_service()
self._poller.register(self.socket, zmq.POLLIN)
def _get_socket_for_service(self):
port = self._registry.next_available_port(self.name, self.guid,
self.host)
self.socket_type = "REP"
self.connect_method = "bind"
try:
self.socket_type = self.config.get("global", "socket_type").upper()
self.connect_method = self.config.get("global",
"connect_method").lower()
except ConfigParser.NoSectionError:
pass
except ConfigParser.NoOptionError:
pass
if self.socket_type not in self.VALID_SCK_TYPES:
raise RuntimeError(
"Socket type %s not in set [%s] of valid socket types" %
(self.connect_method, ", ".join(self.VALID_SCK_TYPES)))
if self.connect_method not in self.VALID_CONN_METHOD:
raise RuntimeError(
"Connect method %s not in set [%s] of valid methods" %
(self.connect_method, ", ".join(self.VALID_CONN_METHOD)))
connect_string = self.BASE_TCP_ADDR % (
"*" if self.connect_method == "bind" else self.host, port
)
socket = zmq_socket_from_socket_type(self._context, self.socket_type)
getattr(socket, self.connect_method)(connect_string)
return port, socket
def _setup_message_handlers(self):
for function, handler_class in self.MESSAGE_HANDLERS.items():
self._message_handlers[function] = handler_class(
self, function, self.socket, logger=self.logger
)
self.logger.debug("Registered handler: service: %s, function: %s, "
"handler: %s", self.name, function,
handler_class.__name__)
def _run(self):
while True:
try:
# self.logger.debug("poller: %s", self._poller)
socks = dict(self._poller.poll())
except KeyboardInterrupt as e:
raise e
except Exception as e:
self.logger.error(e)
raise e
if self.socket in socks:
function, request = self.socket.recv_multipart()
function = function if function in self._message_handlers \
else 'default'
self.function_deque.appendleft(function)
if function != 'heartbeat':
self.logger.debug("Received RPC for function: %s", function)
self.stats['num_messages'] += 1
response_start_time = current_timestamp()
try:
response = self._message_handlers[function].handle(request)
self.stats['num_success'] += 1
except Exception:
response = 'empty response'
self.stats['num_error'] += 1
import traceback
self.log('error', 'Error while processing request for '
'function: %s. Traceback: %s' %
(function, traceback.format_exc()))
response_processing_time = current_timestamp() - \
response_start_time
self.stats['last_response_time'] = response_processing_time
self.stats['max_response_time'] = max(
self.stats['max_response_time'],
response_processing_time
)
if self.stats['min_response_time'] == 0:
self.stats['min_response_time'] = response_processing_time
else:
self.stats['min_response_time'] = min(
self.stats['min_response_time'],
response_processing_time
)
self.stats['avg_response_time'] = (
response_processing_time +
((self.stats['num_messages'] - 1) * self.stats[
'avg_response_time'])
)/float(self.stats['num_messages'])
self.socket.send(response)
if function == 'stop':
raise StopServiceError()
def run(self):
if not self.config:
raise RuntimeError('A config file must be specified')
try:
self._run()
except StopServiceError:
self.logger.debug("Stopping :%s service in response to STOP "
"message.")
except KeyboardInterrupt as e:
self.log('debug', 'KeyboardInterrupt while running %s service' %
self.name)
except Exception as exception:
self.logger.error("Service crashed due to %s. args: [%s], message: "
"%s", exception.__class__.__name__, ", ".join(
exception.args), str(exception))
finally:
self._registry.deregister_service(self.name, self.guid, self.host)
try:
os.remove(self.pid_file)
except:
pass
self.logger.debug("Deregistered %s service", self.name)
self.logger.debug("%s service stopped", self.name)
@classmethod
def get_cmd_line_parser(cls):
argparser = argparse.ArgumentParser(
description="User management service")
argparser.add_argument("-c", "--config_file", required=True,
help="config file")
return argparser
def run_service(self):
set_time_zone()
parser = self.get_cmd_line_parser()
args = parser.parse_args()
self._set_config(args.config_file)
if not os.path.exists(self.pid_dir_path):
os.makedirs(self.pid_dir_path)
f = open(self.pid_file, 'w')
f.write(json.dumps({
'name': self.name,
'env': self.env,
'pid': self.pid,
'guid': self.guid,
'host': self.host,
'port': self.port,
'socket_type': self.socket_type,
'connect_method': self.connect_method,
'functions': self.functions,
'start_time': self.start_time,
'cmdline': self.proc.cmdline()
}, indent=4))
f.close()
print 'Running %r' % self
self.run()
def _set_config(self, config_file=None):
if not config_file:
raise RuntimeError('A config file must be specified')
logging.config.fileConfig(config_file)
self.logger = logging.getLogger(self.__class__.__name__)
self.config = ConfigParser.SafeConfigParser()
self.config.read(config_file)
self.description = self.config.get("global", "description")
self.name = self.config.get("global", "name")
self.env = self.config.get("global", "env")
self.version = self.config.get("global", "version")
self.start_time = current_timestamp()
self.pid = os.getpid()
self.proc = psutil.Process(self.pid)
self.host = self.determine_host()
self.guid = self.determine_guid()
self.function_deque = collections.deque(
maxlen=self.FUNCTIONS_DECK_LENGTH)
self.stats = {
'num_messages': 0,
'num_success': 0,
'num_error': 0,
'avg_response_time': 0,
'min_response_time': 0,
'max_response_time': 0,
'last_response_time': 0
}
self.pid_dir_path = self.PID_DIR
try:
self.pid_dir_path = self.config.get("global", "pid_dir")
except ConfigParser.NoOptionError:
pass
self.pid_dir_path = "%s/%s" % (self.pid_dir_path, self.name)
self.pid_file = "%s/%s" % (self.pid_dir_path, self.pid)
self._registry = RedisServiceRegistry(
**redis_config_from_config_file(
self.config, "redis_service_registry",
RedisServiceRegistry.DEFAULT_REDIS_CONFIG
)
)
for k, v in self.DEFAULT_FUNCTION_MESSAGE_HANDLERS.items():
if k not in self.MESSAGE_HANDLERS:
self.MESSAGE_HANDLERS[k] = v
self.functions = self.MESSAGE_HANDLERS.keys()
self._message_handlers = {}
try:
self._setup_sockets()
self._setup_message_handlers()
self._registry.register_service({
'name': self.name,
'env': self.env,
'guid': self.guid,
'pid': self.pid,
'host': self.host,
'port': self.port,
'socket_type': self.socket_type,
'connect_method': self.connect_method,
'functions': json.dumps(self.functions),
'start_time': json.dumps(self.start_time),
'alive': json.dumps(True)
})
except Exception as exception:
import traceback
self.log('error', 'Error while registering service: %s' %
traceback.format_exc())
self._registry.deregister_service(self.name, self.guid, self.host)
self.log('error', "%s while initializing %s service. Error: "
"%r" % (exception.__class__.__name__, self.name,
exception))
raise exception
class ServiceMethodCaller(object):
"""
This class maintains a pool of service clients.
"""
DEFAULT_POOL_SIZE = 5
CLIENTS_PER_SERVICE_CONFIG = 5
MOCK = False # this is for tests
def __init__(self, service_registry_redis_config, services,
logger=None):
self._registry_redis_config = service_registry_redis_config
self._registry = RedisServiceRegistry(**(self._registry_redis_config
or {}))
self.logger = logger
if self.MOCK:
return
self._managed_services = {}
for service in services:
if isinstance(service, tuple) or isinstance(service, list):
service_name = service[0]
try:
pool_size = int(service[1])
except IndexError:
pool_size = self.DEFAULT_POOL_SIZE
else:
service_name = service
pool_size = self.DEFAULT_POOL_SIZE
self._managed_services[service_name] = [pool_size]
for service_name, value in self._managed_services.items():
self._managed_services[service_name].append(
self._create_service_pool(service_name, value[0])
)
self.log('debug', 'created service method caller')
def _create_service_pool(self, service_name, pool_size):
resources = []
service_configs = self._registry.discover_service(service_name,
num=pool_size)
for config in service_configs:
for i in range(self.CLIENTS_PER_SERVICE_CONFIG):
self.log('debug', 'creating %d client resource for service '
'config: %s' % (i + 1, config))
resource = ServiceClientResource(service_name, config,
self.logger)
resources.append(resource)
self.log('debug', 'created a pool of clients: %r' % resources)
return ResourcePool(resources)
def log(self, level, message):
try:
if not hasattr(self, 'logger'):
return
logger = self.logger
if logger is None:
return
if not hasattr(logger, level):
return
logger_method = getattr(logger, level)
if not logger_method:
return
logger_method(message)
except:
pass
def __call__(self, method, service, request, response_class=None,
timeout=DEFAULT_TIME_OUT, max_tries=DEFAULT_MAX_TRIES,
sleep_before_retry=DEFAULT_SLEEP_BEFORE_RETRY):
"""
calls function 'method' on service 'service'
:param method:
:param service:
:param request:
:param response_class:
:return:
"""
if self.MOCK:
return
if service not in self._managed_services:
raise UnknownServiceError('service: %s unknown' % service)
try:
pool = self._managed_services[service][1]
with pool.acquire(timeout=RESOURCE_ACQUIRING_TIMEOUT) as resource:
client = resource.client
self.log('debug', 'using client: %r' % client)
if hasattr(request, 'SerializeToString'):
request.header.request_guid = str(uuid.uuid4())
self.log('info', 'calling %s method on %s service '
'with request guid: %s' %
(method, service,
request.header.request_guid))
request_message = request.SerializeToString()
else:
request_message = str(request)
response = client.request(method, request_message,
response_class=response_class,
timeout=timeout, max_tries=max_tries,
sleep_before_retry=sleep_before_retry)
if hasattr(request, 'SerializeToString'):
response_type = 'good' if response.header.success else 'bad'
self.log('info', 'received %s response for %s method from %s '
'service for request guid: %s in %s '
'microseconds' %
(response_type, method, service,
response.header.request_guid,
response.header.response_time))
else:
self.log('info', 'received response: %s' % response)
return response
except Queue.Empty:
self.log('error', 'no client to call method: %s on service: %s '
% (method, service))
raise ClientResourceNotAvailableError()
except Exception as exception:
import traceback
self.log('error', 'Error while calling method: %s on '
'service: %s. traceback: %s' %
(method, service, traceback.format_exc()))
raise exception
class ServiceClient(object):
"""
Base class to represent a client for a service
"""
def __init__(self,
service_name,
registry_redis_config=None,
service_config=None,
timeout=DEFAULT_TIME_OUT,
sleep_before_retry=DEFAULT_SLEEP_BEFORE_RETRY,
max_tries=DEFAULT_MAX_TRIES,
heartbeat_frequency=DEFAULT_HEARTBEAT_FREQUENCY,
start_heartbeat_thread=True,
logger=None):
self.logger = logger
self._timeout = timeout
self._sleep_before_retry = sleep_before_retry
self._max_tries = max_tries
self._heartbeat_frequency = heartbeat_frequency
self._service_name = service_name
if service_config:
self._service_config = service_config
else:
self._registry = RedisServiceRegistry(
**(registry_redis_config or {}))
self._service_config = self._registry.discover_service(
self._service_name)[0]
self.start_time = current_timestamp()
self.shutdown_time = None
self.guid = str(uuid.uuid4())
self._context = zmq.Context()
self._socket = socket_from_service_config(self._context,
self._service_config,
self._timeout)
self.alive = True
self.killed_by_error = None
if start_heartbeat_thread:
self._heartbeat_stop_event = threading.Event()
heartbeat = ClientHeartbeat(self, self._service_config,
self._timeout,
self._heartbeat_frequency,
self._max_tries)
self._heartbeat_thread = threading.Thread(
target=heartbeat,
name='%s-client-heartbeat-%s' %
(self._service_name, current_timestamp()),
args=(self._heartbeat_stop_event, ))
self._heartbeat_thread.start()
else:
self._heartbeat_thread = None
def __repr__(self):
return 'ServiceClient(guid=%s, service_name=%s, service_guid=%s)' % \
(self.guid, self._service_name, self._service_config['guid'])
def log(self, level, message):
try:
if not hasattr(self, 'logger'):
return
logger = self.logger
if logger is None:
return
if not hasattr(logger, level):
return
logger_method = getattr(logger, level)
if not logger_method:
return
logger_method(message)
except:
pass
def shutdown(self):
if self._heartbeat_thread is None:
return
if self.shutdown_time is not None:
return
self.alive = False
self._heartbeat_stop_event.set()
self._heartbeat_thread.join()
self.log('error', 'stopped heartbeat thread')
self._socket.close()
self._socket = None
self.shutdown_time = current_timestamp()
def _setup_socket(self, reuse=True, timeout=DEFAULT_TIME_OUT):
if reuse and self._socket is not None:
return
self._socket = socket_from_service_config(self._context,
self._service_config,
timeout)
def ping(self):
return self.request('heartbeat', 'ping')
def healthcheck(self):
return self.request('healthcheck', 'health')
def description(self):
return self.request('description', 'description')
def stop(self):
return self.request('stop', 'stop')
def request(self,
function_name,
request_message,
response_class=None,
timeout=DEFAULT_TIME_OUT,
max_tries=DEFAULT_MAX_TRIES,
sleep_before_retry=DEFAULT_SLEEP_BEFORE_RETRY):
try_num = 0
sleep_duration = None
error = None
self._setup_socket(timeout=timeout)
while self.alive and try_num < max_tries:
if function_name not in self._service_config['functions']:
raise ServiceFunctionNotAvailableError(
'%r: function: %s not available for service: %s' %
(self, function_name, self._service_name))
if sleep_duration:
self.log('debug',
'Will try again in %s milliseconds' % sleep_duration)
time.sleep(sleep_duration / 1000.0)
self._setup_socket(reuse=False, timeout=timeout)
try:
self._socket.send_multipart(
[str(function_name), request_message])
response_string = self._socket.recv()
if response_class is None:
return response_string
else:
response = response_class()
response.ParseFromString(response_string)
return response
except zmq.error.Again:
error = ServiceClientTimeoutError(self._service_name,
function_name, timeout,
max_tries,
sleep_before_retry)
self.log(
'debug', '%r can not complete function: %s of '
'service: %s in %s milliseconds' %
(self, function_name, self._service_name, timeout))
self._socket.close()
self._socket = None
sleep_duration = pow(2, try_num) * sleep_before_retry
try_num += 1
except zmq.error.ZMQError as exception:
error = ServiceClientError(exception)
self.log(
'error', 'ZMQError in %r while requesting '
'function: %s of '
'service: %s. Error: %r' %
(self, function_name, self._service_name, exception))
break
except Exception as exception:
error = ServiceClientError(exception)
self.log(
'error', 'Error in %r while requesting '
'function: %s of service: %s. Error: %r' %
(self, function_name, self._service_name, exception))
break
if not self.alive:
self.log('error', '%r is no longer alive, shutting it '
'down.' % self)
self.shutdown()
self.log('debug', 'heartbeat of %r shutdown' % self)
if error:
self.killed_by_error = error
if isinstance(error, ServiceClientTimeoutError):
self.log(
'error', '%r can not complete function: %s of '
'service: %s in %s tries. Something must be '
'wrong.' %
(self, function_name, self._service_name, max_tries))
raise error
raise ServiceClientError('%r Should never get here' % self)
class ServiceMethodCaller(object):
"""
This class maintains a pool of service clients.
"""
DEFAULT_POOL_SIZE = 5
CLIENTS_PER_SERVICE_CONFIG = 5
MOCK = False # this is for tests
def __init__(self, service_registry_redis_config, services, logger=None):
self._registry_redis_config = service_registry_redis_config
self._registry = RedisServiceRegistry(
**(self._registry_redis_config or {}))
self.logger = logger
if self.MOCK:
return
self._managed_services = {}
for service in services:
if isinstance(service, tuple) or isinstance(service, list):
service_name = service[0]
try:
pool_size = int(service[1])
except IndexError:
pool_size = self.DEFAULT_POOL_SIZE
else:
service_name = service
pool_size = self.DEFAULT_POOL_SIZE
self._managed_services[service_name] = [pool_size]
for service_name, value in self._managed_services.items():
self._managed_services[service_name].append(
self._create_service_pool(service_name, value[0]))
self.log('debug', 'created service method caller')
def _create_service_pool(self, service_name, pool_size):
resources = []
service_configs = self._registry.discover_service(service_name,
num=pool_size)
for config in service_configs:
for i in range(self.CLIENTS_PER_SERVICE_CONFIG):
self.log(
'debug', 'creating %d client resource for service '
'config: %s' % (i + 1, config))
resource = ServiceClientResource(service_name, config,
self.logger)
resources.append(resource)
self.log('debug', 'created a pool of clients: %r' % resources)
return ResourcePool(resources)
def log(self, level, message):
try:
if not hasattr(self, 'logger'):
return
logger = self.logger
if logger is None:
return
if not hasattr(logger, level):
return
logger_method = getattr(logger, level)
if not logger_method:
return
logger_method(message)
except:
pass
def __call__(self,
method,
service,
request,
response_class=None,
timeout=DEFAULT_TIME_OUT,
max_tries=DEFAULT_MAX_TRIES,
sleep_before_retry=DEFAULT_SLEEP_BEFORE_RETRY):
"""
calls function 'method' on service 'service'
:param method:
:param service:
:param request:
:param response_class:
:return:
"""
if self.MOCK:
return
if service not in self._managed_services:
raise UnknownServiceError('service: %s unknown' % service)
try:
pool = self._managed_services[service][1]
with pool.acquire(timeout=RESOURCE_ACQUIRING_TIMEOUT) as resource:
client = resource.client
self.log('debug', 'using client: %r' % client)
if hasattr(request, 'SerializeToString'):
request.header.request_guid = str(uuid.uuid4())
self.log(
'info', 'calling %s method on %s service '
'with request guid: %s' %
(method, service, request.header.request_guid))
request_message = request.SerializeToString()
else:
request_message = str(request)
response = client.request(
method,
request_message,
response_class=response_class,
timeout=timeout,
max_tries=max_tries,
sleep_before_retry=sleep_before_retry)
if hasattr(request, 'SerializeToString'):
response_type = 'good' if response.header.success else 'bad'
self.log(
'info', 'received %s response for %s method from %s '
'service for request guid: %s in %s '
'microseconds' % (response_type, method, service,
response.header.request_guid,
response.header.response_time))
else:
self.log('info', 'received response: %s' % response)
return response
except Queue.Empty:
self.log(
'error', 'no client to call method: %s on service: %s ' %
(method, service))
raise ClientResourceNotAvailableError()
except Exception as exception:
import traceback
self.log(
'error', 'Error while calling method: %s on '
'service: %s. traceback: %s' %
(method, service, traceback.format_exc()))
raise exception
class ServiceClient(object):
"""
Base class to represent a client for a service
"""
def __init__(self, service_name,
registry_redis_config=None,
service_config=None,
timeout=DEFAULT_TIME_OUT,
sleep_before_retry=DEFAULT_SLEEP_BEFORE_RETRY,
max_tries=DEFAULT_MAX_TRIES,
heartbeat_frequency=DEFAULT_HEARTBEAT_FREQUENCY,
start_heartbeat_thread=True,
logger=None):
self.logger = logger
self._timeout = timeout
self._sleep_before_retry = sleep_before_retry
self._max_tries = max_tries
self._heartbeat_frequency = heartbeat_frequency
self._service_name = service_name
if service_config:
self._service_config = service_config
else:
self._registry = RedisServiceRegistry(**(registry_redis_config or
{}))
self._service_config = self._registry.discover_service(
self._service_name)[0]
self.start_time = current_timestamp()
self.shutdown_time = None
self.guid = str(uuid.uuid4())
self._context = zmq.Context()
self._socket = socket_from_service_config(self._context,
self._service_config,
self._timeout)
self.alive = True
self.killed_by_error = None
if start_heartbeat_thread:
self._heartbeat_stop_event = threading.Event()
heartbeat = ClientHeartbeat(self, self._service_config,
self._timeout, self._heartbeat_frequency,
self._max_tries)
self._heartbeat_thread = threading.Thread(
target=heartbeat,
name='%s-client-heartbeat-%s' % (self._service_name,
current_timestamp()),
args=(self._heartbeat_stop_event, )
)
self._heartbeat_thread.start()
else:
self._heartbeat_thread = None
def __repr__(self):
return 'ServiceClient(guid=%s, service_name=%s, service_guid=%s)' % \
(self.guid, self._service_name, self._service_config['guid'])
def log(self, level, message):
try:
if not hasattr(self, 'logger'):
return
logger = self.logger
if logger is None:
return
if not hasattr(logger, level):
return
logger_method = getattr(logger, level)
if not logger_method:
return
logger_method(message)
except:
pass
def shutdown(self):
if self._heartbeat_thread is None:
return
if self.shutdown_time is not None:
return
self.alive = False
self._heartbeat_stop_event.set()
self._heartbeat_thread.join()
self.log('error', 'stopped heartbeat thread')
self._socket.close()
self._socket = None
self.shutdown_time = current_timestamp()
def _setup_socket(self, reuse=True, timeout=DEFAULT_TIME_OUT):
if reuse and self._socket is not None:
return
self._socket = socket_from_service_config(self._context,
self._service_config, timeout)
def ping(self):
return self.request('heartbeat', 'ping')
def healthcheck(self):
return self.request('healthcheck', 'health')
def description(self):
return self.request('description', 'description')
def stop(self):
return self.request('stop', 'stop')
def request(self, function_name, request_message, response_class=None,
timeout=DEFAULT_TIME_OUT, max_tries=DEFAULT_MAX_TRIES,
sleep_before_retry=DEFAULT_SLEEP_BEFORE_RETRY):
try_num = 0
sleep_duration = None
error = None
self._setup_socket(timeout=timeout)
while self.alive and try_num < max_tries:
if function_name not in self._service_config['functions']:
raise ServiceFunctionNotAvailableError(
'%r: function: %s not available for service: %s' % (
self, function_name, self._service_name
))
if sleep_duration:
self.log('debug', 'Will try again in %s milliseconds' %
sleep_duration)
time.sleep(sleep_duration/1000.0)
self._setup_socket(reuse=False, timeout=timeout)
try:
self._socket.send_multipart(
[str(function_name), request_message]
)
response_string = self._socket.recv()
if response_class is None:
return response_string
else:
response = response_class()
response.ParseFromString(response_string)
return response
except zmq.error.Again:
error = ServiceClientTimeoutError(self._service_name,
function_name, timeout,
max_tries,
sleep_before_retry)
self.log('debug', '%r can not complete function: %s of '
'service: %s in %s milliseconds' %
(self, function_name, self._service_name, timeout))
self._socket.close()
self._socket = None
sleep_duration = pow(2, try_num) * sleep_before_retry
try_num += 1
except zmq.error.ZMQError as exception:
error = ServiceClientError(exception)
self.log('error', 'ZMQError in %r while requesting '
'function: %s of '
'service: %s. Error: %r' %
(self, function_name, self._service_name, exception))
break
except Exception as exception:
error = ServiceClientError(exception)
self.log('error', 'Error in %r while requesting '
'function: %s of service: %s. Error: %r' %
(self, function_name, self._service_name, exception))
break
if not self.alive:
self.log('error', '%r is no longer alive, shutting it '
'down.' % self)
self.shutdown()
self.log('debug', 'heartbeat of %r shutdown' % self)
if error:
self.killed_by_error = error
if isinstance(error, ServiceClientTimeoutError):
self.log('error', '%r can not complete function: %s of '
'service: %s in %s tries. Something must be '
'wrong.' % (self, function_name,
self._service_name, max_tries))
raise error
raise ServiceClientError('%r Should never get here' % self)
