def __init__(self, ez_props):
if ez_props is None:
raise Exception("Invalid EzProperties.")
zk_con_str = ZookeeperConfiguration(ez_props).getZookeeperConnectionString()
self.ezd_client = ServiceDiscoveryClient(zk_con_str)
self.__applicationConfiguration = ApplicationConfiguration(ez_props)
self.__applicationName = self.__applicationConfiguration.getApplicationName()
self.__securityConfiguration = SecurityConfiguration(ez_props)
self.__thriftConfiguration = ThriftConfiguration(ez_props)
self.__rLock = threading.RLock()
self.__serviceMap = {}
self.__clientMap = {}
self.__log = logging.getLogger(__name__)
if self.__applicationName is None:
self.__log.warn("No application name was found. Only common services will be discoverable.")
else:
self.__log.info("Application name: " + self.__applicationName)
try:
self.__common_services = list(self.ezd_client.get_common_services())
except Exception:
self.__log.error("Unable to get common services")
raise
self.__refresh_end_points()
self.__refresh_common_endpoints()
thread = threading.Thread(target=self._evict_daemon)
thread.setDaemon(True)
thread.start()
def ca_server(ezconfig, service_name=None, ca_name="ezbakeca", zoo_host=None,
host=None, port=None, verify_pattern=None, ssldir=None):
Crypto.Random.atfork()
# make sure zookeeper is available for registering with service discovery
if zoo_host is None:
zooConf = ZookeeperConfiguration(ezconfig)
zoo_host = zooConf.getZookeeperConnectionString()
if not zoo_host:
raise RuntimeError("Zookeeper connection string must be specified "
"in EzConfiguration")
# make sure the ssl certificate directory is available
if not ssldir:
ac = ApplicationConfiguration(ezconfig)
ssldir = ac.getCertificatesDir()
if not ssldir:
raise RuntimeError("Certificates Directory \"{0}\" must be set in"
" EzConfiguration!".format(
ApplicationConfiguration.CERTIFICATES_DIRECTORY_KEY))
# get a free port to bind to (and figure out our hostname)
if not port:
port = get_port(range(31005,34999))
if not host:
host = socket.gethostname()
# register with ezdiscovery
ezdiscovery = ServiceDiscoveryClient(zoo_host)
try:
if service_name is None:
service_name = ezbake.ezca.constants.SERVICE_NAME
logger.info('Registering with service discovery')
ezdiscovery.register_common_endpoint(service_name=service_name, host=host, port=port)
ezdiscovery.set_security_id_for_common_service(service_name=service_name, security_id="EzCAService")
except TimeoutError as e:
logger.error("Fatal timeout connecting to zookeeper. Unable to "
"register with service discovery.")
raise e
# create the thrift handler
handler = EzCAHandler(ca_name, ezconfig)
# generate/get the server SSL certs and write them to disk
certs = handler._server_certs()
cert_files = []
for k, cert in certs.items():
of = os.path.join(ssldir, k)
cert_files.append(of)
with os.fdopen(os.open(of, os.O_WRONLY | os.O_CREAT, 0o600), 'w') as ofs:
ofs.write(str(cert))
# generate certs for configured clients (read from ezconfig)
clients = ezconfig.get(EzCAHandler.CLIENT_CERTS)
if clients:
gen_client_certs(handler.ca, clients.split(','),
ezconfig.get(EzCAHandler.CLIENT_CERT_O))
# start the thrift server
processor = EzCA.Processor(handler)
transport = TSSLServerSocket(host=host, port=port,
verify_pattern=verify_pattern,
ca_certs=cert_files[0],
cert=cert_files[1],
key=cert_files[2])
tfactory = TTransport.TBufferedTransportFactory()
pfactory = TBinaryProtocol.TBinaryProtocolFactory()
server = TServer.TThreadPoolServer(processor, transport, tfactory, pfactory)
logger.info('Starting ezca service on {}:{}'.format(host,port))
server.serve()
class ThriftClientPool(object):
"""
"""
def __init__(self, ez_props):
if ez_props is None:
raise Exception("Invalid EzProperties.")
zk_con_str = ZookeeperConfiguration(ez_props).getZookeeperConnectionString()
self.ezd_client = ServiceDiscoveryClient(zk_con_str)
self.__applicationConfiguration = ApplicationConfiguration(ez_props)
self.__applicationName = self.__applicationConfiguration.getApplicationName()
self.__securityConfiguration = SecurityConfiguration(ez_props)
self.__thriftConfiguration = ThriftConfiguration(ez_props)
self.__rLock = threading.RLock()
self.__serviceMap = {}
self.__clientMap = {}
self.__log = logging.getLogger(__name__)
if self.__applicationName is None:
self.__log.warn("No application name was found. Only common services will be discoverable.")
else:
self.__log.info("Application name: " + self.__applicationName)
try:
self.__common_services = list(self.ezd_client.get_common_services())
except Exception:
self.__log.error("Unable to get common services")
raise
self.__refresh_end_points()
self.__refresh_common_endpoints()
thread = threading.Thread(target=self._evict_daemon)
thread.setDaemon(True)
thread.start()
def _evict_daemon(self):
check_interval_millis = self.__thriftConfiguration.getMillisBetweenClientEvictionChecks()
idle_threshold_millis = self.__thriftConfiguration.getMillisIdleBeforeEviction()
while True:
time.sleep(check_interval_millis * 0.001)
with self.__rLock:
for client in self.__clientMap.itervalues():
client._pool.evict_check(idle_threshold_millis)
def _get_service_map(self):
return self.__serviceMap
def _get_client_map(self):
return self.__clientMap
def __refresh_end_points(self):
if (self.__applicationName is not None) and (self.__applicationName != ''):
try:
for service in self.ezd_client.get_services(self.__applicationName):
try:
endpoints = self.ezd_client.get_endpoints(self.__applicationName, service)
self._add_endpoints(service, endpoints)
except Exception:
self.__log.warn("No " + service + " for application " + self.__applicationName + " was found")
except Exception:
self.__log.warn(
"Failed to get application services. "
"This might be okay if the application hasn't registered any services."
)
def __refresh_common_endpoints(self):
try:
for service in self.ezd_client.get_common_services():
try:
endpoints = self.ezd_client.get_common_endpoints(service)
self._add_endpoints(service, endpoints)
except Exception:
self.__log.warn("No common service " + service + " was found.")
except Exception:
self.__log.warn("Failed to get common services. This might be okay if no common service has been defined.")
def _add_endpoints(self, service, endpoints):
with self.__rLock:
if service in self.__serviceMap:
del self.__serviceMap[service]
self.__serviceMap[service] = []
for endpoint in endpoints:
self.__serviceMap[service].append(endpoint)
@staticmethod
def __get_thrift_connection_key(service_name, client_class):
return service_name + "|" + str(client_class)
def __get_endpoints(self, service_name, retry=True):
with self.__rLock:
if service_name in self.__serviceMap:
return self.__serviceMap[service_name]
if retry:
self.__refresh_end_points()
self.__refresh_common_endpoints()
return self.__get_endpoints(service_name, retry=False)
return None
def get_client(self, app_name=None, service_name=None, clazz=None):
if not service_name:
raise ValueError("'service_name' does not have a valid value (%s)." % service_name)
if not clazz:
raise ValueError("'clazz' does not have a valid value (%s)." % clazz)
try:
key = self.__get_thrift_connection_key(service_name, clazz)
with self.__rLock:
if app_name:
service = self.__applicationConfiguration.getApplicationServiceName(app_name, service_name)
if service not in self.__serviceMap:
endpoints = self.ezd_client.get_endpoints(app_name, service_name)
self._add_endpoints(service, endpoints)
# get client from client pool, or initialize client
if key in self.__clientMap:
client = self.__clientMap[key]
else:
endpoints = self.__get_endpoints(service_name)
if endpoints is None:
return None
pool_size = self.__thriftConfiguration.getMaxIdleClients()
if self.__thriftConfiguration.useSSL():
sc = self.__securityConfiguration
ca_certs = sc.getTrustedSslCerts()
key_file = sc.getPrivateKey()
cert = sc.getSslCertificate()
client = PoolingThriftClient(endpoints, clazz, pool_size=pool_size,
use_ssl=True, ca_certs=ca_certs, cert=cert, key=key_file)
else:
client = PoolingThriftClient(endpoints, clazz, pool_size=pool_size)
self.__clientMap[key] = client
return client
except Exception, e:
raise TException(str(e))
def ca_server(ezconfig, service_name=None, ca_name="ezbakeca", zoo_host=None,
host=None, port=None, verify_pattern=None, ssldir=None):
Crypto.Random.atfork()
# make sure zookeeper is available for registering with service discovery
if zoo_host is None:
zooConf = ZookeeperConfiguration(ezconfig)
zoo_host = zooConf.getZookeeperConnectionString()
if not zoo_host:
raise RuntimeError("Zookeeper connection string must be specified "
"in EzConfiguration")
# make sure the ssl certificate directory is available
if not ssldir:
ac = ApplicationConfiguration(ezconfig)
ssldir = ac.getCertificatesDir()
if not ssldir:
raise RuntimeError("Certificates Directory \"{0}\" must be set in"
" EzConfiguration!".format(
ApplicationConfiguration.CERTIFICATES_DIRECTORY_KEY))
# get a free port to bind to (and figure out our hostname)
if not port:
port = get_port(range(31005,34999))
if not host:
host = socket.gethostname()
# register with ezdiscovery
ezdiscovery = ServiceDiscoveryClient(zoo_host)
try:
if service_name is None:
service_name = ezbake.ezca.constants.SERVICE_NAME
logger.info('Registering with service discovery')
ezdiscovery.register_common_endpoint(service_name=service_name, host=host, port=port)
except TimeoutError as e:
logger.error("Fatal timeout connecting to zookeeper. Unable to "
"register with service discovery.")
raise e
# create the thrift handler
handler = EzCAHandler(ca_name, ezconfig)
# generate/get the server SSL certs and write them to disk
certs = handler._server_certs()
cert_files = []
for k, cert in certs.items():
of = os.path.join(ssldir, k)
cert_files.append(of)
with os.fdopen(os.open(of, os.O_WRONLY | os.O_CREAT, 0o600), 'w') as ofs:
ofs.write(str(cert))
# generate certs for configured clients (read from ezconfig)
clients = ezconfig.get(EzCAHandler.CLIENT_CERTS)
if clients:
gen_client_certs(handler.ca, clients.split(','),
ezconfig.get(EzCAHandler.CLIENT_CERT_O))
# start the thrift server
processor = EzCA.Processor(handler)
transport = TSSLServerSocket(host=host, port=port,
verify_pattern=verify_pattern,
ca_certs=cert_files[0],
cert=cert_files[1],
key=cert_files[2])
tfactory = TTransport.TBufferedTransportFactory()
pfactory = TBinaryProtocol.TBinaryProtocolFactory()
server = TServer.TThreadPoolServer(processor, transport, tfactory, pfactory)
logger.info('Starting ezca service on {}:{}'.format(host,port))
server.serve()
