def setUpClass(cls):
logging.basicConfig(level=logging.DEBUG)
cls.zookeeper_client = GZookeeperClient(["localdev:2181"])
cls.zookeeper_client.start()
gevent.sleep(1)
cls.watch_path = "/unittest/childrenwatch"
cls.children = [("child1", "child1data"), ("child2", "child2data")]
if cls.zookeeper_client.exists(cls.watch_path):
for child, child_data in cls.children:
cls.zookeeper_client.delete(os.path.join(
cls.watch_path, child))
cls.zookeeper_client.delete(cls.watch_path)
gevent.sleep(1)
cls.watch_observer_count = 0
cls.watch_observer_children = None
def watch_observer(watch):
cls.watch_observer_count += 1
cls.watch_observer_children = cls.watch.get_children()
cls.watch = GChildrenWatch(cls.zookeeper_client, cls.watch_path,
watch_observer)
cls.watch.start()
gevent.sleep(1)
def setUpClass(cls):
logging.basicConfig(level=logging.DEBUG)
cls.zookeeper_client = GZookeeperClient(["localdev:2181"])
cls.zookeeper_client.start()
gevent.sleep(1)
cls.watch_path = "/unittest/datawatch"
cls.watch_data = "unittest_data"
if cls.zookeeper_client.exists(cls.watch_path):
cls.zookeeper_client.delete(cls.watch_path)
cls.zookeeper_client.create(cls.watch_path, cls.watch_data)
gevent.sleep(1)
cls.watch_observer_count = 0
cls.watch_observer_data = None
def watch_observer(watch):
cls.watch_observer_count += 1
cls.watch_observer_data = cls.watch.get_data()
cls.watch = GDataWatch(cls.zookeeper_client, cls.watch_path,
watch_observer)
cls.watch.start()
gevent.sleep(1)
def __init__(self, service, zookeeper_hosts,
database_connection=None, database_connection_pool_size=5):
"""GServiceHandler constructor.
Args:
service: Service object. Note that this will not be fully initialized
until start() is called. It may be neccessary delay some handler
instantiation until then.
zookeeper_hosts: list of zookeeper hosts, i.e. ["localhost:2181", "localdev:2181"]
database_connection: optional database connection string
"""
self.service = service
self.options = {}
self.counters = BasicCounters(0)
self.running = False
#Zookeeper client
self.zookeeper_client = GZookeeperClient(zookeeper_hosts)
#Database session factory
if database_connection:
#Make psycogp2 driver compatible with gevent
from trpycore import psycopg2_gevent
from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker
#Note that max_overflow must be set to -1
#in order to avoid deadlocks in SQLAlchemy's
#QueuePool. QueuePool use's a threading.Lock
#when max_overflow is not set to -1, which
#will result in a deadlock when multiple
#connections are created from a single thread.
engine = create_engine(
database_connection,
pool_size=database_connection_pool_size,
max_overflow=-1)
self.DatabaseSession = sessionmaker(bind=engine)
else:
self.DatabaseSession = None
#Registrar
self.registrar = ZookeeperServiceRegistrar(self.zookeeper_client)
#Add counter decorator to track service method calls
def counter_decorator(func):
requests_counter = self.counters.get_counter("requests")
open_requests_counter = self.counters.get_counter("open_requests")
def wrapper(*args, **kwargs):
try:
requests_counter.increment()
open_requests_counter.increment()
return func(*args, **kwargs)
finally:
open_requests_counter.decrement()
return wrapper
self._decorate_service_methods(counter_decorator)
def setUpClass(cls):
logging.basicConfig(level=logging.DEBUG)
cls.zookeeper_client = GZookeeperClient(["localdev:2181"])
cls.zookeeper_client.start()
gevent.sleep(1)
cls.service = UnittestService()
cls.request_context = RequestContext(userId=0,
impersonatingUserId=0,
sessionId="dummy_session_id",
context="")
def expire_zookeeper_client_session(client, timeout=10):
"""Expire zookeeper session for the given client.
This method should only be used for testing purposed.
It will induce an EXPIRED_SESSION_STATE event in
given client.
Args:
client: GZookeeperClient object
timeout: optional timeout in seconds
to wait for the session to expire.
If None, this call will block.
This is not recommended.
Returns:
True if session exipration occured within
timeout seconds, False otherwise.
"""
#session expiration event to wait on
session_expiration_event = Event()
def observer(event):
if event.state_name == "EXPIRED_SESSION_STATE":
session_expiration_event.set()
client.add_session_observer(observer)
#construct new client with same session_id
#so we can cause a session expiration event
#in our other client.
zookeeper_client = GZookeeperClient(
client.servers,
client.session_id,
client.session_password)
def zookeeper_observer(event):
#Upon connection, immediately stop the client
#which will cause a session expiration in
#self.zookeeper_client.
if event.state_name == "CONNECTED_STATE":
zookeeper_client.stop()
zookeeper_client.add_session_observer(zookeeper_observer)
zookeeper_client.start()
zookeeper_client.join()
session_expiration_event.wait(timeout)
client.remove_session_observer(observer)
return session_expiration_event.is_set()
def setUpClass(cls):
logging.basicConfig(level=logging.DEBUG)
cls.service = ApiService("localhost", 9090)
cls.service.start()
gevent.sleep(1)
cls.service_name = SERVICE_NAME
cls.service_class = TApiService
cls.zookeeper_client = GZookeeperClient(["localdev:2181"])
cls.zookeeper_client.start()
gevent.sleep(1)
cls.service_proxy = ZookeeperServiceProxy(cls.zookeeper_client, cls.service_name, cls.service_class, keepalive=True)
cls.request_context = RequestContext(userId=0, impersonatingUserId=0, sessionId="dummy_session_id", context="")
def setUpClass(cls):
logging.basicConfig(level=logging.DEBUG)
cls.zookeeper_client = GZookeeperClient(["localdev:2181"])
cls.zookeeper_client.start()
gevent.sleep(1)
cls.hashring_path = "/unittest/hashring"
cls.hashring_data = "unittest_data"
cls.hashring_positions = [
0xcfcd208495d565ef66e7dff9f98764da,
0xf899139df5e1059396431415e770c6dd, 0x0
]
cls.hashring_watch = GHashringWatch(cls.zookeeper_client,
cls.hashring_path,
positions=cls.hashring_positions,
position_data=cls.hashring_data)
cls.hashring_watch.start()
gevent.sleep(1)
def setUpClass(cls):
logging.basicConfig(level=logging.DEBUG)
cls.zookeeper_client = GZookeeperClient(["localdev:2181"])
cls.zookeeper_client.start()
gevent.sleep(1)
class GServiceHandler(TRService.Iface, Handler):
"""Gevent Service handler base class.
This class is intended to be subclassed by concrete
service handlers.
"""
def __init__(self, service, zookeeper_hosts,
database_connection=None, database_connection_pool_size=5):
"""GServiceHandler constructor.
Args:
service: Service object. Note that this will not be fully initialized
until start() is called. It may be neccessary delay some handler
instantiation until then.
zookeeper_hosts: list of zookeeper hosts, i.e. ["localhost:2181", "localdev:2181"]
database_connection: optional database connection string
"""
self.service = service
self.options = {}
self.counters = BasicCounters(0)
self.running = False
#Zookeeper client
self.zookeeper_client = GZookeeperClient(zookeeper_hosts)
#Database session factory
if database_connection:
#Make psycogp2 driver compatible with gevent
from trpycore import psycopg2_gevent
from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker
#Note that max_overflow must be set to -1
#in order to avoid deadlocks in SQLAlchemy's
#QueuePool. QueuePool use's a threading.Lock
#when max_overflow is not set to -1, which
#will result in a deadlock when multiple
#connections are created from a single thread.
engine = create_engine(
database_connection,
pool_size=database_connection_pool_size,
max_overflow=-1)
self.DatabaseSession = sessionmaker(bind=engine)
else:
self.DatabaseSession = None
#Registrar
self.registrar = ZookeeperServiceRegistrar(self.zookeeper_client)
#Add counter decorator to track service method calls
def counter_decorator(func):
requests_counter = self.counters.get_counter("requests")
open_requests_counter = self.counters.get_counter("open_requests")
def wrapper(*args, **kwargs):
try:
requests_counter.increment()
open_requests_counter.increment()
return func(*args, **kwargs)
finally:
open_requests_counter.decrement()
return wrapper
self._decorate_service_methods(counter_decorator)
def _decorate_service_methods(self, decorator, cls=None, decorated=None):
"""Decorate service interface methods at runtime.
This method will decorate the service instance methods on the
current handler with the given decorator. Service interface
methods are identified as any callable on a base class
with the name "Iface".
This method should be used judicially to limit the magic.
The main advantage of this method is that decorators can
be added to all service methods from a base class,
without specific knowledge of each service's interface.
Derived classes should NOT add decorators using this method.
"""
cls = cls or self.__class__
decorated = decorated if decorated is not None else {}
if cls.__name__ == "Iface":
for attribute_name in cls.__dict__:
attribute = getattr(self, attribute_name)
if callable(attribute) and attribute_name not in decorated:
setattr(self, attribute_name, decorator(attribute))
decorated[attribute_name] = True
for base_class in cls.__bases__:
self._decorate_service_methods(decorator, base_class, decorated)
def start(self):
"""Start service handler."""
if not self.running:
self.running = True
self.zookeeper_client.start()
self.registrar.register_service(self.service)
def join(self, timeout=None):
"""Join service handler.
Join the handler, waiting for the completion of all threads
or greenlets.
Args:
timeout: Optional timeout in seconds to observe before returning.
If timeout is specified, the status() method must be called
to determine if the handler is still running.
"""
self.zookeeper_client.join(timeout)
def stop(self):
"""Stop service handler."""
if self.running:
self.running = False
self.zookeeper_client.stop()
def status(self):
"""Get the handler status.
Returns Status enum.
"""
if self.running:
return Status.ALIVE
else:
return Status.STOPPED
def get_database_session(self, **kwargs):
"""Return new database SQLAlchemy database session.
Returns:
new SQLAlchemy session
Raises:
RuntimeError: If database_connection not provided to handler.
"""
if self.DatabaseSession:
return self.DatabaseSession(**kwargs)
else:
raise RuntimeError("database_connection not provided")
def getName(self, requestContext):
"""Get service name.
Args:
requestContext: RequestContext object containing user information.
Returns:
service name (string)
"""
return self.service.name()
def getVersion(self, requestContext):
"""Get service version.
Args:
requestContext: RequestContext object containing user information.
Returns:
service version (string)
"""
return self.service.version()
def getBuildNumber(self, requestContext):
"""Get service build number.
Args:
requestContext: RequestContext object containing user information.
Returns:
service build number (string)
"""
return self.service.build() or ""
def getStatus(self, requestContext):
"""Get service status.
Args:
requestContext: RequestContext object containing user information.
Returns:
Status constant
"""
return self.service.status()
def getStatusDetails(self, requestContext):
"""Get service status details.
Args:
requestContext: RequestContext object containing user information.
Returns:
String description of the current Status constant.
"""
if self.running:
return "Alive and well"
else:
return "Dead"
def getCounter(self, requestContext, key):
"""Get service counter.
Args:
requestContext: RequestContext object containing user information.
key: counter name
Returns:
counter value
"""
if key in self.counters:
return self.counters[key]
else:
return -1
def getCounters(self, requestContext):
"""Get service counters.
Args:
requestContext: RequestContext object containing user information.
Returns:
Dict of service specific counters.
"""
return self.counters.as_dict()
def getOption(self, requestContext, key):
"""Get service option.
Args:
requestContext: RequestContext object containing user information.
key: Option name
Returns:
String value for the option.
"""
if key in self.options:
return self.options[key]
else:
return "invalid option"
def getOptions(self, requestContext):
"""Get all service options.
Args:
requestContext: RequestContext object containing user information.
Returns:
Dict of service specific options key / values.
"""
return self.options
def setOption(self, requestContext, key, value):
"""Set service options.
Args:
requestContext: RequestContext object containing user information.
key: Option name (string)
value: Option value
"""
self.options[key] = value
def shutdown(self, requestContext):
"""Shutdown service.
Args:
requestContext: RequestContext object containing user information.
"""
self.service.stop()
def reinitialize(self, requestContext):
"""Reinitialize service.
Args:
requestContext: RequestContext object containing user information.
"""
pass
import settings from trpycore.zookeeper_gevent.client import GZookeeperClient zookeeper_client = GZookeeperClient(settings.ZOOKEEPER_HOSTS) zookeeper_client.start()
class GServiceHandler(TRService.Iface, Handler):
"""Gevent Service handler base class.
This class is intended to be subclassed by concrete
service handlers.
"""
def __init__(self,
service,
zookeeper_hosts,
database_connection=None,
database_connection_pool_size=5):
"""GServiceHandler constructor.
Args:
service: Service object. Note that this will not be fully initialized
until start() is called. It may be neccessary delay some handler
instantiation until then.
zookeeper_hosts: list of zookeeper hosts, i.e. ["localhost:2181", "localdev:2181"]
database_connection: optional database connection string
"""
self.service = service
self.options = {}
self.counters = BasicCounters(0)
self.running = False
#Zookeeper client
self.zookeeper_client = GZookeeperClient(zookeeper_hosts)
#Database session factory
if database_connection:
#Make psycogp2 driver compatible with gevent
from trpycore import psycopg2_gevent
from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker
#Note that max_overflow must be set to -1
#in order to avoid deadlocks in SQLAlchemy's
#QueuePool. QueuePool use's a threading.Lock
#when max_overflow is not set to -1, which
#will result in a deadlock when multiple
#connections are created from a single thread.
engine = create_engine(database_connection,
pool_size=database_connection_pool_size,
max_overflow=-1)
self.DatabaseSession = sessionmaker(bind=engine)
else:
self.DatabaseSession = None
#Registrar
self.registrar = ZookeeperServiceRegistrar(self.zookeeper_client)
#Add counter decorator to track service method calls
def counter_decorator(func):
requests_counter = self.counters.get_counter("requests")
open_requests_counter = self.counters.get_counter("open_requests")
def wrapper(*args, **kwargs):
try:
requests_counter.increment()
open_requests_counter.increment()
return func(*args, **kwargs)
finally:
open_requests_counter.decrement()
return wrapper
self._decorate_service_methods(counter_decorator)
def _decorate_service_methods(self, decorator, cls=None, decorated=None):
"""Decorate service interface methods at runtime.
This method will decorate the service instance methods on the
current handler with the given decorator. Service interface
methods are identified as any callable on a base class
with the name "Iface".
This method should be used judicially to limit the magic.
The main advantage of this method is that decorators can
be added to all service methods from a base class,
without specific knowledge of each service's interface.
Derived classes should NOT add decorators using this method.
"""
cls = cls or self.__class__
decorated = decorated if decorated is not None else {}
if cls.__name__ == "Iface":
for attribute_name in cls.__dict__:
attribute = getattr(self, attribute_name)
if callable(attribute) and attribute_name not in decorated:
setattr(self, attribute_name, decorator(attribute))
decorated[attribute_name] = True
for base_class in cls.__bases__:
self._decorate_service_methods(decorator, base_class, decorated)
def start(self):
"""Start service handler."""
if not self.running:
self.running = True
self.zookeeper_client.start()
self.registrar.register_service(self.service)
def join(self, timeout=None):
"""Join service handler.
Join the handler, waiting for the completion of all threads
or greenlets.
Args:
timeout: Optional timeout in seconds to observe before returning.
If timeout is specified, the status() method must be called
to determine if the handler is still running.
"""
self.zookeeper_client.join(timeout)
def stop(self):
"""Stop service handler."""
if self.running:
self.running = False
self.zookeeper_client.stop()
def status(self):
"""Get the handler status.
Returns Status enum.
"""
if self.running:
return Status.ALIVE
else:
return Status.STOPPED
def get_database_session(self, **kwargs):
"""Return new database SQLAlchemy database session.
Returns:
new SQLAlchemy session
Raises:
RuntimeError: If database_connection not provided to handler.
"""
if self.DatabaseSession:
return self.DatabaseSession(**kwargs)
else:
raise RuntimeError("database_connection not provided")
def getName(self, requestContext):
"""Get service name.
Args:
requestContext: RequestContext object containing user information.
Returns:
service name (string)
"""
return self.service.name()
def getVersion(self, requestContext):
"""Get service version.
Args:
requestContext: RequestContext object containing user information.
Returns:
service version (string)
"""
return self.service.version()
def getBuildNumber(self, requestContext):
"""Get service build number.
Args:
requestContext: RequestContext object containing user information.
Returns:
service build number (string)
"""
return self.service.build() or ""
def getStatus(self, requestContext):
"""Get service status.
Args:
requestContext: RequestContext object containing user information.
Returns:
Status constant
"""
return self.service.status()
def getStatusDetails(self, requestContext):
"""Get service status details.
Args:
requestContext: RequestContext object containing user information.
Returns:
String description of the current Status constant.
"""
if self.running:
return "Alive and well"
else:
return "Dead"
def getCounter(self, requestContext, key):
"""Get service counter.
Args:
requestContext: RequestContext object containing user information.
key: counter name
Returns:
counter value
"""
if key in self.counters:
return self.counters[key]
else:
return -1
def getCounters(self, requestContext):
"""Get service counters.
Args:
requestContext: RequestContext object containing user information.
Returns:
Dict of service specific counters.
"""
return self.counters.as_dict()
def getOption(self, requestContext, key):
"""Get service option.
Args:
requestContext: RequestContext object containing user information.
key: Option name
Returns:
String value for the option.
"""
if key in self.options:
return self.options[key]
else:
return "invalid option"
def getOptions(self, requestContext):
"""Get all service options.
Args:
requestContext: RequestContext object containing user information.
Returns:
Dict of service specific options key / values.
"""
return self.options
def setOption(self, requestContext, key, value):
"""Set service options.
Args:
requestContext: RequestContext object containing user information.
key: Option name (string)
value: Option value
"""
self.options[key] = value
def shutdown(self, requestContext):
"""Shutdown service.
Args:
requestContext: RequestContext object containing user information.
"""
self.service.stop()
def reinitialize(self, requestContext):
"""Reinitialize service.
Args:
requestContext: RequestContext object containing user information.
"""
pass
def __init__(self,
service,
zookeeper_hosts,
database_connection=None,
database_connection_pool_size=5):
"""GServiceHandler constructor.
Args:
service: Service object. Note that this will not be fully initialized
until start() is called. It may be neccessary delay some handler
instantiation until then.
zookeeper_hosts: list of zookeeper hosts, i.e. ["localhost:2181", "localdev:2181"]
database_connection: optional database connection string
"""
self.service = service
self.options = {}
self.counters = BasicCounters(0)
self.running = False
#Zookeeper client
self.zookeeper_client = GZookeeperClient(zookeeper_hosts)
#Database session factory
if database_connection:
#Make psycogp2 driver compatible with gevent
from trpycore import psycopg2_gevent
from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker
#Note that max_overflow must be set to -1
#in order to avoid deadlocks in SQLAlchemy's
#QueuePool. QueuePool use's a threading.Lock
#when max_overflow is not set to -1, which
#will result in a deadlock when multiple
#connections are created from a single thread.
engine = create_engine(database_connection,
pool_size=database_connection_pool_size,
max_overflow=-1)
self.DatabaseSession = sessionmaker(bind=engine)
else:
self.DatabaseSession = None
#Registrar
self.registrar = ZookeeperServiceRegistrar(self.zookeeper_client)
#Add counter decorator to track service method calls
def counter_decorator(func):
requests_counter = self.counters.get_counter("requests")
open_requests_counter = self.counters.get_counter("open_requests")
def wrapper(*args, **kwargs):
try:
requests_counter.increment()
open_requests_counter.increment()
return func(*args, **kwargs)
finally:
open_requests_counter.decrement()
return wrapper
self._decorate_service_methods(counter_decorator)