def get_socket(self, pair=None, force=False):
if self.pool is None:
self.pool = Pool(self._create_connection,
pool_size=self.max_size,
loop=self.store._loop)
self.pool_id += 1
if force:
return self._create_connection(pair)
else:
return self.pool.connect()
def _init(self, namespace=None, parser_class=None, pool_size=50,
decode_responses=False, **kwargs):
self._decode_responses = decode_responses
if not parser_class:
actor = get_actor()
pyparser = actor.cfg.redis_py_parser if actor else False
parser_class = redis_parser(pyparser)
self._parser_class = parser_class
if namespace:
self._urlparams['namespace'] = namespace
self._pool = Pool(self.connect, pool_size=pool_size, loop=self._loop)
self.loaded_scripts = {}
class EchoGreen(AbstractClient):
"""A client for the echo server.
"""
protocol_factory = partial(Connection, EchoProtocol)
def __init__(self, address, wait, pool_size=10, loop=None):
super().__init__(loop)
self.address = address
self.wait = wait
self.pool = Pool(self.connect, pool_size, self._loop)
def connect(self):
return self.create_connection(self.address)
def __call__(self, message):
return self.wait(self._call(message))
@asyncio.coroutine
def _call(self, message):
connection = yield from self.pool.connect()
with connection:
consumer = connection.current_consumer()
consumer.start(message)
yield from consumer.on_finished
return consumer.buffer
def get_socket(self, pair=None, force=False):
if self.pool is None:
self.pool = Pool(self._create_connection, pool_size=self.max_size,
loop=self.store._loop)
self.pool_id += 1
if force:
return self._create_connection(pair)
else:
return self.pool.connect()
class MongoDbPool(object):
'''A connection pool for pymongo client
'''
def __init__(self,
store,
pair,
max_size,
net_timeout,
conn_timeout,
use_ssl,
use_greenlets,
ssl_keyfile=None,
ssl_certfile=None,
ssl_cert_reqs=None,
ssl_ca_certs=None,
wait_queue_timeout=None,
wait_queue_multiple=None):
self.pool = None
self.store = store
self.max_size = max_size
self.pool_id = 0
self.pair = pair
@property
def _loop(self):
return self.store._loop
def _create_connection(self, pair=None):
'''Default method for connecting to remote datastore.
'''
protocol_factory = self.store.create_protocol
host, port = pair or self.store._host
_, protocol = yield self._loop.create_connection(
protocol_factory, host, port)
socket_info = SocketInfo(protocol, self.pool_id)
coroutine_return(socket_info)
def reset(self):
if self.pool is not None:
self.pool.close()
self.pool = None
@green_run
def get_socket(self, pair=None, force=False):
if self.pool is None:
self.pool = Pool(self._create_connection,
pool_size=self.max_size,
loop=self.store._loop)
self.pool_id += 1
if force:
return self._create_connection(pair)
else:
return self.pool.connect()
def maybe_return_socket(self, sock_info):
pass
class Echo(AbstractClient):
'''A client for the echo server.
:param address: set the :attr:`address` attribute
:param full_response: set the :attr:`full_response` attribute
:param pool_size: used when initialising the connetion :attr:`pool`.
:param loop: Optional event loop to set the :attr:`_loop` attribute.
.. attribute:: _loop
The event loop used by the client IO requests.
The event loop is stored at this attribute so that asynchronous
method decorators such as :func:`.task` can be used.
.. attribute:: address
remote server TCP address.
.. attribute:: pool
Asynchronous connection :class:`.Pool`.
.. attribute:: full_response
Flag indicating if the callable method should return the
:class:`EchoProtocol` handling the request (``True``) or
the server response message (``False``).
Default: ``False``
'''
protocol_factory = partial(Connection, EchoProtocol)
def __init__(self, address, full_response=False, pool_size=10, loop=None):
super(Echo, self).__init__(loop)
self.address = address
self.full_response = full_response
self.pool = Pool(self.connect, pool_size, self._loop)
def connect(self):
return self.create_connection(self.address)
@task
def __call__(self, message):
'''Send a ``message`` to the server and wait for a response.
:return: a :class:`.Future`
'''
connection = yield self.pool.connect()
with connection:
consumer = connection.current_consumer()
consumer.start(message)
result = yield consumer.on_finished
result = consumer if self.full_response else consumer.buffer
coroutine_return(result)
def _init(self, namespace=None, parser_class=None, pool_size=50,
decode_responses=False, **kwargs):
self._decode_responses = decode_responses
if not parser_class:
actor = get_actor()
pyparser = actor.cfg.redis_py_parser if actor else False
parser_class = redis_parser(pyparser)
self._parser_class = parser_class
if namespace:
self._urlparams['namespace'] = namespace
self._pool = Pool(self.connect, pool_size=pool_size, loop=self._loop)
self.loaded_scripts = {}
class Echo(pulsar.AbstractUdpClient):
'''A client for the echo server.
:param address: set the :attr:`address` attribute
:param pool_size: used when initialising the connetion :attr:`pool`.
:param loop: Optional event loop to set the :attr:`_loop` attribute.
.. attribute:: _loop
The event loop used by the client IO requests.
The event loop is stored at this attribute so that asynchronous
method decorators such as :func:`.task` can be used.
.. attribute:: address
remote server UDP address.
.. attribute:: pool
Asynchronous client protocol :class:`.Pool`.
'''
protocol_factory = EchoUdpClientProtocol
def __init__(self, address, pool_size=5, loop=None):
super().__init__(loop)
self.address = address
self.pool = Pool(self.create_endpoint, pool_size, self._loop)
def create_endpoint(self):
return self.create_datagram_endpoint(remote_addr=self.address)
def __call__(self, message):
'''Send a ``message`` to the server and wait for a response.
:return: a :class:`.Future`
'''
result = self._call(message)
if not self._loop.is_running():
return self._loop.run_until_complete(result)
else:
return result
@asyncio.coroutine
def _call(self, message):
protocol = yield from self.pool.connect()
with protocol:
result = yield from protocol.send(message)
return result
class Echo(pulsar.AbstractUdpClient):
'''A client for the echo server.
:param address: set the :attr:`address` attribute
:param pool_size: used when initialising the connetion :attr:`pool`.
:param loop: Optional event loop to set the :attr:`_loop` attribute.
.. attribute:: _loop
The event loop used by the client IO requests.
The event loop is stored at this attribute so that asynchronous
method decorators such as :func:`.task` can be used.
.. attribute:: address
remote server UDP address.
.. attribute:: pool
Asynchronous client protocol :class:`.Pool`.
'''
protocol_factory = EchoUdpClientProtocol
def __init__(self, address, pool_size=5, loop=None):
super(Echo, self).__init__(loop)
self.address = address
self.pool = Pool(self.create_endpoint, pool_size, self._loop)
def create_endpoint(self):
return self.create_datagram_endpoint(remote_addr=self.address)
def __call__(self, message):
'''Send a ``message`` to the server and wait for a response.
:return: a :class:`.Future`
'''
result = self._call(message)
if not self._loop.is_running():
return self._loop.run_until_complete(result)
else:
return result
def _call(self, message):
protocol = yield from self.pool.connect()
with protocol:
result = yield from protocol.send(message)
return result
class MongoDbPool(object):
'''A connection pool for pymongo client
'''
def __init__(self, store, pair, max_size, net_timeout, conn_timeout,
use_ssl, use_greenlets, ssl_keyfile=None, ssl_certfile=None,
ssl_cert_reqs=None, ssl_ca_certs=None,
wait_queue_timeout=None, wait_queue_multiple=None):
self.pool = None
self.store = store
self.max_size = max_size
self.pool_id = 0
self.pair = pair
@property
def _loop(self):
return self.store._loop
def _create_connection(self, pair=None):
'''Default method for connecting to remote datastore.
'''
protocol_factory = self.store.create_protocol
host, port = pair or self.store._host
_, protocol = yield self._loop.create_connection(
protocol_factory, host, port)
socket_info = SocketInfo(protocol, self.pool_id)
coroutine_return(socket_info)
def reset(self):
if self.pool is not None:
self.pool.close()
self.pool = None
@green_run
def get_socket(self, pair=None, force=False):
if self.pool is None:
self.pool = Pool(self._create_connection, pool_size=self.max_size,
loop=self.store._loop)
self.pool_id += 1
if force:
return self._create_connection(pair)
else:
return self.pool.connect()
def maybe_return_socket(self, sock_info):
pass
def __init__(self, appname, wait, pool_size=5, loop=None):
super().__init__(loop)
self.app_name = appname
self.wait = wait
self.pool = Pool(self.connect, pool_size, self._loop)
class RedisStore(RemoteStore):
'''Redis :class:`.Store` implementation.
'''
protocol_factory = partial(RedisStoreConnection, Consumer)
supported_queries = frozenset(('filter', 'exclude'))
def _init(self, namespace=None, parser_class=None, pool_size=50,
decode_responses=False, **kwargs):
self._decode_responses = decode_responses
if not parser_class:
actor = get_actor()
pyparser = actor.cfg.redis_py_parser if actor else False
parser_class = redis_parser(pyparser)
self._parser_class = parser_class
if namespace:
self._urlparams['namespace'] = namespace
self._pool = Pool(self.connect, pool_size=pool_size, loop=self._loop)
if self._database is None:
self._database = 0
self._database = int(self._database)
self.loaded_scripts = {}
@property
def pool(self):
return self._pool
@property
def namespace(self):
'''The prefix namespace to append to all transaction on keys
'''
n = self._urlparams.get('namespace')
return '%s:' % n if n else ''
def key(self):
return (self._dns, self._encoding)
def client(self):
'''Get a :class:`.RedisClient` for the Store'''
return RedisClient(self)
def pipeline(self):
'''Get a :class:`.Pipeline` for the Store'''
return Pipeline(self)
def pubsub(self, protocol=None):
return RedisPubSub(self, protocol=protocol)
def ping(self):
return self.client().ping()
def execute(self, *args, **options):
connection = yield from self._pool.connect()
with connection:
result = yield from connection.execute(*args, **options)
return result
def execute_pipeline(self, commands, raise_on_error=True):
conn = yield from self._pool.connect()
with conn:
result = yield from conn.execute_pipeline(commands, raise_on_error)
return result
def connect(self, protocol_factory=None):
protocol_factory = protocol_factory or self.create_protocol
if isinstance(self._host, tuple):
host, port = self._host
transport, connection = yield from self._loop.create_connection(
protocol_factory, host, port)
else:
raise NotImplementedError('Could not connect to %s' %
str(self._host))
if self._password:
yield from connection.execute('AUTH', self._password)
if self._database:
yield from connection.execute('SELECT', self._database)
return connection
def flush(self):
return self.execute('flushdb')
def close(self):
'''Close all open connections.'''
return self._pool.close()
def has_query(self, query_type):
return query_type in self.supported_queries
def basekey(self, meta, *args):
key = '%s%s' % (self.namespace, meta.table_name)
postfix = ':'.join((to_string(p) for p in args if p is not None))
return '%s:%s' % (key, postfix) if postfix else key
def meta(self, meta):
'''Extract model metadata for lua script stdnet/lib/lua/odm.lua'''
# indices = dict(((idx.attname, idx.unique) for idx in meta.indices))
data = meta.as_dict()
data['namespace'] = self.basekey(meta)
return data
def __init__(self, address, pool_size=5, loop=None):
super().__init__(loop)
self.address = address
self.pool = Pool(self.create_endpoint, pool_size, self._loop)
def __init__(self, address, pool_size=5, loop=None):
super(Echo, self).__init__(loop)
self.address = address
self.pool = Pool(self.create_endpoint, pool_size, self._loop)
class RedisStore(RemoteStore):
'''Redis :class:`.Store` implementation.
'''
protocol_factory = partial(RedisStoreConnection, Consumer)
supported_queries = frozenset(('filter', 'exclude'))
def _init(self,
namespace=None,
parser_class=None,
pool_size=50,
decode_responses=False,
**kwargs):
self._decode_responses = decode_responses
if not parser_class:
actor = get_actor()
pyparser = actor.cfg.redis_py_parser if actor else False
parser_class = redis_parser(pyparser)
self._parser_class = parser_class
if namespace:
self._urlparams['namespace'] = namespace
self._pool = Pool(self.connect, pool_size=pool_size, loop=self._loop)
if self._database is None:
self._database = 0
self._database = int(self._database)
self.loaded_scripts = set()
@property
def pool(self):
return self._pool
@property
def namespace(self):
'''The prefix namespace to append to all transaction on keys
'''
n = self._urlparams.get('namespace')
return '%s:' % n if n else ''
def key(self):
return (self._dns, self._encoding)
def client(self):
'''Get a :class:`.RedisClient` for the Store'''
return RedisClient(self)
def pipeline(self):
'''Get a :class:`.Pipeline` for the Store'''
return Pipeline(self)
def pubsub(self, protocol=None):
return RedisPubSub(self, protocol=protocol)
def ping(self):
return self.client().ping()
async def execute(self, *args, **options):
connection = await self._pool.connect()
with connection:
result = await connection.execute(*args, **options)
return result
async def execute_pipeline(self, commands, raise_on_error=True):
conn = await self._pool.connect()
with conn:
result = await conn.execute_pipeline(commands, raise_on_error)
return result
async def connect(self, protocol_factory=None):
protocol_factory = protocol_factory or self.create_protocol
if isinstance(self._host, tuple):
host, port = self._host
transport, connection = await self._loop.create_connection(
protocol_factory, host, port)
else:
raise NotImplementedError('Could not connect to %s' %
str(self._host))
if self._password:
await connection.execute('AUTH', self._password)
if self._database:
await connection.execute('SELECT', self._database)
return connection
def flush(self):
return self.execute('flushdb')
def close(self):
'''Close all open connections.'''
return self._pool.close()
def has_query(self, query_type):
return query_type in self.supported_queries
def basekey(self, meta, *args):
key = '%s%s' % (self.namespace, meta.table_name)
postfix = ':'.join((to_string(p) for p in args if p is not None))
return '%s:%s' % (key, postfix) if postfix else key
def meta(self, meta):
'''Extract model metadata for lua script stdnet/lib/lua/odm.lua'''
# indices = dict(((idx.attname, idx.unique) for idx in meta.indices))
data = meta.as_dict()
data['namespace'] = self.basekey(meta)
return data
class RedisStore(Store):
'''Redis :class:`.Store` implementation.
'''
protocol_factory = partial(RedisStoreConnection, Consumer)
supported_queries = frozenset(('filter', 'exclude'))
def _init(self, namespace=None, parser_class=None, pool_size=50,
decode_responses=False, **kwargs):
self._decode_responses = decode_responses
if not parser_class:
actor = get_actor()
pyparser = actor.cfg.redis_py_parser if actor else False
parser_class = redis_parser(pyparser)
self._parser_class = parser_class
if namespace:
self._urlparams['namespace'] = namespace
self._pool = Pool(self.connect, pool_size=pool_size, loop=self._loop)
self.loaded_scripts = {}
@property
def pool(self):
return self._pool
@property
def namespace(self):
'''The prefix namespace to append to all transaction on keys
'''
n = self._urlparams.get('namespace')
return '%s:' % n if n else ''
def key(self):
return (self._dns, self._encoding)
def client(self):
'''Get a :class:`.RedisClient` for the Store'''
return RedisClient(self)
def pipeline(self):
'''Get a :class:`.Pipeline` for the Store'''
return Pipeline(self)
def pubsub(self, protocol=None):
return PubSub(self, protocol=protocol)
def ping(self):
return self.client().ping()
@task
def execute(self, *args, **options):
connection = yield self._pool.connect()
with connection:
result = yield connection.execute(*args, **options)
if isinstance(result, ResponseError):
raise result.exception
coroutine_return(result)
@task
def execute_pipeline(self, commands, raise_on_error=True):
conn = yield self._pool.connect()
with conn:
result = yield conn.execute_pipeline(commands, raise_on_error)
if isinstance(result, ResponseError):
raise result.exception
coroutine_return(result)
def connect(self, protocol_factory=None):
protocol_factory = protocol_factory or self.create_protocol
if isinstance(self._host, tuple):
host, port = self._host
transport, connection = yield self._loop.create_connection(
protocol_factory, host, port)
else:
raise NotImplementedError('Could not connect to %s' %
str(self._host))
if self._password:
yield connection.execute('AUTH', self._password)
if self._database:
yield connection.execute('SELECT', self._database)
coroutine_return(connection)
def execute_transaction(self, transaction):
'''Execute a :class:`.Transaction`
'''
pipe = self.pipeline()
update_insert = set((Command.INSERT, Command.UPDATE))
#
for command in transaction.commands:
action = command.action
if not action:
pipe.execute(*command.args)
elif action in update_insert:
model = command.args
key = self.basekey(model._meta, model.id)
pipe.hmset(key, self.model_data(model, action))
else:
raise NotImplementedError
response = yield pipe.commit()
for command in transaction.commands:
if command.action == Command.INSERT:
model = command.args
model['_rev'] = 1
def get_model(self, manager, pk):
key = '%s%s:%s' % (self.namespace, manager._meta.table_name,
to_string(pk))
return self.execute('hgetall', key,
factory=partial(self.build_model, manager))
def compile_query(self, query):
compiled = CompiledQuery(self.pipeline())
return compiled
def flush(self):
return self.execute('flushdb')
def close(self):
'''Close all open connections.'''
return self._pool.close()
def has_query(self, query_type):
return query_type in self.supported_queries
def basekey(self, meta, *args):
key = '%s%s' % (self.namespace, meta.table_name)
postfix = ':'.join((to_string(p) for p in args if p is not None))
return '%s:%s' % (key, postfix) if postfix else key
def meta(self, meta):
'''Extract model metadata for lua script stdnet/lib/lua/odm.lua'''
indices = dict(((idx.attname, idx.unique) for idx in meta.indices))
data = meta.as_dict()
data['namespace'] = self.basekey(meta)
return data
def _init(self, pool_size=50, auth_key='', **kwargs):
self.auth_key = auth_key.encode('ascii')
self._pool = Pool(self.connect, pool_size=pool_size, loop=self._loop)
def __init__(self, address, full_response=False, pool_size=10, loop=None):
super(Echo, self).__init__(loop)
self.address = address
self.full_response = full_response
self.pool = Pool(self.connect, pool_size, self._loop)
def __init__(self, address, wait, pool_size=10, loop=None):
super().__init__(loop)
self.address = address
self.wait = wait
self.pool = Pool(self.connect, pool_size, self._loop)
def __init__(self, address, full_response=False, pool_size=10, loop=None):
super(Echo, self).__init__(loop)
self.address = address
self.full_response = full_response
self.pool = Pool(self.connect, pool_size, self._loop)
class RedisStore(Store):
'''Redis :class:`.Store` implementation.
'''
protocol_factory = partial(RedisStoreConnection, Consumer)
supported_queries = frozenset(('filter', 'exclude'))
def _init(self, namespace=None, parser_class=None, pool_size=50,
decode_responses=False, **kwargs):
self._decode_responses = decode_responses
if not parser_class:
actor = get_actor()
pyparser = actor.cfg.redis_py_parser if actor else False
parser_class = redis_parser(pyparser)
self._parser_class = parser_class
if namespace:
self._urlparams['namespace'] = namespace
self._pool = Pool(self.connect, pool_size=pool_size, loop=self._loop)
self.loaded_scripts = {}
@property
def pool(self):
return self._pool
@property
def namespace(self):
'''The prefix namespace to append to all transaction on keys
'''
n = self._urlparams.get('namespace')
return '%s:' % n if n else ''
def key(self):
return (self._dns, self._encoding)
def client(self):
'''Get a :class:`.RedisClient` for the Store'''
return RedisClient(self)
def pipeline(self):
'''Get a :class:`.Pipeline` for the Store'''
return Pipeline(self)
def pubsub(self, protocol=None):
return PubSub(self, protocol=protocol)
def ping(self):
return self.client().ping()
@task
def execute(self, *args, **options):
connection = yield self._pool.connect()
with connection:
result = yield connection.execute(*args, **options)
if isinstance(result, ResponseError):
raise result.exception
coroutine_return(result)
@task
def execute_pipeline(self, commands, raise_on_error=True):
conn = yield self._pool.connect()
with conn:
result = yield conn.execute_pipeline(commands, raise_on_error)
if isinstance(result, ResponseError):
raise result.exception
coroutine_return(result)
def connect(self, protocol_factory=None):
protocol_factory = protocol_factory or self.create_protocol
if isinstance(self._host, tuple):
host, port = self._host
transport, connection = yield From(self._loop.create_connection(
protocol_factory, host, port))
else:
raise NotImplementedError('Could not connect to %s' %
str(self._host))
if self._password:
yield From(connection.execute('AUTH', self._password))
if self._database:
yield From(connection.execute('SELECT', self._database))
coroutine_return(connection)
def execute_transaction(self, transaction):
'''Execute a :class:`.Transaction`
'''
models = []
pipe = self.pipeline()
update_insert = set((Command.INSERT, Command.UPDATE))
#
for command in transaction.commands:
action = command.action
if not action:
pipe.execute(*command.args)
elif action in update_insert:
model = command.args
model['_rev'] = model.get('_rev', 0) + 1
models.append(model)
key = self.basekey(model._meta, model.id)
pipe.hmset(key, self.model_data(model, action))
else:
raise NotImplementedError
yield From(pipe.commit())
coroutine_return(models)
def get_model(self, manager, pk):
key = '%s%s:%s' % (self.namespace, manager._meta.table_name,
to_string(pk))
return self.execute('hgetall', key,
factory=partial(self.build_model, manager))
def compile_query(self, query):
compiled = CompiledQuery(self.pipeline())
return compiled
def flush(self):
return self.execute('flushdb')
def close(self):
'''Close all open connections.'''
return self._pool.close()
def has_query(self, query_type):
return query_type in self.supported_queries
def basekey(self, meta, *args):
key = '%s%s' % (self.namespace, meta.table_name)
postfix = ':'.join((to_string(p) for p in args if p is not None))
return '%s:%s' % (key, postfix) if postfix else key
def meta(self, meta):
'''Extract model metadata for lua script stdnet/lib/lua/odm.lua'''
# indices = dict(((idx.attname, idx.unique) for idx in meta.indices))
data = meta.as_dict()
data['namespace'] = self.basekey(meta)
return data