def before_server_start(app, _):
cluster = Cluster([CLUSTER_HOST])
session = cluster.connect()
session.execute("""
CREATE KEYSPACE IF NOT EXISTS %s
WITH REPLICATION = {'class' : 'SimpleStrategy', 'replication_factor' : 1};
""" % CLUSTER_KEY_SPACE)
session = cluster.connect(keyspace=CLUSTER_KEY_SPACE)
aiosession(session)
# session.execute(f"DROP KEYSPACE {CLUSTER_KEY_SPACE}")
connection.register_connection(CLUSTER_NAME, session=session)
create_keyspace_simple(CLUSTER_KEY_SPACE, 1, connections=[CLUSTER_NAME])
connection.setup([CLUSTER_HOST], CLUSTER_KEY_SPACE)
setup_db()
app.db_session = session
logger = logging.getLogger()
logger.setLevel(DEBUG_LEVEL)
handler = logging.StreamHandler()
handler.setFormatter(logging.Formatter(LOGGER_FORMAT))
logger.addHandler(handler)
app.log = logger
def setup_cassandra_session_listener(app, loop):
global cassandra_cluster
cassandra_cluster = Cluster(app.config.CASSANDRA_NODES)
session = cassandra_cluster.connect(app.config.CASSANDRA_KEYSPACE)
aiosession(session)
app.cassandra = session
def connect(self):
"""连接数据库
"""
if self._is_connected:
return
# auth = PlainTextAuthProvider(username=self.config.CASSANDRA_USER,
# password=self.config.CASSANDRA_PASSWORD)
# cluster = Cluster(self.config.CASSANDRA_NODES, auth_provider=auth)
cluster = Cluster(self.config.CASSANDRA_NODES)
try:
self.db_session = cluster.connect()
except NoHostAvailable:
# 再尝试连接一次
self.db_session = cluster.connect()
self.db_session.set_keyspace(self.keyspace)
self.db_session.row_factory = dict_factory
self.db_session.default_timeout = self.timeout
self.db_session.default_consistency_level = \
ConsistencyLevel.LOCAL_QUORUM
cqlengine_connection.set_session(self.db_session)
aiosession(self.db_session)
self._is_connected = True
async def init(loop):
# Load configuration file
config = load_config(PROJECT_ROOT / 'config' / 'config.yml')
# Connect to Cassandra cluster
cluster = Cluster(
[config["cassandra"]["host"]],
load_balancing_policy=DCAwareRoundRobinPolicy(),
port=config["cassandra"]["port"])
session = cluster.connect()
# Set keyspace
session.set_keyspace(config["cassandra"]["keyspace"])
# Threaded Cassandra wrapper for asyncio
aiosession(session)
# Setup database store
db_store = DbStore(session, config)
# Setup server application
app = web.Application(loop=loop)
handler = RoutesHandler(db_store, config)
register_routes(app, handler)
host, port = config["api"]["host"], config["api"]["port"]
return app, host, port
async def setup_db(app, loop):
cluster = Cluster()
app.session = cluster.connect()
aiosession(app.session)
app.query1 = app.session.prepare(
"SELECT * FROM keyspace1.movie_data WHERE primaryTitle LIKE ? LIMIT 20 ALLOW FILTERING ;"
)
async def connect(self, load_balancing_policy=None):
name = self.name
opts = options.group_dict(f'{name} cassandra')
uri = opts[option_name(name, 'uri')]
r = urlparse(uri)
query_as_dict(r.query)
if r.scheme.lower() != 'cassandra':
raise CassandraConnectorError(
f'{uri} is not a cassandra connection scheme')
self._hosts = r.hostname.split(',') if r.hostname else ['127.0.0.1']
self._port = r.port or 9042
self._keyspace = None
self._user = r.username
self._password = r.password
keyspace = r.path.lstrip('/')
if keyspace:
self._keyspace = keyspace
executor_threads = opts[option_name(name, 'executor-threads')]
_load_balancing_policy = load_balancing_policy or RoundRobinPolicy()
LOG.info(f'connecting cassandra [{self.name}] {uri}')
auth_provider = PlainTextAuthProvider(
username=self._user, password=self._password)
# TODO 选取合适的loadbalancingpolicy
# Cluster.__init__ called with contact_points specified
# should specify a load-balancing policy
# http://datastax.github.io/python-driver/_modules/cassandra/cluster.html#Cluster # NOQA
# RoundRobinPolicy:
# http://datastax.github.io/python-driver/_modules/cassandra/policies.html#RoundRobinPolicy # NOQA
cluster = Cluster(contact_points=self._hosts,
port=self._port, executor_threads=executor_threads,
load_balancing_policy=_load_balancing_policy,
auth_provider=auth_provider)
self._session = cluster.connect(keyspace=self._keyspace)
aiosession(self._session)
def setUp(self):
trollius.set_event_loop(None)
self.loop = trollius.new_event_loop()
self.loop.set_debug(True)
self.cluster = Cluster()
self.session = self.cluster.connect()
aiosession(self.session, loop=self.loop)
def test_main_thread_loop_missing(self):
with self.assertRaises(RuntimeError):
try:
cluster = Cluster()
session = cluster.connect()
aiosession(session)
finally:
cluster.shutdown()
def connect(self, loop):
self.cassandra_cluster = Cluster([os.getenv("CLUSTER_HOST")],
os.getenv("CASSANDRA_PORT"),
os.getenv("CASSANDRA_DC"))
key_space = os.getenv("CASSANDRA_KEY_SPACE")
session = self.cassandra_cluster.connect(key_space)
metadata = self.cassandra_cluster.metadata
logger.info('Connected to cluster for [pid:{}]: {}'.format(
os.getpid(), metadata.cluster_name))
aiosession(session, loop=loop)
return session
def prepare(self):
node_profile = ExecutionProfile(
load_balancing_policy=cassandra.policies.RoundRobinPolicy())
self.cassandra_cluster = Cluster(**CASSANDRA_CONF,
execution_profiles={
EXEC_PROFILE_DEFAULT: node_profile
})
self.cassandra_session = self.cassandra_cluster.connect('favourite')
aiosession(self.cassandra_session)
self.cassandra_manager = CassandraManager(self.cassandra_session)
self.db_proxy = DBProxy(cassandra_manager=self.cassandra_manager)
def async_start(settings, cassandra_session):
"""Starts the main async loop."""
loop = asyncio.get_event_loop()
# loop.set_debug(True)
consumer = MoisturizerKafkaConsumer(
cluster=settings.get('kafka.cluster'),
topics=settings.get('kafka.topics').split(','),
group=settings.get('kafka.group'),
event_loop=loop,
)
aiosession(cassandra_session, loop=loop)
loop.run_until_complete(consumer.start())
def __init__(self,
nodes=None,
keyspace="hdb",
address_map=None,
fetch_size=50000,
cache_size=1e9):
self.nodes = nodes if nodes else ["localhost"]
if address_map:
translator = LocalNetworkAdressTranslator(address_map)
self.cluster = Cluster(self.nodes, address_translator=translator)
else:
self.cluster = Cluster(self.nodes)
s = self.cluster.connect(keyspace)
# TODO: Might be useful to be able to set the consistency
# level in the configuration
s.default_consistency_level = ConsistencyLevel.ONE
self.session = aiosession(s) # asyncio wrapper
self.session.default_fetch_size = fetch_size
# set up the deserializer to use numpy
self.session.row_factory = tuple_factory
self.session.client_protocol_handler = NumpyProtocolHandler
self.prepare_queries()
self._cache = SizeLimitedCache(
max_size=cache_size,
get_item_size=lambda df: df.memory_usage().sum())
def test_main_thread_loop(self):
try:
loop = asyncio.new_event_loop()
loop.set_debug(True)
asyncio.set_event_loop(loop)
cluster = Cluster()
session = cluster.connect()
aiosession(session)
self.assertIs(loop, session._loop)
finally:
cluster.shutdown()
loop.call_soon(loop.stop)
loop.run_forever()
loop.close()
async def setup_cassandra(app, loop):
logger.warning('Initializing Database')
cluster = Cluster(['cassandra_seed_node'])
session = cluster.connect()
session.execute(
"""CREATE KEYSPACE IF NOT EXISTS soda WITH replication = {'class': 'SimpleStrategy', 'replication_factor': '1'} AND durable_writes = true;"""
)
session.execute('USE soda')
register_connection(str(session), session=session)
set_default_connection(str(session))
sync_table(User, keyspaces=['soda'])
sync_table(Record, keyspaces=['soda'])
sync_table(Project, keyspaces=['soda'])
aiosession(session)
app.db = session
return app
async def main(session):
aiosession(session)
redis = await aioredis.create_redis("redis://localhost")
last_id = await redis.get(LAST_ID_KEY)
if last_id:
log.info(f"Starting consuming messages from {last_id}")
last_id = [last_id]
else:
log.info(f"Starting consuming messages from the beginning")
last_id = [0]
streams = redis.streams.consumer([STREAM],
latest_ids=last_id,
encoding="utf-8")
async for message in streams:
# message is a tuple (stream, id, order_dict)
log.info(f"Got message from device {message[2]['device']}")
last_id = streams.last_ids_for_stream[STREAM]
log.info(f"Set last id to: {last_id}")
await redis.set(LAST_ID_KEY, last_id)
data = dict(**message[2])
deviceid = data.pop("device")
timestamp = util.datetime_from_timestamp(float(data.pop("timestamp")))
query = session.prepare("""
INSERT INTO devices ("deviceid", "timestamp", "data") VALUES (?, ?, ?)
""")
await session.execute_future(
query, (UUID(deviceid), timestamp, json.dumps(data)))
await asyncio.sleep(2)
async def on_startup():
from ..game_god import (
GameAction as ga,
game_action_processor as gap,
Prayer as p,
ear as e,
)
# global game_store
global game_store, GameAction, game_action_processor, Prayer, ear
GameAction, game_action_processor, Prayer, ear = ga, gap, p, e
# TODO: azure docker compose doesn't support depends so sets 5
# connection timeout
logger.info("connecting to redis")
game_store = await aioredis.create_redis_pool("redis://game_store",
timeout=120)
logger.info("connected to redis")
# here because requires event loop
logger.info("upgrading cqlengine")
aiocassandra.aiosession(session)
logger.info("upgraded cqlengine")
async def main():
# patches and adds `execute_future`, `execute_futures` and `prepare_future`
# to `cassandra.cluster.Session`
aiosession(session)
# best way is to use cassandra prepared statements
# https://cassandra-zone.com/prepared-statements/
# https://datastax.github.io/python-driver/api/cassandra/cluster.html#cassandra.cluster.Session.prepare
# try to create them once on application init
query = session.prepare('SELECT now() FROM system.local;')
# if non-blocking prepared statements is really needed:
query = await session.prepare_future('SELECT now() FROM system.local;')
print(await session.execute_future(query))
# pagination is also supported
query = 'SELECT * FROM system.size_estimates;'
statement = SimpleStatement(query, fetch_size=100)
# don't miss *s* (execute_futureS)
async with session.execute_futures(statement) as paginator:
async for row in paginator:
print(row)
def test_malformed_session(self):
with self.assertRaises(AssertionError):
aiosession(None)
def test_patched_twice(self):
with self.assertRaises(RuntimeError):
aiosession(self.session, loop=self.loop)
def cassandra(session, loop):
return aiosession(session, loop=loop)
def setup_cql(self):
global session
session = cluster.connect(self.config.CASSANDRA_KEYSPACE,
wait_for_all_pools=True)
aiosession(session)
logger.info('CQL session prepared')