def before_server_start(app, _): cluster = Cluster([CLUSTER_HOST]) session = cluster.connect() session.execute(""" CREATE KEYSPACE IF NOT EXISTS %s WITH REPLICATION = {'class' : 'SimpleStrategy', 'replication_factor' : 1}; """ % CLUSTER_KEY_SPACE) session = cluster.connect(keyspace=CLUSTER_KEY_SPACE) aiosession(session) # session.execute(f"DROP KEYSPACE {CLUSTER_KEY_SPACE}") connection.register_connection(CLUSTER_NAME, session=session) create_keyspace_simple(CLUSTER_KEY_SPACE, 1, connections=[CLUSTER_NAME]) connection.setup([CLUSTER_HOST], CLUSTER_KEY_SPACE) setup_db() app.db_session = session logger = logging.getLogger() logger.setLevel(DEBUG_LEVEL) handler = logging.StreamHandler() handler.setFormatter(logging.Formatter(LOGGER_FORMAT)) logger.addHandler(handler) app.log = logger
def setup_cassandra_session_listener(app, loop): global cassandra_cluster cassandra_cluster = Cluster(app.config.CASSANDRA_NODES) session = cassandra_cluster.connect(app.config.CASSANDRA_KEYSPACE) aiosession(session) app.cassandra = session
def connect(self): """连接数据库 """ if self._is_connected: return # auth = PlainTextAuthProvider(username=self.config.CASSANDRA_USER, # password=self.config.CASSANDRA_PASSWORD) # cluster = Cluster(self.config.CASSANDRA_NODES, auth_provider=auth) cluster = Cluster(self.config.CASSANDRA_NODES) try: self.db_session = cluster.connect() except NoHostAvailable: # 再尝试连接一次 self.db_session = cluster.connect() self.db_session.set_keyspace(self.keyspace) self.db_session.row_factory = dict_factory self.db_session.default_timeout = self.timeout self.db_session.default_consistency_level = \ ConsistencyLevel.LOCAL_QUORUM cqlengine_connection.set_session(self.db_session) aiosession(self.db_session) self._is_connected = True
async def init(loop): # Load configuration file config = load_config(PROJECT_ROOT / 'config' / 'config.yml') # Connect to Cassandra cluster cluster = Cluster( [config["cassandra"]["host"]], load_balancing_policy=DCAwareRoundRobinPolicy(), port=config["cassandra"]["port"]) session = cluster.connect() # Set keyspace session.set_keyspace(config["cassandra"]["keyspace"]) # Threaded Cassandra wrapper for asyncio aiosession(session) # Setup database store db_store = DbStore(session, config) # Setup server application app = web.Application(loop=loop) handler = RoutesHandler(db_store, config) register_routes(app, handler) host, port = config["api"]["host"], config["api"]["port"] return app, host, port
async def setup_db(app, loop): cluster = Cluster() app.session = cluster.connect() aiosession(app.session) app.query1 = app.session.prepare( "SELECT * FROM keyspace1.movie_data WHERE primaryTitle LIKE ? LIMIT 20 ALLOW FILTERING ;" )
async def connect(self, load_balancing_policy=None): name = self.name opts = options.group_dict(f'{name} cassandra') uri = opts[option_name(name, 'uri')] r = urlparse(uri) query_as_dict(r.query) if r.scheme.lower() != 'cassandra': raise CassandraConnectorError( f'{uri} is not a cassandra connection scheme') self._hosts = r.hostname.split(',') if r.hostname else ['127.0.0.1'] self._port = r.port or 9042 self._keyspace = None self._user = r.username self._password = r.password keyspace = r.path.lstrip('/') if keyspace: self._keyspace = keyspace executor_threads = opts[option_name(name, 'executor-threads')] _load_balancing_policy = load_balancing_policy or RoundRobinPolicy() LOG.info(f'connecting cassandra [{self.name}] {uri}') auth_provider = PlainTextAuthProvider( username=self._user, password=self._password) # TODO 选取合适的loadbalancingpolicy # Cluster.__init__ called with contact_points specified # should specify a load-balancing policy # http://datastax.github.io/python-driver/_modules/cassandra/cluster.html#Cluster # NOQA # RoundRobinPolicy: # http://datastax.github.io/python-driver/_modules/cassandra/policies.html#RoundRobinPolicy # NOQA cluster = Cluster(contact_points=self._hosts, port=self._port, executor_threads=executor_threads, load_balancing_policy=_load_balancing_policy, auth_provider=auth_provider) self._session = cluster.connect(keyspace=self._keyspace) aiosession(self._session)
def setUp(self): trollius.set_event_loop(None) self.loop = trollius.new_event_loop() self.loop.set_debug(True) self.cluster = Cluster() self.session = self.cluster.connect() aiosession(self.session, loop=self.loop)
def test_main_thread_loop_missing(self): with self.assertRaises(RuntimeError): try: cluster = Cluster() session = cluster.connect() aiosession(session) finally: cluster.shutdown()
def connect(self, loop): self.cassandra_cluster = Cluster([os.getenv("CLUSTER_HOST")], os.getenv("CASSANDRA_PORT"), os.getenv("CASSANDRA_DC")) key_space = os.getenv("CASSANDRA_KEY_SPACE") session = self.cassandra_cluster.connect(key_space) metadata = self.cassandra_cluster.metadata logger.info('Connected to cluster for [pid:{}]: {}'.format( os.getpid(), metadata.cluster_name)) aiosession(session, loop=loop) return session
def prepare(self): node_profile = ExecutionProfile( load_balancing_policy=cassandra.policies.RoundRobinPolicy()) self.cassandra_cluster = Cluster(**CASSANDRA_CONF, execution_profiles={ EXEC_PROFILE_DEFAULT: node_profile }) self.cassandra_session = self.cassandra_cluster.connect('favourite') aiosession(self.cassandra_session) self.cassandra_manager = CassandraManager(self.cassandra_session) self.db_proxy = DBProxy(cassandra_manager=self.cassandra_manager)
def async_start(settings, cassandra_session): """Starts the main async loop.""" loop = asyncio.get_event_loop() # loop.set_debug(True) consumer = MoisturizerKafkaConsumer( cluster=settings.get('kafka.cluster'), topics=settings.get('kafka.topics').split(','), group=settings.get('kafka.group'), event_loop=loop, ) aiosession(cassandra_session, loop=loop) loop.run_until_complete(consumer.start())
def __init__(self, nodes=None, keyspace="hdb", address_map=None, fetch_size=50000, cache_size=1e9): self.nodes = nodes if nodes else ["localhost"] if address_map: translator = LocalNetworkAdressTranslator(address_map) self.cluster = Cluster(self.nodes, address_translator=translator) else: self.cluster = Cluster(self.nodes) s = self.cluster.connect(keyspace) # TODO: Might be useful to be able to set the consistency # level in the configuration s.default_consistency_level = ConsistencyLevel.ONE self.session = aiosession(s) # asyncio wrapper self.session.default_fetch_size = fetch_size # set up the deserializer to use numpy self.session.row_factory = tuple_factory self.session.client_protocol_handler = NumpyProtocolHandler self.prepare_queries() self._cache = SizeLimitedCache( max_size=cache_size, get_item_size=lambda df: df.memory_usage().sum())
def test_main_thread_loop(self): try: loop = asyncio.new_event_loop() loop.set_debug(True) asyncio.set_event_loop(loop) cluster = Cluster() session = cluster.connect() aiosession(session) self.assertIs(loop, session._loop) finally: cluster.shutdown() loop.call_soon(loop.stop) loop.run_forever() loop.close()
async def setup_cassandra(app, loop): logger.warning('Initializing Database') cluster = Cluster(['cassandra_seed_node']) session = cluster.connect() session.execute( """CREATE KEYSPACE IF NOT EXISTS soda WITH replication = {'class': 'SimpleStrategy', 'replication_factor': '1'} AND durable_writes = true;""" ) session.execute('USE soda') register_connection(str(session), session=session) set_default_connection(str(session)) sync_table(User, keyspaces=['soda']) sync_table(Record, keyspaces=['soda']) sync_table(Project, keyspaces=['soda']) aiosession(session) app.db = session return app
async def main(session): aiosession(session) redis = await aioredis.create_redis("redis://localhost") last_id = await redis.get(LAST_ID_KEY) if last_id: log.info(f"Starting consuming messages from {last_id}") last_id = [last_id] else: log.info(f"Starting consuming messages from the beginning") last_id = [0] streams = redis.streams.consumer([STREAM], latest_ids=last_id, encoding="utf-8") async for message in streams: # message is a tuple (stream, id, order_dict) log.info(f"Got message from device {message[2]['device']}") last_id = streams.last_ids_for_stream[STREAM] log.info(f"Set last id to: {last_id}") await redis.set(LAST_ID_KEY, last_id) data = dict(**message[2]) deviceid = data.pop("device") timestamp = util.datetime_from_timestamp(float(data.pop("timestamp"))) query = session.prepare(""" INSERT INTO devices ("deviceid", "timestamp", "data") VALUES (?, ?, ?) """) await session.execute_future( query, (UUID(deviceid), timestamp, json.dumps(data))) await asyncio.sleep(2)
async def on_startup(): from ..game_god import ( GameAction as ga, game_action_processor as gap, Prayer as p, ear as e, ) # global game_store global game_store, GameAction, game_action_processor, Prayer, ear GameAction, game_action_processor, Prayer, ear = ga, gap, p, e # TODO: azure docker compose doesn't support depends so sets 5 # connection timeout logger.info("connecting to redis") game_store = await aioredis.create_redis_pool("redis://game_store", timeout=120) logger.info("connected to redis") # here because requires event loop logger.info("upgrading cqlengine") aiocassandra.aiosession(session) logger.info("upgraded cqlengine")
async def main(): # patches and adds `execute_future`, `execute_futures` and `prepare_future` # to `cassandra.cluster.Session` aiosession(session) # best way is to use cassandra prepared statements # https://cassandra-zone.com/prepared-statements/ # https://datastax.github.io/python-driver/api/cassandra/cluster.html#cassandra.cluster.Session.prepare # try to create them once on application init query = session.prepare('SELECT now() FROM system.local;') # if non-blocking prepared statements is really needed: query = await session.prepare_future('SELECT now() FROM system.local;') print(await session.execute_future(query)) # pagination is also supported query = 'SELECT * FROM system.size_estimates;' statement = SimpleStatement(query, fetch_size=100) # don't miss *s* (execute_futureS) async with session.execute_futures(statement) as paginator: async for row in paginator: print(row)
def test_malformed_session(self): with self.assertRaises(AssertionError): aiosession(None)
def test_patched_twice(self): with self.assertRaises(RuntimeError): aiosession(self.session, loop=self.loop)
def cassandra(session, loop): return aiosession(session, loop=loop)
def setup_cql(self): global session session = cluster.connect(self.config.CASSANDRA_KEYSPACE, wait_for_all_pools=True) aiosession(session) logger.info('CQL session prepared')