def test_connection_param_validation(self): """ Test to validate that invalid parameter combinations for registering connections via session are not tolerated @since 3.8 @jira_ticket PYTHON-649 @expected_result queries should execute appropriately @test_category object_mapper """ cluster = Cluster([CASSANDRA_IP]) session = cluster.connect() with self.assertRaises(CQLEngineException): conn.register_connection("bad_coonection1", session=session, consistency="not_null") with self.assertRaises(CQLEngineException): conn.register_connection("bad_coonection2", session=session, lazy_connect="not_null") with self.assertRaises(CQLEngineException): conn.register_connection("bad_coonection3", session=session, retry_connect="not_null") with self.assertRaises(CQLEngineException): conn.register_connection("bad_coonection4", session=session, cluster_options="not_null") with self.assertRaises(CQLEngineException): conn.register_connection("bad_coonection5", hosts="not_null", session=session) cluster.shutdown() cluster.shutdown() cluster.shutdown()
def mapper_step2(stream): """This mapper receives records from the SuperSpecificCartesianInputFormat, aka the cartesian product of the morning flights and +2 days afternoon flights. It simply have to reduce to the input where dest1 == origin2. """ from cassandra.cluster import Cluster from cassandra.cqlengine import connection from decimal import Decimal from cassandra.cqlengine import columns from cassandra.cqlengine.models import Model from setup_cassandradb import Q3Entry cluster = Cluster(['node7', 'node8', 'node9']) connection.register_connection('con', session=cluster.connect(), default=True) for first_leg, second_leg in mapred.iter_key_values(stream): date1, origin1, dest1, delay1, time1, carrier1, flight1 = first_leg date2, origin2, dest2, delay2, time2, carrier2, flight2 = second_leg if dest1 == origin2: total_delay = float(delay1) + float(delay2) Q3Entry.create(date_origin_dest1='_'.join([date1, origin1, dest1]), date1=date1, origin=origin1, dest1=dest1, dest2=dest2, total_delay=Decimal(total_delay), datetime1=date1 + ' ' + time1, flight1=carrier1 + ' ' + flight1, datetime2=date2 + ' ' + time2, flight2=carrier2 + ' ' + flight2)
def connect(self): auth_provider = PlainTextAuthProvider( username=self.username, password=self.password ) self.cluster = Cluster( ["node_0", "node_1"], auth_provider=auth_provider, # executor_threads=int(os.getenv('CASSANDRA_EXECUTOR_THREADS')), # protocol_version=int(os.getenv('CASSANDRA_PROTOCOL_VERSION')), ) self.session = self.cluster.connect() self.session.execute( """ CREATE KEYSPACE IF NOT EXISTS test_key_space WITH REPLICATION = { 'class' : 'SimpleStrategy', 'replication_factor': '3'} AND durable_writes = true; """ ) self.session.set_keyspace(self.key_space) register_connection(str(self.session), session=self.session) set_default_connection(str(self.session))
def before_server_start(app, _): cluster = Cluster([CLUSTER_HOST]) session = cluster.connect() session.execute(""" CREATE KEYSPACE IF NOT EXISTS %s WITH REPLICATION = {'class' : 'SimpleStrategy', 'replication_factor' : 1}; """ % CLUSTER_KEY_SPACE) session = cluster.connect(keyspace=CLUSTER_KEY_SPACE) aiosession(session) # session.execute(f"DROP KEYSPACE {CLUSTER_KEY_SPACE}") connection.register_connection(CLUSTER_NAME, session=session) create_keyspace_simple(CLUSTER_KEY_SPACE, 1, connections=[CLUSTER_NAME]) connection.setup([CLUSTER_HOST], CLUSTER_KEY_SPACE) setup_db() app.db_session = session logger = logging.getLogger() logger.setLevel(DEBUG_LEVEL) handler = logging.StreamHandler() handler.setFormatter(logging.Formatter(LOGGER_FORMAT)) logger.addHandler(handler) app.log = logger
def test_connection_param_validation(self): """ Test to validate that invalid parameter combinations for registering connections via session are not tolerated @since 3.8 @jira_ticket PYTHON-649 @expected_result queries should execute appropriately @test_category object_mapper """ session = Cluster(['127.0.0.1']).connect() with self.assertRaises(CQLEngineException): conn.register_connection("bad_coonection1", session=session, consistency="not_null") with self.assertRaises(CQLEngineException): conn.register_connection("bad_coonection2", session=session, lazy_connect="not_null") with self.assertRaises(CQLEngineException): conn.register_connection("bad_coonection3", session=session, retry_connect="not_null") with self.assertRaises(CQLEngineException): conn.register_connection("bad_coonection4", session=session, cluster_options="not_null") with self.assertRaises(CQLEngineException): conn.register_connection("bad_coonection5", hosts="not_null", session=session)
def setUpClass(cls): super(ManagementConnectionTests, cls).setUpClass() conn.unregister_connection('default') conn.register_connection('fake_cluster', ['127.0.0.100'], lazy_connect=True, retry_connect=True, default=True) conn.register_connection('cluster', [CASSANDRA_IP])
def setUpClass(cls): conn.register_connection('cluster', [CASSANDRA_IP]) test_queryset.TestModel.__queryset__ = ModelQuerySetNew test_queryset.IndexedTestModel.__queryset__ = ModelQuerySetNew test_queryset.IndexedCollectionsTestModel.__queryset__ = ModelQuerySetNew test_queryset.TestMultiClusteringModel.__queryset__ = ModelQuerySetNew super(BaseConnectionTestNoDefault, cls).setUpClass() conn.unregister_connection('default')
def connection_factory(): contact_point = os.environ['CASSANDRA_CONTACT_POINT'] cluster = Cluster([contact_point]) connection = cluster.connect(keyspace=os.environ['CASSANDRA_KEYSPACE']) register_connection(str(connection), session=connection) set_default_connection(str(connection)) return connection
def db_set(keyspace): try: connection.set_default_connection(CONN_NM) except Exception as err: print('db set exception occured...') connection.unregister_connection(CONN_NM) connection.register_connection(CONN_NM, config.CASSANDRA_CONTACT_POINTS[config.DTP], default=True) connection.session.set_keyspace(keyspace) print('...fix db connection. return to process')
def setUpClass(cls): super(UsingDescriptorTests, cls).setUpClass() conn.unregister_connection('default') conn.register_connection('fake_cluster', ['127.0.0.100'], lazy_connect=True, retry_connect=True, default=True) conn.register_connection('cluster', ['127.0.0.1'])
def setUpClass(cls): conn.register_connection('cluster', ['127.0.0.1']) test_queryset.TestModel.__queryset__ = ModelQuerySetNew test_queryset.IndexedTestModel.__queryset__ = ModelQuerySetNew test_queryset.IndexedCollectionsTestModel.__queryset__ = ModelQuerySetNew test_queryset.TestMultiClusteringModel.__queryset__ = ModelQuerySetNew super(BaseConnectionTestNoDefault, cls).setUpClass() conn.unregister_connection('default')
def setUpClass(cls): super(ContextQueryConnectionTests, cls).setUpClass() create_keyspace_simple('ks1', 1) conn.unregister_connection('default') conn.register_connection('fake_cluster', ['1.2.3.4'], lazy_connect=True, retry_connect=True, default=True) conn.register_connection('cluster', [CASSANDRA_IP]) with ContextQuery(TestModel, connection='cluster') as tm: sync_table(tm)
def setUpClass(cls): super(ContextQueryConnectionTests, cls).setUpClass() create_keyspace_simple('ks1', 1) conn.unregister_connection('default') conn.register_connection('fake_cluster', ['127.0.0.100'], lazy_connect=True, retry_connect=True, default=True) conn.register_connection('cluster', ['127.0.0.1']) with ContextQuery(TestModel, connection='cluster') as tm: sync_table(tm)
def pool_initializer(): from nexustiles.nexustiles import NexusTileService global tile_service tile_service = NexusTileService() # TODO This is a hack to make sure each sub-process uses it's own connection to cassandra. data-access needs to be updated from cassandra.cqlengine import connection from multiprocessing import current_process connection.register_connection(current_process().name, [host.address for host in connection.get_session().hosts]) connection.set_default_connection(current_process().name)
def setUpClass(cls): super(BatchQueryConnectionTests, cls).setUpClass() create_keyspace_simple('ks1', 1) sync_table(TestModel) sync_table(AnotherTestModel) conn.unregister_connection('default') conn.register_connection('fake_cluster', ['127.0.0.100'], lazy_connect=True, retry_connect=True, default=True) conn.register_connection('cluster', [CASSANDRA_IP])
def setUpClass(cls): super(BatchQueryConnectionTests, cls).setUpClass() create_keyspace_simple('ks1', 1) sync_table(TestModel) sync_table(AnotherTestModel) conn.unregister_connection('default') conn.register_connection('fake_cluster', ['127.0.0.100'], lazy_connect=True, retry_connect=True, default=True) conn.register_connection('cluster', ['127.0.0.1'])
def drop_keyspace(cls, nodes=None, keyspace='results_database', auth_provider=None): nodes = nodes if nodes else ['localhost'] connection_id = uuid.uuid4() try: register_connection(name=str(connection_id), session=Cluster(nodes, auth_provider=auth_provider).connect()) does_keyspace_exist = keyspace in get_cluster(str(connection_id)).metadata.keyspaces if does_keyspace_exist: drop_keyspace(keyspace, connections=[str(connection_id)]) finally: unregister_connection(name=str(connection_id))
def setup_connection() -> None: """ Setup the connection that the mapper should use. This should be called when the api starts. """ cluster_session = get_cluster().connect() connection.register_connection('cluster', session=cluster_session, default=True)
def init(sess): conn.register_connection("swapi", session=sess, default=True) mgmt.create_keyspace_simple("swapi", replication_factor=1, durable_writes=True) mgmt.sync_table(Films) mgmt.sync_table(People) mgmt.sync_table(Planets) mgmt.sync_table(Species) mgmt.sync_table(Starships) mgmt.sync_table(Vehicles)
def _create_cluster_with_cp_options(cls, name, cp_options): execution_profiles = { EXEC_PROFILE_DEFAULT: ExecutionProfile(continuous_paging_options=cp_options) } cls.cluster_default = Cluster(protocol_version=cls.protocol_version, execution_profiles=execution_profiles) cls.session_default = cls.cluster_default.connect( wait_for_all_pools=True) connection.register_connection(name, default=True, session=cls.session_default) cls.connections.add(name)
def test_connection_from_hosts(self): """ Test to ensure that you can register a connection from a list of hosts @since 3.8 @jira_ticket PYTHON-692 @expected_result queries should execute appropriately @test_category object_mapper """ connection_name = 'from_hosts' conn.register_connection(connection_name, hosts=[CASSANDRA_IP]) self.assertIsNotNone(conn.get_connection(connection_name).cluster.metadata.get_host(CASSANDRA_IP)) self.addCleanup(conn.unregister_connection, connection_name)
def test_connection_creation_from_session(self): """ Test to ensure that you can register a connection from a session @since 3.8 @jira_ticket PYTHON-649 @expected_result queries should execute appropriately @test_category object_mapper """ session = Cluster(['127.0.0.1']).connect() connection_name = 'from_session' conn.register_connection(connection_name, session=session) self.addCleanup(conn.unregister_connection, connection_name)
def connect(self): auth_provider = PlainTextAuthProvider(username='******', password='******') self.cluster = Cluster( ['node_0', 'node_1'], auth_provider=auth_provider, # executor_threads=int(os.getenv('CASSANDRA_EXECUTOR_THREADS')), # protocol_version=int(os.getenv('CASSANDRA_PROTOCOL_VERSION')), ) self.session = self.cluster.connect() register_connection(str(self.session), session=self.session) set_default_connection(str(self.session))
def test_connection_creation_from_session(self): """ Test to ensure that you can register a connection from a session @since 3.8 @jira_ticket PYTHON-649 @expected_result queries should execute appropriately @test_category object_mapper """ cluster = Cluster([CASSANDRA_IP]) session = cluster.connect() connection_name = 'from_session' conn.register_connection(connection_name, session=session) self.assertIsNotNone(conn.get_connection(connection_name).cluster.metadata.get_host(CASSANDRA_IP)) self.addCleanup(conn.unregister_connection, connection_name) cluster.shutdown()
def setup(self): if not self.keyspace or not isinstance(self.keyspace, text_type): self.keyspace = self.container.service_name cluster_options = dict( reconnection_policy=ExponentialReconnectionPolicy(base_delay=3, max_delay=15), connection_class=EventletConnection) if self.models: cluster_options['protocol_version'] = 3 os.environ.setdefault('CQLENG_ALLOW_SCHEMA_MANAGEMENT', '1') self.connection = register_connection( name=self.connection_name, hosts=self.nodes, lazy_connect=False, retry_connect=True, cluster_options=cluster_options, default=self.connection_name == DEFAULT_CONNECTION_NAME) topology = self.cassandra_config.get('topology', SIMPLE_KEYSPACE) if topology not in (SIMPLE_KEYSPACE, NETWORK_KEYSPACE): topology = SIMPLE_KEYSPACE topology_func = { SIMPLE_KEYSPACE: management.create_keyspace_simple, NETWORK_KEYSPACE: management.create_keyspace_network_topology }.get(topology) topology_func(**self.topology_options(topology)) if self.session: self.session.set_keyspace(self.keyspace) for model in self.models: sync_table(model, [self.keyspace], [self.connection])
def cqlengine_management(cassandra): """Setup session for cqlengine """ # create test keyspace os.environ['CQLENG_ALLOW_SCHEMA_MANAGEMENT'] = 'true' test_keyspace = 'test_async_cqlengine' cqlengine_connection.register_connection( 'cqlengine', session=cassandra, default=True) management.create_keyspace_simple(test_keyspace, replication_factor=1) # setup cqlengine session cassandra.set_keyspace(test_keyspace) cqlengine_connection.set_session(cassandra) yield management management.drop_keyspace(test_keyspace) cqlengine_connection.unregister_connection('cqlengine')
def connect(self): """ DB connection.""" auth_provider = PlainTextAuthProvider(username=os.getenv('CASSANDRA_USERNAME'), password=os.getenv('CASSANDRA_PASSWORD')) self.cluster = Cluster( [os.getenv('CASSANDRA_HOST')], load_balancing_policy=TokenAwarePolicy(DCAwareRoundRobinPolicy()), port=os.getenv('CASSANDRA_PORT'), auth_provider=auth_provider, executor_threads=2, protocol_version=4, ) self.session = self.cluster.connect() register_connection(str(self.session), session=self.session) set_default_connection(str(self.session))
def load_db(stream): from cassandra.cluster import Cluster from cassandra.cqlengine import connection from decimal import Decimal from cassandra.cqlengine import columns from cassandra.cqlengine.models import Model from setup_cassandradb import Q22Entry cluster = Cluster(['node7', 'node8', 'node9']) connection.register_connection('con', session=cluster.connect(), default=True) for (origin, dest), (delay, _) in mapred.iter_key_values(stream): Q22Entry.create(origin=origin, dest=dest, delay=Decimal(delay))
def __new__(cls, *args, **kwargs): try: cls.__cluster = Cluster( auth_provider=AUTH_PROVIDER, cloud=CLOUD_CONFIG, load_balancing_policy=DCAwareRoundRobinPolicy( local_dc="dc-1")) # db hosts and port to start clusters cls.__cluster.connection_class = LibevConnection # use libev event loop cls.__session = cls.__cluster.connect( ) # connect to hosts - cls is server.db.init cls.__session.execute(f'USE {KEYSPACE}') connection.register_connection( 'DML', session=cls.__session) # DML cluster for sync_table ops return super(init, cls).__new__(cls, *args, **kwargs) except Exception as e: typer.secho(f"➲ [Could not initialize database] ::: {e}", fg=typer.colors.RED, bold=True)
def create_session(): cluster = Cluster() session = cluster.connect() # Create keyspace, if already have keyspace your can skip this os.environ['CQLENG_ALLOW_SCHEMA_MANAGEMENT'] = 'true' connection.register_connection('cqlengine', session=session, default=True) management.create_keyspace_simple('example', replication_factor=1) management.sync_table(User, keyspaces=['example']) # Run the example function in asyncio loop loop = asyncio.get_event_loop() # Wrap cqlengine connection with aiosession aiosession_for_cqlengine(session, loop=loop) session.set_keyspace('example') connection.set_session(session) return session, loop
async def setup_cassandra(app, loop): logger.warning('Initializing Database') cluster = Cluster(['cassandra_seed_node']) session = cluster.connect() session.execute( """CREATE KEYSPACE IF NOT EXISTS soda WITH replication = {'class': 'SimpleStrategy', 'replication_factor': '1'} AND durable_writes = true;""" ) session.execute('USE soda') register_connection(str(session), session=session) set_default_connection(str(session)) sync_table(User, keyspaces=['soda']) sync_table(Record, keyspaces=['soda']) sync_table(Project, keyspaces=['soda']) aiosession(session) app.db = session return app
async def connect_to_cassandra(app): config = app['config'] sleep = config.get('sleep', 0) max_retry = config.get('max_retry', 3) for i in range(max_retry): try: session = Cluster(config['cassandra_host']).connect() except NoHostAvailable: if i == (max_retry - 1): msg = ( 'Failed to connect to cassandra, retrying ' 'in %ss (%s/%s)' ) await app['logger'].error('main.connect_to_cassndra', msg % (sleep, i + 1, max_retry)) raise else: msg = ( 'Failed to connect to cassandra, retrying ' 'in %ss (%s/%s)' ) await app['logger'].debug('main.connect_to_cassndra', msg % (sleep, i + 1, max_retry)) time.sleep(sleep) connection.register_connection( config['default_connection'], session=session ) create_keyspace_simple( config['default_keyspace'], replication_factor=1, connections=[config['default_connection']] ) User.__keyspace__ = config['default_keyspace'] User.__connection__ = config['default_connection'] sync_table( User, keyspaces=[config['default_keyspace']], connections=[config['default_connection']] )
def __init__(self): cloud_config= { 'secure_connect_bundle': './private/secure-connect-CoursesDB.zip' } user = None password = None with open('./private/CassandraPassword.json') as f: data = json.load(f) user = data['user'] password = data['password'] auth_provider = PlainTextAuthProvider(user, password) cluster = Cluster(cloud=cloud_config, auth_provider=auth_provider) self.session = cluster.connect() self.session.execute("USE {}".format(_keyspace)) _session = self.session register_connection(str(_session), session=_session) set_default_connection(str(_session))
def __new__(cls, *args, **kwargs): try: cls.__cluster = Cluster( contact_points=HOSTS, port=PORT, load_balancing_policy=DCAwareRoundRobinPolicy( local_dc="datacenter1"), auth_provider=AUTH_PROVIDER, protocol_version=4) # db hosts and port to start clusters cls.__cluster.connection_class = LibevConnection # use libev event loop cls.__session = cls.__cluster.connect( ) # connect to hosts - cls is server.db.init cls.__session.execute( f"CREATE KEYSPACE IF NOT EXISTS {KEYSPACE} WITH REPLICATION = {REPLICATION};" ) cls.__session.execute(f'USE {KEYSPACE}') connection.register_connection( 'DML', session=cls.__session) # DML cluster for sync_table ops return super(init, cls).__new__(cls, *args, **kwargs) except Exception as e: typer.secho(f"➲ [Could not initialize database] ::: {e}", fg=typer.colors.RED, bold=True)
def setUpClass(cls): super(ManagementConnectionTests, cls).setUpClass() conn.unregister_connection('default') conn.register_connection('fake_cluster', ['127.0.0.100'], lazy_connect=True, retry_connect=True, default=True) conn.register_connection('cluster', ['127.0.0.1'])
def setUpClass(cls): super(UsingDescriptorTests, cls).setUpClass() conn.unregister_connection('default') conn.register_connection('fake_cluster', ['127.0.0.100'], lazy_connect=True, retry_connect=True, default=True) conn.register_connection('cluster', [CASSANDRA_IP])