def test_connection_param_validation(self):
"""
Test to validate that invalid parameter combinations for registering connections via session are not tolerated
@since 3.8
@jira_ticket PYTHON-649
@expected_result queries should execute appropriately
@test_category object_mapper
"""
cluster = Cluster([CASSANDRA_IP])
session = cluster.connect()
with self.assertRaises(CQLEngineException):
conn.register_connection("bad_coonection1", session=session, consistency="not_null")
with self.assertRaises(CQLEngineException):
conn.register_connection("bad_coonection2", session=session, lazy_connect="not_null")
with self.assertRaises(CQLEngineException):
conn.register_connection("bad_coonection3", session=session, retry_connect="not_null")
with self.assertRaises(CQLEngineException):
conn.register_connection("bad_coonection4", session=session, cluster_options="not_null")
with self.assertRaises(CQLEngineException):
conn.register_connection("bad_coonection5", hosts="not_null", session=session)
cluster.shutdown()
cluster.shutdown()
cluster.shutdown()
def mapper_step2(stream):
"""This mapper receives records from the SuperSpecificCartesianInputFormat, aka the cartesian product
of the morning flights and +2 days afternoon flights. It simply have to reduce to the input where
dest1 == origin2.
"""
from cassandra.cluster import Cluster
from cassandra.cqlengine import connection
from decimal import Decimal
from cassandra.cqlengine import columns
from cassandra.cqlengine.models import Model
from setup_cassandradb import Q3Entry
cluster = Cluster(['node7', 'node8', 'node9'])
connection.register_connection('con',
session=cluster.connect(),
default=True)
for first_leg, second_leg in mapred.iter_key_values(stream):
date1, origin1, dest1, delay1, time1, carrier1, flight1 = first_leg
date2, origin2, dest2, delay2, time2, carrier2, flight2 = second_leg
if dest1 == origin2:
total_delay = float(delay1) + float(delay2)
Q3Entry.create(date_origin_dest1='_'.join([date1, origin1, dest1]),
date1=date1,
origin=origin1,
dest1=dest1,
dest2=dest2,
total_delay=Decimal(total_delay),
datetime1=date1 + ' ' + time1,
flight1=carrier1 + ' ' + flight1,
datetime2=date2 + ' ' + time2,
flight2=carrier2 + ' ' + flight2)
def connect(self):
auth_provider = PlainTextAuthProvider(
username=self.username, password=self.password
)
self.cluster = Cluster(
["node_0", "node_1"],
auth_provider=auth_provider,
# executor_threads=int(os.getenv('CASSANDRA_EXECUTOR_THREADS')),
# protocol_version=int(os.getenv('CASSANDRA_PROTOCOL_VERSION')),
)
self.session = self.cluster.connect()
self.session.execute(
"""
CREATE KEYSPACE IF NOT EXISTS test_key_space
WITH REPLICATION =
{ 'class' : 'SimpleStrategy', 'replication_factor': '3'}
AND durable_writes = true;
"""
)
self.session.set_keyspace(self.key_space)
register_connection(str(self.session), session=self.session)
set_default_connection(str(self.session))
def before_server_start(app, _):
cluster = Cluster([CLUSTER_HOST])
session = cluster.connect()
session.execute("""
CREATE KEYSPACE IF NOT EXISTS %s
WITH REPLICATION = {'class' : 'SimpleStrategy', 'replication_factor' : 1};
""" % CLUSTER_KEY_SPACE)
session = cluster.connect(keyspace=CLUSTER_KEY_SPACE)
aiosession(session)
# session.execute(f"DROP KEYSPACE {CLUSTER_KEY_SPACE}")
connection.register_connection(CLUSTER_NAME, session=session)
create_keyspace_simple(CLUSTER_KEY_SPACE, 1, connections=[CLUSTER_NAME])
connection.setup([CLUSTER_HOST], CLUSTER_KEY_SPACE)
setup_db()
app.db_session = session
logger = logging.getLogger()
logger.setLevel(DEBUG_LEVEL)
handler = logging.StreamHandler()
handler.setFormatter(logging.Formatter(LOGGER_FORMAT))
logger.addHandler(handler)
app.log = logger
def test_connection_param_validation(self):
"""
Test to validate that invalid parameter combinations for registering connections via session are not tolerated
@since 3.8
@jira_ticket PYTHON-649
@expected_result queries should execute appropriately
@test_category object_mapper
"""
session = Cluster(['127.0.0.1']).connect()
with self.assertRaises(CQLEngineException):
conn.register_connection("bad_coonection1",
session=session,
consistency="not_null")
with self.assertRaises(CQLEngineException):
conn.register_connection("bad_coonection2",
session=session,
lazy_connect="not_null")
with self.assertRaises(CQLEngineException):
conn.register_connection("bad_coonection3",
session=session,
retry_connect="not_null")
with self.assertRaises(CQLEngineException):
conn.register_connection("bad_coonection4",
session=session,
cluster_options="not_null")
with self.assertRaises(CQLEngineException):
conn.register_connection("bad_coonection5",
hosts="not_null",
session=session)
def setUpClass(cls):
super(ManagementConnectionTests, cls).setUpClass()
conn.unregister_connection('default')
conn.register_connection('fake_cluster', ['127.0.0.100'],
lazy_connect=True,
retry_connect=True,
default=True)
conn.register_connection('cluster', [CASSANDRA_IP])
def setUpClass(cls):
conn.register_connection('cluster', [CASSANDRA_IP])
test_queryset.TestModel.__queryset__ = ModelQuerySetNew
test_queryset.IndexedTestModel.__queryset__ = ModelQuerySetNew
test_queryset.IndexedCollectionsTestModel.__queryset__ = ModelQuerySetNew
test_queryset.TestMultiClusteringModel.__queryset__ = ModelQuerySetNew
super(BaseConnectionTestNoDefault, cls).setUpClass()
conn.unregister_connection('default')
def connection_factory():
contact_point = os.environ['CASSANDRA_CONTACT_POINT']
cluster = Cluster([contact_point])
connection = cluster.connect(keyspace=os.environ['CASSANDRA_KEYSPACE'])
register_connection(str(connection), session=connection)
set_default_connection(str(connection))
return connection
def db_set(keyspace):
try:
connection.set_default_connection(CONN_NM)
except Exception as err:
print('db set exception occured...')
connection.unregister_connection(CONN_NM)
connection.register_connection(CONN_NM, config.CASSANDRA_CONTACT_POINTS[config.DTP], default=True)
connection.session.set_keyspace(keyspace)
print('...fix db connection. return to process')
def setUpClass(cls):
super(UsingDescriptorTests, cls).setUpClass()
conn.unregister_connection('default')
conn.register_connection('fake_cluster', ['127.0.0.100'],
lazy_connect=True,
retry_connect=True,
default=True)
conn.register_connection('cluster', ['127.0.0.1'])
def setUpClass(cls):
conn.register_connection('cluster', ['127.0.0.1'])
test_queryset.TestModel.__queryset__ = ModelQuerySetNew
test_queryset.IndexedTestModel.__queryset__ = ModelQuerySetNew
test_queryset.IndexedCollectionsTestModel.__queryset__ = ModelQuerySetNew
test_queryset.TestMultiClusteringModel.__queryset__ = ModelQuerySetNew
super(BaseConnectionTestNoDefault, cls).setUpClass()
conn.unregister_connection('default')
def setUpClass(cls):
super(ContextQueryConnectionTests, cls).setUpClass()
create_keyspace_simple('ks1', 1)
conn.unregister_connection('default')
conn.register_connection('fake_cluster', ['1.2.3.4'], lazy_connect=True, retry_connect=True, default=True)
conn.register_connection('cluster', [CASSANDRA_IP])
with ContextQuery(TestModel, connection='cluster') as tm:
sync_table(tm)
def setUpClass(cls):
super(ContextQueryConnectionTests, cls).setUpClass()
create_keyspace_simple('ks1', 1)
conn.unregister_connection('default')
conn.register_connection('fake_cluster', ['127.0.0.100'], lazy_connect=True, retry_connect=True, default=True)
conn.register_connection('cluster', ['127.0.0.1'])
with ContextQuery(TestModel, connection='cluster') as tm:
sync_table(tm)
def pool_initializer():
from nexustiles.nexustiles import NexusTileService
global tile_service
tile_service = NexusTileService()
# TODO This is a hack to make sure each sub-process uses it's own connection to cassandra. data-access needs to be updated
from cassandra.cqlengine import connection
from multiprocessing import current_process
connection.register_connection(current_process().name, [host.address for host in connection.get_session().hosts])
connection.set_default_connection(current_process().name)
def setUpClass(cls):
super(BatchQueryConnectionTests, cls).setUpClass()
create_keyspace_simple('ks1', 1)
sync_table(TestModel)
sync_table(AnotherTestModel)
conn.unregister_connection('default')
conn.register_connection('fake_cluster', ['127.0.0.100'], lazy_connect=True, retry_connect=True, default=True)
conn.register_connection('cluster', [CASSANDRA_IP])
def setUpClass(cls):
super(BatchQueryConnectionTests, cls).setUpClass()
create_keyspace_simple('ks1', 1)
sync_table(TestModel)
sync_table(AnotherTestModel)
conn.unregister_connection('default')
conn.register_connection('fake_cluster', ['127.0.0.100'], lazy_connect=True, retry_connect=True, default=True)
conn.register_connection('cluster', ['127.0.0.1'])
def drop_keyspace(cls, nodes=None, keyspace='results_database', auth_provider=None):
nodes = nodes if nodes else ['localhost']
connection_id = uuid.uuid4()
try:
register_connection(name=str(connection_id), session=Cluster(nodes, auth_provider=auth_provider).connect())
does_keyspace_exist = keyspace in get_cluster(str(connection_id)).metadata.keyspaces
if does_keyspace_exist:
drop_keyspace(keyspace, connections=[str(connection_id)])
finally:
unregister_connection(name=str(connection_id))
def setup_connection() -> None:
"""
Setup the connection that the mapper should use. This should be called when
the api starts.
"""
cluster_session = get_cluster().connect()
connection.register_connection('cluster',
session=cluster_session,
default=True)
def init(sess):
conn.register_connection("swapi", session=sess, default=True)
mgmt.create_keyspace_simple("swapi",
replication_factor=1,
durable_writes=True)
mgmt.sync_table(Films)
mgmt.sync_table(People)
mgmt.sync_table(Planets)
mgmt.sync_table(Species)
mgmt.sync_table(Starships)
mgmt.sync_table(Vehicles)
def _create_cluster_with_cp_options(cls, name, cp_options):
execution_profiles = {
EXEC_PROFILE_DEFAULT:
ExecutionProfile(continuous_paging_options=cp_options)
}
cls.cluster_default = Cluster(protocol_version=cls.protocol_version,
execution_profiles=execution_profiles)
cls.session_default = cls.cluster_default.connect(
wait_for_all_pools=True)
connection.register_connection(name,
default=True,
session=cls.session_default)
cls.connections.add(name)
def test_connection_from_hosts(self):
"""
Test to ensure that you can register a connection from a list of hosts
@since 3.8
@jira_ticket PYTHON-692
@expected_result queries should execute appropriately
@test_category object_mapper
"""
connection_name = 'from_hosts'
conn.register_connection(connection_name, hosts=[CASSANDRA_IP])
self.assertIsNotNone(conn.get_connection(connection_name).cluster.metadata.get_host(CASSANDRA_IP))
self.addCleanup(conn.unregister_connection, connection_name)
def test_connection_creation_from_session(self):
"""
Test to ensure that you can register a connection from a session
@since 3.8
@jira_ticket PYTHON-649
@expected_result queries should execute appropriately
@test_category object_mapper
"""
session = Cluster(['127.0.0.1']).connect()
connection_name = 'from_session'
conn.register_connection(connection_name, session=session)
self.addCleanup(conn.unregister_connection, connection_name)
def test_connection_from_hosts(self):
"""
Test to ensure that you can register a connection from a list of hosts
@since 3.8
@jira_ticket PYTHON-692
@expected_result queries should execute appropriately
@test_category object_mapper
"""
connection_name = 'from_hosts'
conn.register_connection(connection_name, hosts=[CASSANDRA_IP])
self.assertIsNotNone(conn.get_connection(connection_name).cluster.metadata.get_host(CASSANDRA_IP))
self.addCleanup(conn.unregister_connection, connection_name)
def connect(self):
auth_provider = PlainTextAuthProvider(username='******',
password='******')
self.cluster = Cluster(
['node_0', 'node_1'],
auth_provider=auth_provider,
# executor_threads=int(os.getenv('CASSANDRA_EXECUTOR_THREADS')),
# protocol_version=int(os.getenv('CASSANDRA_PROTOCOL_VERSION')),
)
self.session = self.cluster.connect()
register_connection(str(self.session), session=self.session)
set_default_connection(str(self.session))
def test_connection_creation_from_session(self):
"""
Test to ensure that you can register a connection from a session
@since 3.8
@jira_ticket PYTHON-649
@expected_result queries should execute appropriately
@test_category object_mapper
"""
cluster = Cluster([CASSANDRA_IP])
session = cluster.connect()
connection_name = 'from_session'
conn.register_connection(connection_name, session=session)
self.assertIsNotNone(conn.get_connection(connection_name).cluster.metadata.get_host(CASSANDRA_IP))
self.addCleanup(conn.unregister_connection, connection_name)
cluster.shutdown()
def test_connection_creation_from_session(self):
"""
Test to ensure that you can register a connection from a session
@since 3.8
@jira_ticket PYTHON-649
@expected_result queries should execute appropriately
@test_category object_mapper
"""
cluster = Cluster([CASSANDRA_IP])
session = cluster.connect()
connection_name = 'from_session'
conn.register_connection(connection_name, session=session)
self.assertIsNotNone(conn.get_connection(connection_name).cluster.metadata.get_host(CASSANDRA_IP))
self.addCleanup(conn.unregister_connection, connection_name)
cluster.shutdown()
def setup(self):
if not self.keyspace or not isinstance(self.keyspace, text_type):
self.keyspace = self.container.service_name
cluster_options = dict(
reconnection_policy=ExponentialReconnectionPolicy(base_delay=3,
max_delay=15),
connection_class=EventletConnection)
if self.models:
cluster_options['protocol_version'] = 3
os.environ.setdefault('CQLENG_ALLOW_SCHEMA_MANAGEMENT', '1')
self.connection = register_connection(
name=self.connection_name,
hosts=self.nodes,
lazy_connect=False,
retry_connect=True,
cluster_options=cluster_options,
default=self.connection_name == DEFAULT_CONNECTION_NAME)
topology = self.cassandra_config.get('topology', SIMPLE_KEYSPACE)
if topology not in (SIMPLE_KEYSPACE, NETWORK_KEYSPACE):
topology = SIMPLE_KEYSPACE
topology_func = {
SIMPLE_KEYSPACE: management.create_keyspace_simple,
NETWORK_KEYSPACE: management.create_keyspace_network_topology
}.get(topology)
topology_func(**self.topology_options(topology))
if self.session:
self.session.set_keyspace(self.keyspace)
for model in self.models:
sync_table(model, [self.keyspace], [self.connection])
def cqlengine_management(cassandra):
"""Setup session for cqlengine
"""
# create test keyspace
os.environ['CQLENG_ALLOW_SCHEMA_MANAGEMENT'] = 'true'
test_keyspace = 'test_async_cqlengine'
cqlengine_connection.register_connection(
'cqlengine', session=cassandra, default=True)
management.create_keyspace_simple(test_keyspace, replication_factor=1)
# setup cqlengine session
cassandra.set_keyspace(test_keyspace)
cqlengine_connection.set_session(cassandra)
yield management
management.drop_keyspace(test_keyspace)
cqlengine_connection.unregister_connection('cqlengine')
def connect(self):
""" DB connection."""
auth_provider = PlainTextAuthProvider(username=os.getenv('CASSANDRA_USERNAME'),
password=os.getenv('CASSANDRA_PASSWORD'))
self.cluster = Cluster(
[os.getenv('CASSANDRA_HOST')],
load_balancing_policy=TokenAwarePolicy(DCAwareRoundRobinPolicy()),
port=os.getenv('CASSANDRA_PORT'),
auth_provider=auth_provider,
executor_threads=2,
protocol_version=4,
)
self.session = self.cluster.connect()
register_connection(str(self.session), session=self.session)
set_default_connection(str(self.session))
def load_db(stream):
from cassandra.cluster import Cluster
from cassandra.cqlengine import connection
from decimal import Decimal
from cassandra.cqlengine import columns
from cassandra.cqlengine.models import Model
from setup_cassandradb import Q22Entry
cluster = Cluster(['node7', 'node8', 'node9'])
connection.register_connection('con',
session=cluster.connect(),
default=True)
for (origin, dest), (delay, _) in mapred.iter_key_values(stream):
Q22Entry.create(origin=origin, dest=dest, delay=Decimal(delay))
def __new__(cls, *args, **kwargs):
try:
cls.__cluster = Cluster(
auth_provider=AUTH_PROVIDER,
cloud=CLOUD_CONFIG,
load_balancing_policy=DCAwareRoundRobinPolicy(
local_dc="dc-1")) # db hosts and port to start clusters
cls.__cluster.connection_class = LibevConnection # use libev event loop
cls.__session = cls.__cluster.connect(
) # connect to hosts - cls is server.db.init
cls.__session.execute(f'USE {KEYSPACE}')
connection.register_connection(
'DML', session=cls.__session) # DML cluster for sync_table ops
return super(init, cls).__new__(cls, *args, **kwargs)
except Exception as e:
typer.secho(f"➲ [Could not initialize database] ::: {e}",
fg=typer.colors.RED,
bold=True)
def create_session():
cluster = Cluster()
session = cluster.connect()
# Create keyspace, if already have keyspace your can skip this
os.environ['CQLENG_ALLOW_SCHEMA_MANAGEMENT'] = 'true'
connection.register_connection('cqlengine', session=session, default=True)
management.create_keyspace_simple('example', replication_factor=1)
management.sync_table(User, keyspaces=['example'])
# Run the example function in asyncio loop
loop = asyncio.get_event_loop()
# Wrap cqlengine connection with aiosession
aiosession_for_cqlengine(session, loop=loop)
session.set_keyspace('example')
connection.set_session(session)
return session, loop
async def setup_cassandra(app, loop):
logger.warning('Initializing Database')
cluster = Cluster(['cassandra_seed_node'])
session = cluster.connect()
session.execute(
"""CREATE KEYSPACE IF NOT EXISTS soda WITH replication = {'class': 'SimpleStrategy', 'replication_factor': '1'} AND durable_writes = true;"""
)
session.execute('USE soda')
register_connection(str(session), session=session)
set_default_connection(str(session))
sync_table(User, keyspaces=['soda'])
sync_table(Record, keyspaces=['soda'])
sync_table(Project, keyspaces=['soda'])
aiosession(session)
app.db = session
return app
async def connect_to_cassandra(app):
config = app['config']
sleep = config.get('sleep', 0)
max_retry = config.get('max_retry', 3)
for i in range(max_retry):
try:
session = Cluster(config['cassandra_host']).connect()
except NoHostAvailable:
if i == (max_retry - 1):
msg = (
'Failed to connect to cassandra, retrying '
'in %ss (%s/%s)'
)
await app['logger'].error('main.connect_to_cassndra',
msg % (sleep, i + 1, max_retry))
raise
else:
msg = (
'Failed to connect to cassandra, retrying '
'in %ss (%s/%s)'
)
await app['logger'].debug('main.connect_to_cassndra',
msg % (sleep, i + 1, max_retry))
time.sleep(sleep)
connection.register_connection(
config['default_connection'],
session=session
)
create_keyspace_simple(
config['default_keyspace'],
replication_factor=1,
connections=[config['default_connection']]
)
User.__keyspace__ = config['default_keyspace']
User.__connection__ = config['default_connection']
sync_table(
User,
keyspaces=[config['default_keyspace']],
connections=[config['default_connection']]
)
def __init__(self):
cloud_config= {
'secure_connect_bundle': './private/secure-connect-CoursesDB.zip'
}
user = None
password = None
with open('./private/CassandraPassword.json') as f:
data = json.load(f)
user = data['user']
password = data['password']
auth_provider = PlainTextAuthProvider(user, password)
cluster = Cluster(cloud=cloud_config, auth_provider=auth_provider)
self.session = cluster.connect()
self.session.execute("USE {}".format(_keyspace))
_session = self.session
register_connection(str(_session), session=_session)
set_default_connection(str(_session))
def __new__(cls, *args, **kwargs):
try:
cls.__cluster = Cluster(
contact_points=HOSTS,
port=PORT,
load_balancing_policy=DCAwareRoundRobinPolicy(
local_dc="datacenter1"),
auth_provider=AUTH_PROVIDER,
protocol_version=4) # db hosts and port to start clusters
cls.__cluster.connection_class = LibevConnection # use libev event loop
cls.__session = cls.__cluster.connect(
) # connect to hosts - cls is server.db.init
cls.__session.execute(
f"CREATE KEYSPACE IF NOT EXISTS {KEYSPACE} WITH REPLICATION = {REPLICATION};"
)
cls.__session.execute(f'USE {KEYSPACE}')
connection.register_connection(
'DML', session=cls.__session) # DML cluster for sync_table ops
return super(init, cls).__new__(cls, *args, **kwargs)
except Exception as e:
typer.secho(f"➲ [Could not initialize database] ::: {e}",
fg=typer.colors.RED,
bold=True)
def setUpClass(cls):
super(ManagementConnectionTests, cls).setUpClass()
conn.unregister_connection('default')
conn.register_connection('fake_cluster', ['127.0.0.100'], lazy_connect=True, retry_connect=True, default=True)
conn.register_connection('cluster', ['127.0.0.1'])
def setUpClass(cls):
super(UsingDescriptorTests, cls).setUpClass()
conn.unregister_connection('default')
conn.register_connection('fake_cluster', ['127.0.0.100'], lazy_connect=True, retry_connect=True, default=True)
conn.register_connection('cluster', [CASSANDRA_IP])
