def create_keyspace(self):
cluster.max_schema_agreement_wait = 0
setup_cass(self.seeds, 'system')
self.session = get_session()
set_session(self.session)
create_keyspace_simple(name=self.keyspace, replication_factor=3)
self.logger.debug("ks created")
def test_uncommitted_session_uses_legacy(self):
cluster = Cluster()
session = cluster.connect()
session.row_factory = dict_factory
connection.set_session(session)
conn = connection.get_connection()
self.assertEqual(conn.cluster._config_mode, _ConfigMode.LEGACY)
def __init__(self, manager):
self.manager = manager
settings = manager.settings
cluster_ips = settings.get('CASSANDRABACKEND_CLUSTER_IPS') # Format: ['192.168.0.1', '192.168.0.2']
cluster_port = settings.get('CASSANDRABACKEND_CLUSTER_PORT')
keyspace = settings.get('CASSANDRABACKEND_KEYSPACE')
keyspace_create = settings.get('CASSANDRABACKEND_CREATE_KEYSPACE_IF_NOT_EXISTS') # Default: true
models = settings.get('CASSANDRABACKEND_MODELS')
self.cluster = Cluster(cluster_ips, cluster_port)
self.models = dict([(name, load_object(klass)) for name, klass in models.items()])
self.session = self.cluster.connect()
self.session.row_factory = dict_factory
if keyspace_create:
query = """CREATE KEYSPACE IF NOT EXISTS \"%s\"
WITH replication = {'class': 'SimpleStrategy', 'replication_factor' : 3}""" % (keyspace, )
self.session.execute(query)
self.session.set_keyspace(keyspace)
connection.set_session(self.session)
self._metadata = None
self._queue = None
self._states = None
def connect(self):
"""连接数据库
"""
if self._is_connected:
return
# auth = PlainTextAuthProvider(username=self.config.CASSANDRA_USER,
# password=self.config.CASSANDRA_PASSWORD)
# cluster = Cluster(self.config.CASSANDRA_NODES, auth_provider=auth)
cluster = Cluster(self.config.CASSANDRA_NODES)
try:
self.db_session = cluster.connect()
except NoHostAvailable:
# 再尝试连接一次
self.db_session = cluster.connect()
self.db_session.set_keyspace(self.keyspace)
self.db_session.row_factory = dict_factory
self.db_session.default_timeout = self.timeout
self.db_session.default_consistency_level = \
ConsistencyLevel.LOCAL_QUORUM
cqlengine_connection.set_session(self.db_session)
aiosession(self.db_session)
self._is_connected = True
def test_connection_from_session_with_legacy_settings(self):
cluster = Cluster(load_balancing_policy=RoundRobinPolicy())
session = cluster.connect()
session.row_factory = dict_factory
connection.set_session(session)
conn = connection.get_connection()
self.assertEqual(conn.cluster._config_mode, _ConfigMode.LEGACY)
def create_keyspace(self):
setup_cass(self.seeds, 'system')
self.session = get_session()
set_session(self.session)
drop_keyspace(self.keyspace)
create_keyspace_simple(name=self.keyspace, replication_factor=3)
self.logger.debug("ks created")
def create_network_keyspace(self):
cluster.max_schema_agreement_wait = 0
setup_cass(self.seeds, 'system')
self.session = get_session()
set_session(self.session)
dc_map = {'DC1-Data': 3, 'DC1-Analytics': 3}
create_keyspace_network_topology(name=self.keyspace, dc_replication_map=dc_map)
create_keyspace_simple(name=self.keyspace, replication_factor=3)
self.logger.debug("ks network topo created")
def test_set_session_without_existing_connection(self):
"""
Users can set the default session without having a default connection set.
"""
mock_session = Mock(
row_factory=dict_factory,
encoder=Mock(mapping={})
)
connection.set_session(mock_session)
def test_set_session_without_existing_connection(self):
"""
Users can set the default session without having a default connection set.
"""
mock_session = Mock(
row_factory=dict_factory,
encoder=Mock(mapping={})
)
connection.set_session(mock_session)
def create_keyspace(session, keyspace):
connection.set_session(session)
try:
management.create_keyspace_simple(
keyspace["name"], replication_factor=keyspace["factor"])
except cassandra.AlreadyExists:
pass
session.set_keyspace(keyspace["name"])
connection.set_session(session)
return session
def __init__(self, host: str = 'localhost', port: int = 9042):
self.cluster: Cluster = Cluster([host], port=port)
self.session: Session = self.cluster.connect()
connection.set_session(self.session)
management.create_keyspace_simple(UserProfile.__keyspace__,
replication_factor=1)
connection.session.set_keyspace(UserProfile.__keyspace__)
# Creates tables if non existent
management.sync_table(UserProfile)
def test_connection_from_session_with_execution_profile(self):
cluster = Cluster(
execution_profiles={
EXEC_PROFILE_DEFAULT: ExecutionProfile(
row_factory=dict_factory)
})
session = cluster.connect()
connection.default()
connection.set_session(session)
conn = connection.get_connection()
self.assertEqual(conn.cluster._config_mode, _ConfigMode.PROFILES)
def __init__(self, seeds, keyspace):
self.seeds = seeds
self.keyspace = keyspace
# TODO for metrics
# setup_cass(self.seeds, self.keyspace,
# consistency=ConsistencyLevel.TWO, lazy_connect=False,
# retry_connect=True, metrics_enabled=True)
setup_cass(self.seeds, self.keyspace, consistency=ConsistencyLevel.TWO, lazy_connect=False, retry_connect=True)
self.session = get_session()
set_session(self.session)
self.cluster = get_cluster()
def main():
# Apache Kafka connection
consumer = Consumer(conf)
# Apache Cassandra connection
list_of_ip = (['192.168.56.101', '192.168.56.102', '192.168.56.103'])
cluster = Cluster(list_of_ip,
load_balancing_policy=TokenAwarePolicy(
RoundRobinPolicy()))
session = cluster.connect()
session.set_keyspace('thirdeye_test')
connection.set_session(session)
basic_consume_loop(consumer, ["airline_raw"], connection)
def test_set_session_without_existing_connection(self):
"""
Users can set the default session without having a default connection set.
"""
mock_cluster = Mock(
_config_mode=_ConfigMode.LEGACY,
)
mock_session = Mock(
row_factory=dict_factory,
encoder=Mock(mapping={}),
cluster=mock_cluster,
)
connection.set_session(mock_session)
def __init__(self, seeds, keyspace):
self.seeds = seeds
self.keyspace = keyspace
# TODO configure ConsitencyLevel
setup_cass(self.seeds, self.keyspace,
consistency=ConsistencyLevel.TWO, lazy_connect=False,
retry_connect=True, metrics_enabled=True)
#setup_cass(self.seeds, self.keyspace, consistency=ConsistencyLevel.ONE, lazy_connect=False, retry_connect=True)
self.session = get_session()
set_session(self.session)
self.cluster = get_cluster()
self.logger = logging.getLogger('pet_race_job')
def __init__(self, manager):
self.manager = manager
settings = manager.settings
cluster_ips = settings.get('CASSANDRABACKEND_CLUSTER_IPS')
cluster_port = settings.get('CASSANDRABACKEND_CLUSTER_PORT')
drop_all_tables = settings.get('CASSANDRABACKEND_DROP_ALL_TABLES')
keyspace = settings.get('CASSANDRABACKEND_KEYSPACE')
keyspace_create = settings.get('CASSANDRABACKEND_CREATE_KEYSPACE_IF_NOT_EXISTS')
models = settings.get('CASSANDRABACKEND_MODELS')
crawl_id = settings.get('CASSANDRABACKEND_CRAWL_ID')
generate_stats = settings.get('CASSANDRABACKEND_GENERATE_STATS')
self.models = dict([(name, load_object(klass)) for name, klass in models.items()])
self.cluster = Cluster(
contact_points=cluster_ips,
port=cluster_port,
compression=True,
default_retry_policy=RetryPolicy(),
reconnection_policy=ConstantReconnectionPolicy(10, 100)
)
self.session = self.cluster.connect()
self.session.row_factory = dict_factory
self.session.encoder.mapping[dict] = self.session.encoder.cql_encode_map_collection
self.crawl_id = crawl_id
self.generate_stats = generate_stats
if keyspace_create:
query = """CREATE KEYSPACE IF NOT EXISTS \"%s\"
WITH replication = {'class': 'SimpleStrategy', 'replication_factor' : 3}""" % (keyspace, )
self.session.execute(query)
self.session.set_keyspace(keyspace)
connection.set_session(self.session)
if drop_all_tables:
for key, value in self.models.iteritems():
drop_table(value)
for key, value in self.models.iteritems():
if (self.generate_stats is False and key != 'CrawlStatsModel') or self.generate_stats is True:
sync_table(value)
self._metadata = Metadata(self.session, self.models['MetadataModel'], self.crawl_id, self.generate_stats)
self._states = States(self.session, self.models['StateModel'],
settings.get('STATE_CACHE_SIZE_LIMIT'), self.crawl_id)
self._queue = self._create_queue(settings)
def test_execution_profile_insert_query(self):
cluster = Cluster(
execution_profiles={
EXEC_PROFILE_DEFAULT: ExecutionProfile(
row_factory=dict_factory)
})
session = cluster.connect()
connection.default()
connection.set_session(session)
self.assertEqual(connection.get_connection().cluster._config_mode,
_ConfigMode.PROFILES)
sync_table(ConnectionModel)
ConnectionModel.objects.create(key=0, some_data='text0')
ConnectionModel.objects.create(key=1, some_data='text1')
self.assertEqual(ConnectionModel.objects(key=0)[0].some_data, 'text0')
def cqlengine_management(cassandra):
"""Setup session for cqlengine
"""
# create test keyspace
os.environ['CQLENG_ALLOW_SCHEMA_MANAGEMENT'] = 'true'
test_keyspace = 'test_async_cqlengine'
cqlengine_connection.register_connection(
'cqlengine', session=cassandra, default=True)
management.create_keyspace_simple(test_keyspace, replication_factor=1)
# setup cqlengine session
cassandra.set_keyspace(test_keyspace)
cqlengine_connection.set_session(cassandra)
yield management
management.drop_keyspace(test_keyspace)
cqlengine_connection.unregister_connection('cqlengine')
def test_empty_columns_with_cqlengine(self):
self._prime_testtable_query()
self.cluster = Cluster(protocol_version=PROTOCOL_VERSION)
self.session = self.cluster.connect(wait_for_all_pools=True)
set_session(self.session)
class TestModel(Model):
__keyspace__ = 'testks'
__table_name__ = 'testtable'
empty = columns.Text(db_field='', primary_key=True)
space = columns.Text(db_field=' ')
self.assertEqual([TestModel(empty='testval', space='testval1')],
list(
TestModel.objects.only(['empty', 'space']).all()))
def test_empty_columns_with_cqlengine(self):
self._prime_testtable_query()
self.cluster = Cluster(protocol_version=PROTOCOL_VERSION)
self.session = self.cluster.connect(wait_for_all_pools=True)
set_session(self.session)
class TestModel(Model):
__keyspace__ = 'testks'
__table_name__ = 'testtable'
empty = columns.Text(db_field='', primary_key=True)
space = columns.Text(db_field=' ')
self.assertEqual(
[TestModel(empty='testval', space='testval1')],
list(TestModel.objects.only(['empty', 'space']).all())
)
def create_session():
cluster = Cluster()
session = cluster.connect()
# Create keyspace, if already have keyspace your can skip this
os.environ['CQLENG_ALLOW_SCHEMA_MANAGEMENT'] = 'true'
connection.register_connection('cqlengine', session=session, default=True)
management.create_keyspace_simple('example', replication_factor=1)
management.sync_table(User, keyspaces=['example'])
# Run the example function in asyncio loop
loop = asyncio.get_event_loop()
# Wrap cqlengine connection with aiosession
aiosession_for_cqlengine(session, loop=loop)
session.set_keyspace('example')
connection.set_session(session)
return session, loop
def connect(self):
"""连接数据库
"""
if self._is_connected:
return
auth = PlainTextAuthProvider(username=config.CASSANDRA_USER,
password=config.CASSANDRA_PASSWORD)
cluster = Cluster(config.CASSANDRA_NODES, auth_provider=auth)
self.db_session = cluster.connect()
self.db_session.set_keyspace(self.keyspace)
self.db_session.row_factory = dict_factory
self.db_session.default_timeout = self.timeout
cqlengine_connection.set_session(self.db_session)
aiosession(self.db_session)
self._is_connected = True
def __init__(self):
global KEYSPACE
self.cluster = Cluster(["10.244.35.35"])
print("start a cluster")
self.session = self.cluster.connect(KEYSPACE)
#self.session.execute("CREATE KEYSPACE IF NOT EXISTS demo WITH REPLICATION = \
# { 'class' : 'SimpleStrategy', 'replication_factor' : 1 }")
#print("create demo keyspace.")
self.session.default_timeout = 30
self.session.set_keyspace(KEYSPACE)
print("start a demo session")
''' config cqlengine connection'''
self.session.row_factory = dict_factory
connection.set_session(self.session)
print("cassandra setup completed.")
def connect(self):
"""连接数据库
"""
if self._is_connected:
return
cluster = Cluster(self.config.CASSANDRA_NODES)
try:
self.db_session = cluster.connect()
except NoHostAvailable:
# 再尝试连接一次
self.db_session = cluster.connect()
self.db_session.set_keyspace(self.keyspace)
self.db_session.row_factory = dict_factory
self.db_session.default_timeout = self.timeout
cqlengine_connection.set_session(self.db_session)
aiosession(self.db_session)
self._is_connected = True
def main(settings):
cluster = Cluster([settings['cassandra.cluster']])
session = cluster.connect()
session.row_factory = dict_factory
connection.set_session(session)
allow_migration = not settings['cassandra.immutable_schema']
# Prevent CQL engine migration warnings.
os.environ['CQLENG_ALLOW_SCHEMA_MANAGEMENT'] = str(allow_migration)
if allow_migration:
migrate_keyspaces(settings)
session.set_keyspace(settings['cassandra.keyspace_default'])
connection.set_session(session)
if allow_migration:
migrate_tables(settings)
logger.info("Starting consumer async loop.")
return async_start(settings, session)
def init_app(self, app):
"""Bind the CQLAlchemy object to the app.
This method set all the config options for the connection to
the Cassandra cluster and creates a connection at startup.
"""
self._hosts_ = app.config.get('CASSANDRA_HOSTS', '')
self._keyspace_ = app.config.get('CASSANDRA_KEYSPACE', '')
self._username = app.config.get('CASSANDRA_USERNAME', '')
self._password = app.config.get('CASSANDRA_PASSWORD', '')
self._cloud_bundle = app.config.get('ASTRA_SECURE_CONNECT_BUNDLE', '')
consistency = app.config.get('CASSANDRA_CONSISTENCY', 1)
lazy_connect = app.config.get('CASSANDRA_LAZY_CONNECT', False)
retry_connect = app.config.get('CASSANDRA_RETRY_CONNECT', False)
setup_kwargs = app.config.get('CASSANDRA_SETUP_KWARGS', {})
self._auth_provider = None
if self._username and self._password:
self._auth_provider = PlainTextAuthProvider(
username=self._username, password=self._password)
if self._cloud_bundle != None and self._auth_provider != None:
cloud = {'secure_connect_bundle': self._cloud_bundle}
cluster = Cluster(cloud=cloud, auth_provider=self._auth_provider)
session = cluster.connect(self._keyspace_)
connection.set_session(session)
return
if not self._hosts_ and self._keyspace_:
raise NoConfig("""No Configuration options defined.
At least CASSANDRA_HOSTS and CASSANDRA_CONSISTENCY
must be supplied""")
connection.setup(self._hosts_,
self._keyspace_,
consistency=consistency,
lazy_connect=lazy_connect,
retry_connect=retry_connect,
**setup_kwargs)
class AlunoModel(Model):
'''
Classe responsável por realizar a interação com a base de dados.
'''
__table_name__ = "aluno"
connection.set_session(carrega_sessao())
id = columns.Integer(primary_key=True)
nome = columns.Text(index=True, required=True)
telefone = columns.Integer(required=True)
email = columns.Text(required=True)
cidade_est = columns.Text(required=True)
curso_matriculado_atual = columns.Text(required=True)
turma = columns.Text(required=True)
dt_inicio = columns.Text(required=True)
dt_conclusao_prevista = columns.Text(required=True)
def test_connection_session_switch(self):
"""
Test to ensure that when the default keyspace is changed in a session and that session,
is set in the connection class, that the new defaul keyspace is honored.
@since 3.1
@jira_ticket PYTHON-486
@expected_result CQLENGINE adopts whatever keyspace is passed in vai the set_session method as default
@test_category object_mapper
"""
connection.set_session(self.session1)
sync_table(TestConnectModel)
TCM1 = TestConnectModel.create(id=1, keyspace=self.keyspace1)
connection.set_session(self.session2)
sync_table(TestConnectModel)
TCM2 = TestConnectModel.create(id=1, keyspace=self.keyspace2)
connection.set_session(self.session1)
self.assertEqual(1, TestConnectModel.objects.count())
self.assertEqual(TestConnectModel.objects.first(), TCM1)
connection.set_session(self.session2)
self.assertEqual(1, TestConnectModel.objects.count())
self.assertEqual(TestConnectModel.objects.first(), TCM2)
def test_connection_session_switch(self):
"""
Test to ensure that when the default keyspace is changed in a session and that session,
is set in the connection class, that the new defaul keyspace is honored.
@since 3.1
@jira_ticket PYTHON-486
@expected_result CQLENGINE adopts whatever keyspace is passed in vai the set_session method as default
@test_category object_mapper
"""
connection.set_session(self.session1)
sync_table(TestConnectModel)
TCM1 = TestConnectModel.create(id=1, keyspace=self.keyspace1)
connection.set_session(self.session2)
sync_table(TestConnectModel)
TCM2 = TestConnectModel.create(id=1, keyspace=self.keyspace2)
connection.set_session(self.session1)
self.assertEqual(1, TestConnectModel.objects.count())
self.assertEqual(TestConnectModel.objects.first(), TCM1)
connection.set_session(self.session2)
self.assertEqual(1, TestConnectModel.objects.count())
self.assertEqual(TestConnectModel.objects.first(), TCM2)
import os
from cassandra.cqlengine.management import sync_table
from cassandra.cqlengine.connection import set_session
from models import Tweet
from cassandra.cluster import Cluster
from cassandra.query import dict_factory
CASSANDRA_ADDRESS = [os.environ.get('CASSANDRA_ADDRESS', '127.0.0.1')]
CASSANDRA_KEYSPACE = os.environ.get('CASSANDRA_KEYSPACE')
if __name__ == '__main__':
if raw_input('Are you sure you want to sync the database? [Y/N]')=='Y':
cluster = Cluster(CASSANDRA_ADDRESS)
session = cluster.connect(CASSANDRA_KEYSPACE)
session.row_factory = dict_factory
set_session(session)
sync_table(Tweet)
def drop_keyspace(self):
setup_cass(self.seeds, 'system')
self.session = get_session()
set_session(self.session)
drop_keyspace(self.keyspace)
self.logger.debug("ks dropped")
from cassandra.cqlengine import connection
from cassandra.auth import PlainTextAuthProvider
from variantstore import Variant
from variantstore import SampleVariant
from variantstore import TargetVariant
if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument('-a', '--address', help="IP Address for Cassandra connection", default='127.0.0.1')
parser.add_argument('-u', '--username', help='Cassandra username for login', default=None)
parser.add_argument('-r', '--replication_factor', help="Cassandra replication factor", default=3)
args = parser.parse_args()
if args.username:
password = getpass.getpass()
auth_provider = PlainTextAuthProvider(username=args.username, password=password)
cluster = Cluster([args.address], auth_provider=auth_provider)
session = cluster.connect()
session.row_factory = query.dict_factory
else:
cluster = Cluster([args.address])
session = cluster.connect()
session.row_factory = query.dict_factory
connection.set_session(session)
create_keyspace_simple("variantstore", args.replication_factor)
sync_table(Variant)
sync_table(SampleVariant)
sync_table(TargetVariant)
def getConnection(self):
session = self.cluster.connect()
session.set_keyspace('thirdeye_test')
return connection.set_session(session)
# ssl_opts["ca_certs"] = cfg.config["certpath"]
logger.info("connecting to database")
session = cassandra.cluster.Cluster(
contact_points=[CONTACT_POINT],
port=int(PORT),
auth_provider=cassandra.auth.PlainTextAuthProvider(
username=USERNAME, password=PASSWORD
),
ssl_options={
# "ca_certs": requests.utils.DEFAULT_CA_BUNDLE_PATH,
"ssl_version": ssl.PROTOCOL_TLSv1_2
},
connect_timeout=10,
control_connection_timeout=None,
).connect()
logger.info("connected to database")
session.row_factory = cassandra.query.dict_factory
try:
session.set_keyspace("presidents")
except cassandra.InvalidRequest: # if keyspace does not exist
# TODO: these settings are currently ignored
# https://docs.microsoft.com/en-us/azure/cosmos-db/cassandra-support#keyspace-and-table-options
management.create_keyspace_network_topology(
name="presidents", dc_replication_map={"datacenter": 1}
)
session.set_keyspace("presidents")
connection.set_session(session)
def connect_cass(self):
setup_cass(self.seeds, self.keyspace)
self.session = get_session()
set_session(self.session)
