def setup():
auth_provider = PlainTextAuthProvider(username=config.cassandra_user,
password=config.cassandra_password)
res = None
if config.dev_mode:
res = connection.setup(config.cassandra_hosts,
config.cassandra_keyspace,
protocol_version=3)
else:
logger.info("Using cert: %s", config.cassandra_client_cert)
logger.info("Using key: %s", config.cassandra_client_key)
res = connection.setup(config.cassandra_hosts,
config.cassandra_keyspace,
protocol_version=3,
auth_provider=auth_provider,
ssl_options={
'keyfile': config.cassandra_client_key,
'certfile': config.cassandra_client_cert,
'ssl_version': ssl.PROTOCOL_TLSv1,
})
print(res)
if (config.dev_mode):
management.create_keyspace_simple(config.cassandra_keyspace, 1)
logger.debug("Processing tables...")
for module, _ in inspect.getmembers(models, inspect.ismodule):
for name, model in inspect.getmembers(models.__dict__[module],
inspect.isclass):
if not model.__dict__.get('__ignore__', False) and issubclass(
model, BaseModel.Model):
logger.debug("Syncing table %s", name)
management.sync_table(model)
logger.info("Successfully setup Cassandra!")
def main():
connection.default()
# Management functions would normally be used in development, and possibly for deployments.
# They are typically not part of a core application.
log.info("### creating keyspace...")
management.create_keyspace_simple(KEYSPACE, 1)
log.info("### syncing model...")
management.sync_table(FamilyMembers)
# default uuid is assigned
simmons = FamilyMembers.create(surname='Simmons', name='Gene', birth_year=1949, sex='m')
# add members to his family later
FamilyMembers.create(id=simmons.id, surname='Simmons', name='Nick', birth_year=1989, sex='m')
sophie = FamilyMembers.create(id=simmons.id, surname='Simmons', name='Sophie', sex='f')
nick = FamilyMembers.objects(id=simmons.id, surname='Simmons', name='Nick')
try:
nick.iff(birth_year=1988).update(birth_year=1989)
except LWTException:
print "precondition not met"
# showing validation
try:
FamilyMembers.create(id=simmons.id, surname='Tweed', name='Shannon', birth_year=1957, sex='f')
except ValidationError:
log.exception('INTENTIONAL VALIDATION EXCEPTION; Failed creating instance:')
FamilyMembers.create(id=simmons.id, surname='Tweed', name='Shannon', sex='f')
log.info("### add multiple as part of a batch")
# If creating many at one time, can use a batch to minimize round-trips
hogan_id = uuid4()
with BatchQuery() as b:
FamilyMembers.batch(b).create(id=hogan_id, surname='Hogan', name='Hulk', sex='m')
FamilyMembers.batch(b).create(id=hogan_id, surname='Hogan', name='Linda', sex='f')
FamilyMembers.batch(b).create(id=hogan_id, surname='Hogan', name='Nick', sex='m')
FamilyMembers.batch(b).create(id=hogan_id, surname='Hogan', name='Brooke', sex='f')
log.info("### All members")
for m in FamilyMembers.all():
print m, m.birth_year, m.sex
log.info("### Select by partition key")
for m in FamilyMembers.objects(id=simmons.id):
print m, m.birth_year, m.sex
log.info("### Constrain on clustering key")
for m in FamilyMembers.objects(id=simmons.id, surname=simmons.surname):
print m, m.birth_year, m.sex
log.info("### Constrain on clustering key")
kids = FamilyMembers.objects(id=simmons.id, surname=simmons.surname, name__in=['Nick', 'Sophie'])
log.info("### Delete a record")
FamilyMembers(id=hogan_id, surname='Hogan', name='Linda').delete()
for m in FamilyMembers.objects(id=hogan_id):
print m, m.birth_year, m.sex
management.drop_keyspace(KEYSPACE)
def before_server_start(app, _):
cluster = Cluster([CLUSTER_HOST])
session = cluster.connect()
session.execute("""
CREATE KEYSPACE IF NOT EXISTS %s
WITH REPLICATION = {'class' : 'SimpleStrategy', 'replication_factor' : 1};
""" % CLUSTER_KEY_SPACE)
session = cluster.connect(keyspace=CLUSTER_KEY_SPACE)
aiosession(session)
# session.execute(f"DROP KEYSPACE {CLUSTER_KEY_SPACE}")
connection.register_connection(CLUSTER_NAME, session=session)
create_keyspace_simple(CLUSTER_KEY_SPACE, 1, connections=[CLUSTER_NAME])
connection.setup([CLUSTER_HOST], CLUSTER_KEY_SPACE)
setup_db()
app.db_session = session
logger = logging.getLogger()
logger.setLevel(DEBUG_LEVEL)
handler = logging.StreamHandler()
handler.setFormatter(logging.Formatter(LOGGER_FORMAT))
logger.addHandler(handler)
app.log = logger
def test_keywords_as_names(self):
create_keyspace_simple('keyspace', 1)
class table(Model):
__keyspace__ = 'keyspace'
select = columns.Integer(primary_key=True)
table = columns.Text()
# create should work
drop_table(table)
sync_table(table)
created = table.create(select=0, table='table')
selected = table.objects(select=0)[0]
self.assertEqual(created.select, selected.select)
self.assertEqual(created.table, selected.table)
# alter should work
class table(Model):
__keyspace__ = 'keyspace'
select = columns.Integer(primary_key=True)
table = columns.Text()
where = columns.Text()
sync_table(table)
created = table.create(select=1, table='table')
selected = table.objects(select=1)[0]
self.assertEqual(created.select, selected.select)
self.assertEqual(created.table, selected.table)
self.assertEqual(created.where, selected.where)
drop_keyspace('keyspace')
def _reset_data(self):
for ks in self.keyspaces:
drop_keyspace(ks, connections=self.conns)
for ks in self.keyspaces:
create_keyspace_simple(ks, 1, connections=self.conns)
sync_table(TestModel, keyspaces=self.keyspaces, connections=self.conns)
def setup_storage(settings=None):
"""Create cassandra models."""
from caliopen_storage.core import core_registry
# Make discovery happen
from caliopen_main.user.core import User
from caliopen_main.user.core import (UserIdentity, IdentityLookup,
IdentityTypeLookup)
from caliopen_main.contact.objects.contact import Contact
from caliopen_main.message.objects.message import Message
from caliopen_main.common.objects.tag import ResourceTag
from caliopen_main.device.core import Device
from caliopen_main.notification.core import Notification, NotificationTtl
from cassandra.cqlengine.management import sync_table, \
create_keyspace_simple
keyspace = Configuration('global').get('cassandra.keyspace')
if not keyspace:
raise Exception('Configuration missing for cassandra keyspace')
# XXX tofix : define strategy and replication_factor in configuration
create_keyspace_simple(keyspace, 1)
for name, kls in core_registry.items():
log.info('Creating cassandra model %s' % name)
if hasattr(kls._model_class, 'pk'):
# XXX find a better way to detect model from udt
sync_table(kls._model_class)
def sync(self, alias):
engine = get_engine_from_db_alias(alias)
if engine != 'django_cassandra_engine':
raise CommandError('Database {} is not cassandra!'.format(alias))
connection = connections[alias]
connection.connect()
options = connection.settings_dict.get('OPTIONS', {})
keyspace = connection.settings_dict['NAME']
replication_opts = options.get('replication', {})
strategy_class = replication_opts.pop('strategy_class',
'SimpleStrategy')
replication_factor = replication_opts.pop('replication_factor', 1)
self.stdout.write('Creating keyspace {}..'.format(keyspace))
if strategy_class == 'SimpleStrategy':
create_keyspace_simple(keyspace, replication_factor)
else:
create_keyspace_network_topology(keyspace, replication_opts)
for app_name, app_models \
in connection.introspection.cql_models.items():
for model in app_models:
self.stdout.write('Syncing %s.%s' % (app_name, model.__name__))
sync_table(model)
def setup_module():
if DSE_VERSION:
os.environ[CQLENG_ALLOW_SCHEMA_MANAGEMENT] = '1'
use_single_node_with_graph_and_solr()
setup_connection(DEFAULT_KEYSPACE)
create_keyspace_simple(DEFAULT_KEYSPACE, 1)
sync_table(SimpleNullableModel)
def create_keyspace(self):
setup_cass(self.seeds, 'system')
self.session = get_session()
set_session(self.session)
drop_keyspace(self.keyspace)
create_keyspace_simple(name=self.keyspace, replication_factor=3)
self.logger.debug("ks created")
def recreate_db():
"""Cleans everything (!) and sets up database"""
drop_keyspace('documents')
drop_table(Documents)
create_keyspace_simple('documents', replication_factor=1)
sync_table(Documents)
def _reset_data(self):
for ks in self.keyspaces:
drop_keyspace(ks, connections=self.conns)
for ks in self.keyspaces:
create_keyspace_simple(ks, 1, connections=self.conns)
sync_table(TestModel, keyspaces=self.keyspaces, connections=self.conns)
def create_keyspace(self):
cluster.max_schema_agreement_wait = 0
setup_cass(self.seeds, 'system')
self.session = get_session()
set_session(self.session)
create_keyspace_simple(name=self.keyspace, replication_factor=3)
self.logger.debug("ks created")
def create_keyspace(
self,
):
'''
Ensure self.keyspace is set appropriately.
'''
self.current_keyspace()
settings = self.settings_dict
keyspace_default_settings = {
'durable_writes': True,
'strategy_class': SimpleStrategy.name,
'replication_factor': 3
}
keyspace_settings = settings.get(
'KEYSPACES', {}
).get(
self.keyspace, {}
)
keyspace_default_settings.update(keyspace_settings)
keyspace_settings = keyspace_default_settings
self.ensure_connection()
create_keyspace_simple(
self.keyspace,
keyspace_settings.get(
'replication_factor',
1
)
)
def sync(self, alias):
engine = get_engine_from_db_alias(alias)
if engine != 'django_cassandra_engine':
raise CommandError('Database {} is not cassandra!'.format(alias))
connection = connections[alias]
connection.connect()
options = connection.settings_dict.get('OPTIONS', {})
keyspace = connection.settings_dict['NAME']
replication_opts = options.get('replication', {})
strategy_class = replication_opts.pop('strategy_class',
'SimpleStrategy')
replication_factor = replication_opts.pop('replication_factor', 1)
self.stdout.write('Creating keyspace {}..'.format(keyspace))
if strategy_class == 'SimpleStrategy':
create_keyspace_simple(keyspace, replication_factor)
else:
create_keyspace_network_topology(keyspace, replication_opts)
for app_name, app_models \
in connection.introspection.cql_models.items():
for model in app_models:
self.stdout.write('Syncing %s.%s' % (app_name, model.__name__))
sync_table(model)
def test_create_drop_table(self):
for ks in self.keyspaces:
create_keyspace_simple(ks, 1, connections=self.conns)
# No connection (default is fake)
with self.assertRaises(NoHostAvailable):
sync_table(TestModel)
# Explicit connections
sync_table(TestModel, connections=self.conns)
# Explicit drop
drop_table(TestModel, connections=self.conns)
# Model connection
TestModel.__connection__ = 'cluster'
sync_table(TestModel)
TestModel.__connection__ = None
# No connection (default is fake)
with self.assertRaises(NoHostAvailable):
drop_table(TestModel)
# Model connection
TestModel.__connection__ = 'cluster'
drop_table(TestModel)
TestModel.__connection__ = None
# Model connection
for ks in self.keyspaces:
drop_keyspace(ks, connections=self.conns)
def test_create_drop_succeeeds(self):
cluster = get_cluster()
keyspace_ss = 'test_ks_ss'
self.assertFalse(keyspace_ss in cluster.metadata.keyspaces)
management.create_keyspace_simple(keyspace_ss, 2)
self.assertTrue(keyspace_ss in cluster.metadata.keyspaces)
management.drop_keyspace(keyspace_ss)
self.assertFalse(keyspace_ss in cluster.metadata.keyspaces)
with warnings.catch_warnings(record=True) as w:
management.create_keyspace(keyspace_ss, strategy_class="SimpleStrategy", replication_factor=1)
self.assertEqual(len(w), 1)
self.assertEqual(w[-1].category, DeprecationWarning)
self.assertTrue(keyspace_ss in cluster.metadata.keyspaces)
management.drop_keyspace(keyspace_ss)
self.assertFalse(keyspace_ss in cluster.metadata.keyspaces)
keyspace_nts = 'test_ks_nts'
self.assertFalse(keyspace_nts in cluster.metadata.keyspaces)
management.create_keyspace_simple(keyspace_nts, 2)
self.assertTrue(keyspace_nts in cluster.metadata.keyspaces)
with warnings.catch_warnings(record=True) as w:
management.delete_keyspace(keyspace_nts)
self.assertEqual(len(w), 1)
self.assertEqual(w[-1].category, DeprecationWarning)
self.assertFalse(keyspace_nts in cluster.metadata.keyspaces)
def test_create_drop_succeeeds(self):
cluster = get_cluster()
keyspace_ss = 'test_ks_ss'
self.assertFalse(keyspace_ss in cluster.metadata.keyspaces)
management.create_keyspace_simple(keyspace_ss, 2)
self.assertTrue(keyspace_ss in cluster.metadata.keyspaces)
management.drop_keyspace(keyspace_ss)
self.assertFalse(keyspace_ss in cluster.metadata.keyspaces)
with warnings.catch_warnings(record=True) as w:
management.create_keyspace(keyspace_ss,
strategy_class="SimpleStrategy",
replication_factor=1)
self.assertEqual(len(w), 1)
self.assertEqual(w[-1].category, DeprecationWarning)
self.assertTrue(keyspace_ss in cluster.metadata.keyspaces)
management.drop_keyspace(keyspace_ss)
self.assertFalse(keyspace_ss in cluster.metadata.keyspaces)
keyspace_nts = 'test_ks_nts'
self.assertFalse(keyspace_nts in cluster.metadata.keyspaces)
management.create_keyspace_simple(keyspace_nts, 2)
self.assertTrue(keyspace_nts in cluster.metadata.keyspaces)
with warnings.catch_warnings(record=True) as w:
management.delete_keyspace(keyspace_nts)
self.assertEqual(len(w), 1)
self.assertEqual(w[-1].category, DeprecationWarning)
self.assertFalse(keyspace_nts in cluster.metadata.keyspaces)
def create_table(session):
'''
Creates a "weather" table with four attributes:
id, date, wdire, and wspdi.
Parameters
----------
session: A cassandra.cluster.Session instance.
Returns
-------
A cassandra.cluster.ResultSet instance.
'''
#Create Keyspace
management.create_keyspace_simple(ks_name, 1)
session.set_keyspace(ks_name)
#Defines the weather table
create_schema = '''
CREATE TABLE weather (
id int,
date text,
wdire text,
wspdi double,
PRIMARY KEY(id, wdire)
);
'''
#Creates weather table
result = session.execute(create_schema)
return result
def create_table(keyspace,
tablename,
columns,
partition_key=None,
cluster_key=None,
replication_factor=1,
durable_writes=True,
connections=None,
create_keyspace_if_not_exists=True):
'''
Creates a keyspace.tablename Cassamdra table.
keyspace: keyspace to hold the new table
tablename: new table name
columns: has a dictionary description for data fields
partition_key: list of partition key fields
cluster_key: list of clustering key fields
replication_factor: keyspace replication factor if not created yet
durable_writes: disable commmit log (increase data loss risk)
connections: allow to sync in more connections when multiple connections are used. Default connections if None
create_keyspace_if_not_exists: creates a new keyspace if `keyspace` didn't exist
'''
if create_keyspace_if_not_exists:
create_keyspace_simple(keyspace,
replication_factor,
durable_writes=durable_writes,
connections=connections)
fields = _build_fields(keyspace, tablename, columns)
# meta_class_object has table structure
meta_class_object = type('MetaClass', (Model, object), fields)
# sync Cassandra table with MetaClass structure
sync_table(meta_class_object)
def test_set_keyspace_no_args(self):
create_keyspace_simple("test2", 1)
self.db.set_keyspace('test2')
self.db.set_keyspace()
self.assertEqual(models.DEFAULT_KEYSPACE,
self.app.config['CASSANDRA_KEYSPACE'])
self.assertEqual(self.db._keyspace_,
self.app.config['CASSANDRA_KEYSPACE'])
def test_set_keyspace_no_args(self):
create_keyspace_simple("test2", 1)
self.db.set_keyspace('test2')
self.db.set_keyspace()
self.assertEqual(models.DEFAULT_KEYSPACE,
self.app.config['CASSANDRA_KEYSPACE'])
self.assertEqual(self.db._keyspace_,
self.app.config['CASSANDRA_KEYSPACE'])
def setup_package():
warnings.simplefilter('always') # for testing warnings, make sure all are let through
os.environ[CQLENG_ALLOW_SCHEMA_MANAGEMENT] = '1'
use_single_node()
setup_connection(DEFAULT_KEYSPACE)
create_keyspace_simple(DEFAULT_KEYSPACE, 1)
def setup_package():
warnings.simplefilter('always') # for testing warnings, make sure all are let through
os.environ[CQLENG_ALLOW_SCHEMA_MANAGEMENT] = '1'
use_single_node()
setup_connection(DEFAULT_KEYSPACE)
create_keyspace_simple(DEFAULT_KEYSPACE, 1)
def create_cassandra_keyspace_and_tables(default_keyspace):
os.environ['CQLENG_ALLOW_SCHEMA_MANAGEMENT'] = '1'
try:
create_keyspace_simple(default_keyspace, replication_factor=1)
except AlreadyExists:
pass
else:
sync_table(CqlStoredEvent)
def reset_db():
print('Connecting to cluster')
connection.setup(['127.0.0.1'], 'system', protocol_version=3)
print('Dropping existing keyspace')
drop_keyspace(db_keyspace)
print('Creating keyspace for project')
create_keyspace_simple(db_keyspace,
replication_factor=replication_factor)
sync_all_tables()
def create_network_keyspace(self):
cluster.max_schema_agreement_wait = 0
setup_cass(self.seeds, 'system')
self.session = get_session()
set_session(self.session)
dc_map = {'DC1-Data': 3, 'DC1-Analytics': 3}
create_keyspace_network_topology(name=self.keyspace, dc_replication_map=dc_map)
create_keyspace_simple(name=self.keyspace, replication_factor=3)
self.logger.debug("ks network topo created")
def build(keyspace, res_names=None):
'''
Creates a new keyspace and populates it with blank tables
'''
# create the keyspace if it doesn't already exist
create_keyspace_simple(keyspace,
replication_factor=REPLICATION_FACTOR)
switch(keyspace)
create_all_tables(res_names)
def __init__(self,
hosts,
keyspace='gunregistry',
replication_factor=1,
**cqlengine_params):
self.keyspace = keyspace
connection.setup(hosts, keyspace, **cqlengine_params)
create_keyspace_simple(keyspace, replication_factor)
sync_table(Record)
def setUp(self):
app = Flask(__name__)
app.config['CASSANDRA_HOSTS'] = ['127.0.0.1']
app.config['CASSANDRA_KEYSPACE'] = "test1"
db = CQLAlchemy(app)
self.User = make_user_model(db)
create_keyspace_simple("test1", 1)
self.app = app
self.db = db
def create_cassandra_keyspace_and_tables(keyspace=DEFAULT_CASSANDRA_KEYSPACE, replication_factor=1):
# Avoid warnings about this variable not being set.
os.environ['CQLENG_ALLOW_SCHEMA_MANAGEMENT'] = '1'
try:
create_keyspace_simple(keyspace, replication_factor=replication_factor)
except AlreadyExists:
pass
else:
sync_table(CqlStoredEvent)
sync_table(CqlEntityVersion)
def migrate_keyspaces(settings):
"""Creates if not exists the base keyspaces."""
override_keyspace = settings['cassandra.override_keyspaces']
if override_keyspace:
management.drop_keyspace(settings['cassandra.keyspace_default'])
management.create_keyspace_simple(settings['cassandra.keyspace_default'],
replication_factor=1)
def setUpClass(cls):
super(BatchQueryConnectionTests, cls).setUpClass()
create_keyspace_simple('ks1', 1)
sync_table(TestModel)
sync_table(AnotherTestModel)
conn.unregister_connection('default')
conn.register_connection('fake_cluster', ['127.0.0.100'], lazy_connect=True, retry_connect=True, default=True)
conn.register_connection('cluster', ['127.0.0.1'])
def setUpClass(cls):
super(ContextQueryConnectionTests, cls).setUpClass()
create_keyspace_simple('ks1', 1)
conn.unregister_connection('default')
conn.register_connection('fake_cluster', ['127.0.0.100'], lazy_connect=True, retry_connect=True, default=True)
conn.register_connection('cluster', ['127.0.0.1'])
with ContextQuery(TestModel, connection='cluster') as tm:
sync_table(tm)
def cassandra_reset():
from stream_framework.feeds.cassandra import CassandraFeed
from stream_framework.feeds.aggregated_feed.cassandra import CassandraAggregatedFeed
from cassandra.cqlengine.management import create_keyspace_simple, sync_table
from stream_framework import settings
create_keyspace_simple(settings.STREAM_DEFAULT_KEYSPACE, 1)
aggregated_timeline = CassandraAggregatedFeed.get_timeline_storage()
timeline = CassandraFeed.get_timeline_storage()
sync_table(aggregated_timeline.model)
sync_table(timeline.model)
def setUp(self):
app = Flask(__name__)
app.config['CASSANDRA_HOSTS'] = ['127.0.0.1']
app.config['CASSANDRA_KEYSPACE'] = "test1"
db = CQLAlchemy(app)
self.User = make_user_model(db)
create_keyspace_simple("test1", 1)
self.app = app
self.db = db
def test_can_create_same_udt_different_keyspaces(self):
class User(UserType):
age = columns.Integer()
name = columns.Text()
sync_type("cqlengine_test", User)
create_keyspace_simple("simplex", 1)
sync_type("simplex", User)
drop_keyspace("simplex")
def create_cassandra_keyspace_and_tables(keyspace=DEFAULT_CASSANDRA_KEYSPACE, replication_factor=1):
# Avoid warnings about this variable not being set.
os.environ['CQLENG_ALLOW_SCHEMA_MANAGEMENT'] = '1'
try:
create_keyspace_simple(keyspace, replication_factor=replication_factor)
except AlreadyExists:
pass
else:
sync_table(CqlStoredEvent)
sync_table(CqlEntityVersion)
def test_can_create_same_udt_different_keyspaces(self):
class User(UserType):
age = columns.Integer()
name = columns.Text()
sync_type("cqlengine_test", User)
create_keyspace_simple("simplex", 1)
sync_type("simplex", User)
drop_keyspace("simplex")
def setUpClass(cls):
super(BatchQueryConnectionTests, cls).setUpClass()
create_keyspace_simple('ks1', 1)
sync_table(TestModel)
sync_table(AnotherTestModel)
conn.unregister_connection('default')
conn.register_connection('fake_cluster', ['127.0.0.100'], lazy_connect=True, retry_connect=True, default=True)
conn.register_connection('cluster', [CASSANDRA_IP])
def __init__(self, host: str = 'localhost', port: int = 9042):
self.cluster: Cluster = Cluster([host], port=port)
self.session: Session = self.cluster.connect()
connection.set_session(self.session)
management.create_keyspace_simple(UserProfile.__keyspace__,
replication_factor=1)
connection.session.set_keyspace(UserProfile.__keyspace__)
# Creates tables if non existent
management.sync_table(UserProfile)
def create_keyspace(session, keyspace):
connection.set_session(session)
try:
management.create_keyspace_simple(
keyspace["name"], replication_factor=keyspace["factor"])
except cassandra.AlreadyExists:
pass
session.set_keyspace(keyspace["name"])
connection.set_session(session)
return session
def setUpClass(cls):
super(ContextQueryConnectionTests, cls).setUpClass()
create_keyspace_simple('ks1', 1)
conn.unregister_connection('default')
conn.register_connection('fake_cluster', ['1.2.3.4'], lazy_connect=True, retry_connect=True, default=True)
conn.register_connection('cluster', [CASSANDRA_IP])
with ContextQuery(TestModel, connection='cluster') as tm:
sync_table(tm)
def _reset_db(app, db):
'''
Get a clean and empty database.
When testing, the app is initiated with keyspace 'testing'
'''
drop_keyspace(app.config['CASSANDRA_KEYSPACE'])
create_keyspace_simple(app.config['CASSANDRA_KEYSPACE'],
replication_factor=1)
# create the tables from our models.
db.sync_db()
def setup_tables(self):
# Avoid warnings about this variable not being set.
os.environ['CQLENG_ALLOW_SCHEMA_MANAGEMENT'] = '1'
# Attempt to create the keyspace.
create_keyspace_simple(
name=self.settings.default_keyspace,
replication_factor=self.settings.replication_factor,
)
for table in self.tables:
sync_table(table)
def create_cass_keyspace(cls):
"""
Create Cassandra's testing keyspace.
CE Cassandra doesn't currently have in-memory DBs, so we can't just use
the factory like we do in SQL.
"""
connection.setup(TESTING_IPS, INITIAL_KEYSPACE_NAME)
# Make sure the keyspace is empty
cls.teardown_cass_keyspace()
management.create_keyspace_simple(TEMP_KEYSPACE_NAME, 1)
def setup_tables(self):
# Avoid warnings about this variable not being set.
os.environ["CQLENG_ALLOW_SCHEMA_MANAGEMENT"] = "1"
# Attempt to create the keyspace.
create_keyspace_simple(
name=self.settings.default_keyspace,
replication_factor=self.settings.replication_factor,
)
for table in self.tables:
self.setup_table(table)
def setup_package():
warnings.simplefilter('always') # for testing warnings, make sure all are let through
os.environ[CQLENG_ALLOW_SCHEMA_MANAGEMENT] = '1'
use_single_node()
keyspace = 'cqlengine_test'
connection.setup(['localhost'],
protocol_version=PROTOCOL_VERSION,
default_keyspace=keyspace)
create_keyspace_simple(keyspace, 1)
def create_test_db(self, verbosity=1, autoclobber=False, **kwargs):
"""
Creates a test database, prompting the user for confirmation if the
database already exists. Returns the name of the test database created.
"""
# Don't import django.core.management if it isn't needed.
from django.core.management import call_command
from django.conf import settings
self.connection.connect()
# If using django-nose, its runner has already set the db name
# to test_*, so restore it here so that all the models for the
# live keyspace can be found.
self.connection.connection.keyspace = \
self.connection.settings_dict['NAME']
test_database_name = self._get_test_db_name()
# Set all models keyspace to the test keyspace
self.set_models_keyspace(test_database_name)
if verbosity >= 1:
test_db_repr = ''
if verbosity >= 2:
test_db_repr = " ('%s')" % test_database_name
print("Creating test database for alias '%s'%s..." % (
self.connection.alias, test_db_repr))
options = self.connection.settings_dict.get('OPTIONS', {})
replication_opts = options.get('replication', {})
replication_factor = replication_opts.pop('replication_factor', 1)
create_keyspace_simple(self.connection.settings_dict['NAME'],
replication_factor)
settings.DATABASES[self.connection.alias]["NAME"] = test_database_name
self.connection.settings_dict["NAME"] = test_database_name
self.connection.reconnect()
# Report syncdb messages at one level lower than that requested.
# This ensures we don't get flooded with messages during testing
# (unless you really ask to be flooded)
call_command(
'sync_cassandra',
verbosity=max(verbosity - 1, 0),
interactive=False,
database=self.connection.alias,
load_initial_data=False
)
return test_database_name
def test_keywords_as_names(self):
"""
Test for CQL keywords as names
test_keywords_as_names tests that CQL keywords are properly and automatically quoted in cqlengine. It creates
a keyspace, keyspace, which should be automatically quoted to "keyspace" in CQL. It then creates a table, table,
which should also be automatically quoted to "table". It then verfies that operations can be done on the
"keyspace"."table" which has been created. It also verifies that table alternations work and operations can be
performed on the altered table.
@since 2.6.0
@jira_ticket PYTHON-244
@expected_result Cqlengine should quote CQL keywords properly when creating keyspaces and tables.
@test_category schema:generation
"""
# If the keyspace exists, it will not be re-created
create_keyspace_simple('keyspace', 1)
class table(Model):
__keyspace__ = 'keyspace'
select = columns.Integer(primary_key=True)
table = columns.Text()
# In case the table already exists in keyspace
drop_table(table)
# Create should work
sync_table(table)
created = table.create(select=0, table='table')
selected = table.objects(select=0)[0]
self.assertEqual(created.select, selected.select)
self.assertEqual(created.table, selected.table)
# Alter should work
class table(Model):
__keyspace__ = 'keyspace'
select = columns.Integer(primary_key=True)
table = columns.Text()
where = columns.Text()
sync_table(table)
created = table.create(select=1, table='table')
selected = table.objects(select=1)[0]
self.assertEqual(created.select, selected.select)
self.assertEqual(created.table, selected.table)
self.assertEqual(created.where, selected.where)
drop_keyspace('keyspace')
def main():
# create a keyspace "test"
connection.setup(['140.121.101.164'], "weather1")
management.create_keyspace_simple('weather1', 3)
# connect to test keyspace
connection.setup(['140.121.101.164'], "weather1", protocol_version=3)
management.sync_type('test', Air)
management.sync_type('test', Sun)
management.sync_type('test', Rain)
management.sync_type('test', Basic)
management.sync_type('test', Value)
management.sync_table(Weather)
def setup_storage(settings=None):
"""Create cassandra models."""
from caliopen.base.core import core_registry
# Make discovery happen
from caliopen.base.user.core import User
from caliopen.base.message.core.thread import Thread
from caliopen.base.message.core.message import Message
from cassandra.cqlengine.management import sync_table, create_keyspace_simple
keyspace = Configuration('global').get('cassandra.keyspace')
if not keyspace:
raise Exception('Configuration missing for cassandra keyspace')
# XXX tofix : define strategy and replication_factor in configuration
create_keyspace_simple(keyspace, 1)
for name, kls in core_registry.items():
log.info('Creating cassandra model %s' % name)
sync_table(kls._model_class)
def __init__(self, instance_id):
self.instance_id = int(instance_id)
# setup cassadnra connection
connection.setup(['127.0.0.1'], "estate1", consistency=ConsistencyLevel.ALL)
# TODO move this to an indipened reset DB script
if self.instance_id == 0:
print "FLUSH DB"
# flush database by first node (for development)
management.drop_keyspace("estate1")
# create needed tables
management.create_keyspace_simple("estate1", replication_factor=1, durable_writes=True)
management.sync_table(KeyValueItem)
print "ES: Initialized estate for instance: %s" % self.instance_id
def test_create_drop_succeeeds(self):
cluster = get_cluster()
keyspace_ss = 'test_ks_ss'
self.assertNotIn(keyspace_ss, cluster.metadata.keyspaces)
management.create_keyspace_simple(keyspace_ss, 2)
self.assertIn(keyspace_ss, cluster.metadata.keyspaces)
management.drop_keyspace(keyspace_ss)
self.assertNotIn(keyspace_ss, cluster.metadata.keyspaces)
keyspace_nts = 'test_ks_nts'
self.assertNotIn(keyspace_nts, cluster.metadata.keyspaces)
management.create_keyspace_network_topology(keyspace_nts, {'dc1': 1})
self.assertIn(keyspace_nts, cluster.metadata.keyspaces)
management.drop_keyspace(keyspace_nts)
self.assertNotIn(keyspace_nts, cluster.metadata.keyspaces)
def initialize_keyspace():
logger.info("Initializing keyspace")
create_keyspace_simple('plays', 1)
# TODO: Activate automatic creation of udf an uda
logger.info("Creating user-defined functions")
udf = """
-- Create a function that takes in state (any Cassandra type) as the first
-- parameter and any number of additional parameters
CREATE OR REPLACE FUNCTION state_group_and_count( state map<text, int>, type_1 text, type_2 text)
CALLED ON NULL INPUT
RETURNS map<text, int>
LANGUAGE java
AS '
// Clean
type_1 = type_1.replaceAll("\"", "\\\"");
type_2 = type_2.replaceAll("\"", "\\\"");
// Json list
String key = "[\"" + type_1 + "\", \"" + type_2 + "\"]";
Integer count = (Integer) state.get(key);
if (count == null) count = 1;
else count++;
state.put(key, count);
return state;
' ;
"""
#session.execute(udf)
logger.info("Creating user-defined aggregations")
uda = """
-- Create a final function that is called after the state function has been
-- called on every row
CREATE OR REPLACE AGGREGATE group_and_count(text, text)
SFUNC state_group_and_count
STYPE map<text, int>
INITCOND {};
"""
#session.execute(uda)
def test_create_drop_keyspace(self):
"""
Tests drop and create keyspace with connections explicitly set
@since 3.7
@jira_ticket PYTHON-613
@expected_result keyspaces should be created and dropped
@test_category object_mapper
"""
# No connection (default is fake)
with self.assertRaises(NoHostAvailable):
create_keyspace_simple(self.keyspaces[0], 1)
# Explicit connections
for ks in self.keyspaces:
create_keyspace_simple(ks, 1, connections=self.conns)
for ks in self.keyspaces:
drop_keyspace(ks, connections=self.conns)
def test_create_drop_table(self):
"""
Tests drop and create Table with connections explicitly set
@since 3.7
@jira_ticket PYTHON-613
@expected_result Tables should be created and dropped
@test_category object_mapper
"""
for ks in self.keyspaces:
create_keyspace_simple(ks, 1, connections=self.conns)
# No connection (default is fake)
with self.assertRaises(NoHostAvailable):
sync_table(TestModel)
# Explicit connections
sync_table(TestModel, connections=self.conns)
# Explicit drop
drop_table(TestModel, connections=self.conns)
# Model connection
TestModel.__connection__ = 'cluster'
sync_table(TestModel)
TestModel.__connection__ = None
# No connection (default is fake)
with self.assertRaises(NoHostAvailable):
drop_table(TestModel)
# Model connection
TestModel.__connection__ = 'cluster'
drop_table(TestModel)
TestModel.__connection__ = None
# Model connection
for ks in self.keyspaces:
drop_keyspace(ks, connections=self.conns)
def test_can_create_same_udt_different_keyspaces(self):
sync_type(DEFAULT_KEYSPACE, User)
create_keyspace_simple("simplex", 1)
sync_type("simplex", User)
drop_keyspace("simplex")