def prepare_all_keys_statement(cass_table,
primary_key,
consistency_level=ConsistencyLevel.QUORUM):
# read every key of the table
query = 'SELECT {} FROM {}'.format(protect_name(primary_key),
protect_name(cass_table))
return SimpleStatement(query,
fetch_size=1000,
consistency_level=consistency_level)
def column_family_name(cls, include_keyspace=True):
"""
Returns the column family name if it's been defined
otherwise, it creates it from the module and class name
"""
cf_name = protect_name(cls._raw_column_family_name())
if include_keyspace:
return '{0}.{1}'.format(protect_name(cls._get_keyspace()), cf_name)
return cf_name
def createPrepStatement(source_data):
for field in source_data[0]._fields:
if 'field_list' in locals():
field_list = field_list + ',' + protect_name(field) # See note1
value_list = value_list + ',?'
else:
field_list = protect_name(field) # See note1
value_list = '?'
prep_statement = 'INSERT INTO ' + tgt_keyspace + '.' + tgt_table + ' (' + field_list + ') VALUES (' + value_list + ');'
return prep_statement
def column_family_name(cls, include_keyspace=True):
"""
Returns the column family name if it's been defined
otherwise, it creates it from the module and class name
"""
cf_name = protect_name(cls._raw_column_family_name())
if include_keyspace:
keyspace = cls._get_keyspace()
if not keyspace:
raise CQLEngineException("Model keyspace is not set and no default is available. Set model keyspace or setup connection before attempting to generate a query.")
return '{0}.{1}'.format(protect_name(keyspace), cf_name)
return cf_name
def prepare_export_query(self, partition_key, token_range):
"""
Return a query where we select all the data for this token range
"""
pk_cols = ", ".join(protect_names(col.name for col in partition_key))
columnlist = ', '.join(protect_names(self.columns))
start_token, end_token = token_range
query = 'SELECT %s FROM %s.%s' % (columnlist, protect_name(self.ks), protect_name(self.cf))
if start_token is not None or end_token is not None:
query += ' WHERE'
if start_token is not None:
query += ' token(%s) > %s' % (pk_cols, start_token)
if start_token is not None and end_token is not None:
query += ' AND'
if end_token is not None:
query += ' token(%s) <= %s' % (pk_cols, end_token)
return query
def create_keyspace(name,
strategy_class,
replication_factor,
durable_writes=True,
**replication_values):
"""
*Deprecated - use :func:`create_keyspace_simple` or :func:`create_keyspace_network_topology` instead*
Creates a keyspace
If the keyspace already exists, it will not be modified.
**This function should be used with caution, especially in production environments.
Take care to execute schema modifications in a single context (i.e. not concurrently with other clients).**
*There are plans to guard schema-modifying functions with an environment-driven conditional.*
:param str name: name of keyspace to create
:param str strategy_class: keyspace replication strategy class (:attr:`~.SimpleStrategy` or :attr:`~.NetworkTopologyStrategy`
:param int replication_factor: keyspace replication factor, used with :attr:`~.SimpleStrategy`
:param bool durable_writes: Write log is bypassed if set to False
:param \*\*replication_values: Additional values to ad to the replication options map
"""
if not _allow_schema_modification():
return
msg = "Deprecated. Use create_keyspace_simple or create_keyspace_network_topology instead"
warnings.warn(msg, DeprecationWarning)
log.warning(msg)
cluster = get_cluster()
if name not in cluster.metadata.keyspaces:
# try the 1.2 method
replication_map = {
'class': strategy_class,
'replication_factor': replication_factor
}
replication_map.update(replication_values)
if strategy_class.lower() != 'simplestrategy':
# Although the Cassandra documentation states for `replication_factor`
# that it is "Required if class is SimpleStrategy; otherwise,
# not used." we get an error if it is present.
replication_map.pop('replication_factor', None)
query = """
CREATE KEYSPACE {0}
WITH REPLICATION = {1}
""".format(metadata.protect_name(name),
json.dumps(replication_map).replace('"', "'"))
if strategy_class != 'SimpleStrategy':
query += " AND DURABLE_WRITES = {0}".format(
'true' if durable_writes else 'false')
execute(query)
def _get_index_name_by_column(table, column_name):
"""
Find the index name for a given table and column.
"""
protected_name = metadata.protect_name(column_name)
possible_index_values = [protected_name, "values(%s)" % protected_name]
for index_metadata in table.indexes.values():
options = dict(index_metadata.index_options)
if options.get('target') in possible_index_values:
return index_metadata.name
def get_pct_ownership(keyspace):
my_ip = str(get_network_ip())
nodestats = subprocess.check_output(
["nodetool", "status", protect_name(keyspace)])
for i in nodestats.splitlines():
if my_ip in i:
for j in i.split(" "):
if "%" in j:
return float(j[:-1]) / 100
logging.warning(
"unable to calculate local ownership, assuming 100% ownership")
return 1
def create_keyspace(name, strategy_class, replication_factor, durable_writes=True, **replication_values):
"""
*Deprecated - use :func:`create_keyspace_simple` or :func:`create_keyspace_network_topology` instead*
Creates a keyspace
If the keyspace already exists, it will not be modified.
**This function should be used with caution, especially in production environments.
Take care to execute schema modifications in a single context (i.e. not concurrently with other clients).**
*There are plans to guard schema-modifying functions with an environment-driven conditional.*
:param str name: name of keyspace to create
:param str strategy_class: keyspace replication strategy class (:attr:`~.SimpleStrategy` or :attr:`~.NetworkTopologyStrategy`
:param int replication_factor: keyspace replication factor, used with :attr:`~.SimpleStrategy`
:param bool durable_writes: Write log is bypassed if set to False
:param \*\*replication_values: Additional values to ad to the replication options map
"""
if not _allow_schema_modification():
return
msg = "Deprecated. Use create_keyspace_simple or create_keyspace_network_topology instead"
warnings.warn(msg, DeprecationWarning)
log.warning(msg)
cluster = get_cluster()
if name not in cluster.metadata.keyspaces:
# try the 1.2 method
replication_map = {
'class': strategy_class,
'replication_factor': replication_factor
}
replication_map.update(replication_values)
if strategy_class.lower() != 'simplestrategy':
# Although the Cassandra documentation states for `replication_factor`
# that it is "Required if class is SimpleStrategy; otherwise,
# not used." we get an error if it is present.
replication_map.pop('replication_factor', None)
query = """
CREATE KEYSPACE {}
WITH REPLICATION = {}
""".format(metadata.protect_name(name), json.dumps(replication_map).replace('"', "'"))
if strategy_class != 'SimpleStrategy':
query += " AND DURABLE_WRITES = {}".format('true' if durable_writes else 'false')
execute(query)
def test_protect_name(self):
"""
Test cassandra.metadata.protect_name output
"""
self.assertEqual(protect_name('tests'), 'tests')
self.assertEqual(protect_name('test\'s'), '"test\'s"')
self.assertEqual(protect_name('test\'s'), "\"test's\"")
self.assertEqual(protect_name('tests ?!@#$%^&*()'), '"tests ?!@#$%^&*()"')
self.assertEqual(protect_name('1'), '"1"')
self.assertEqual(protect_name('1test'), '"1test"')
def test_protect_name(self):
"""
Test cassandra.metadata.protect_name output
"""
self.assertEqual(protect_name("tests"), "tests")
self.assertEqual(protect_name("test's"), '"test\'s"')
self.assertEqual(protect_name("test's"), '"test\'s"')
self.assertEqual(protect_name("tests ?!@#$%^&*()"), '"tests ?!@#$%^&*()"')
self.assertEqual(protect_name("1"), '"1"')
self.assertEqual(protect_name("1test"), '"1test"')
def drop_keyspace(name):
"""
Drops a keyspace, if it exists.
*There are plans to guard schema-modifying functions with an environment-driven conditional.*
**This function should be used with caution, especially in production environments.
Take care to execute schema modifications in a single context (i.e. not concurrently with other clients).**
:param str name: name of keyspace to drop
"""
if not _allow_schema_modification():
return
cluster = get_cluster()
if name in cluster.metadata.keyspaces:
execute("DROP KEYSPACE {}".format(metadata.protect_name(name)))
cluster = Cluster(
[cass_ip],
port=cass_port, compression=False)
# Connect to the KS
session = cluster.connect(cass_keyspace)
# get the primary key of the table
# May have interesting behavior if the table has a composite PK
primary_key = cluster.metadata.keyspaces[cass_keyspace].tables[cass_table].primary_key[0].name
# print primary_key # debug
if primary_key:
# read every key of the table
query = 'SELECT id FROM ' + protect_name(cass_table)
statement = SimpleStatement(query, fetch_size=1000, consistency_level=ConsistencyLevel.QUORUM)
print 'Starting to repair table ' + cass_table
repair_query = 'SELECT COUNT(1) FROM {} WHERE {} = ?'.format(protect_name(cass_table),
protect_name(primary_key))
repair_statement = session.prepare(repair_query)
repair_statement.consistency_level = ConsistencyLevel.ALL
row_count = 0
for user_row in session.execute(statement):
# print user_row.id # debug
row_count += 1
session.execute(repair_statement, [user_row.id])
time.sleep(float(cass_throttle) / 1000000) # delay in microseconds between reading each row
if (row_count % 1000) == 0:
print str(row_count) + ' rows processed'
print 'Repair of table ' + cass_table + ' complete'
def sync_table(model):
"""
Inspects the model and creates / updates the corresponding table and columns.
Any User Defined Types used in the table are implicitly synchronized.
This function can only add fields that are not part of the primary key.
Note that the attributes removed from the model are not deleted on the database.
They become effectively ignored by (will not show up on) the model.
**This function should be used with caution, especially in production environments.
Take care to execute schema modifications in a single context (i.e. not concurrently with other clients).**
*There are plans to guard schema-modifying functions with an environment-driven conditional.*
"""
if not _allow_schema_modification():
return
if not issubclass(model, Model):
raise CQLEngineException("Models must be derived from base Model.")
if model.__abstract__:
raise CQLEngineException("cannot create table from abstract model")
cf_name = model.column_family_name()
raw_cf_name = model._raw_column_family_name()
ks_name = model._get_keyspace()
cluster = get_cluster()
keyspace = cluster.metadata.keyspaces[ks_name]
tables = keyspace.tables
syncd_types = set()
for col in model._columns.values():
udts = []
columns.resolve_udts(col, udts)
for udt in [u for u in udts if u not in syncd_types]:
_sync_type(ks_name, udt, syncd_types)
# check for an existing column family
if raw_cf_name not in tables:
log.debug("sync_table creating new table %s", cf_name)
qs = _get_create_table(model)
try:
execute(qs)
except CQLEngineException as ex:
# 1.2 doesn't return cf names, so we have to examine the exception
# and ignore if it says the column family already exists
if "Cannot add already existing column family" not in unicode(ex):
raise
else:
log.debug("sync_table checking existing table %s", cf_name)
# see if we're missing any columns
field_names = _get_non_pk_field_names(tables[raw_cf_name])
model_fields = set()
# # TODO: does this work with db_name??
for name, col in model._columns.items():
if col.primary_key or col.partition_key:
continue # we can't mess with the PK
model_fields.add(name)
if col.db_field_name in field_names:
continue # skip columns already defined
# add missing column using the column def
query = "ALTER TABLE {0} add {1}".format(cf_name, col.get_column_def())
execute(query)
db_fields_not_in_model = model_fields.symmetric_difference(field_names)
if db_fields_not_in_model:
log.info("Table %s has fields not referenced by model: %s", cf_name, db_fields_not_in_model)
_update_options(model)
table = cluster.metadata.keyspaces[ks_name].tables[raw_cf_name]
indexes = [c for n, c in model._columns.items() if c.index]
# TODO: support multiple indexes in C* 3.0+
for column in indexes:
index_name = 'index_{0}_{1}'.format(raw_cf_name, column.db_field_name)
if table.indexes.get(index_name):
continue
qs = ['CREATE INDEX {0}'.format(metadata.protect_name(index_name))]
qs += ['ON {0}'.format(cf_name)]
qs += ['("{0}")'.format(column.db_field_name)]
qs = ' '.join(qs)
execute(qs)
def _drop_keyspace(name, connection=None):
cluster = get_cluster(connection)
if name in cluster.metadata.keyspaces:
execute("DROP KEYSPACE {0}".format(metadata.protect_name(name)), connection=connection)
def prepare_repair_statement(cass_table, primary_key, session):
repair_query = 'SELECT COUNT(1) FROM {} WHERE {} = ?'.format(
protect_name(cass_table), protect_name(primary_key))
repair_statement = session.prepare(repair_query)
repair_statement.consistency_level = ConsistencyLevel.ALL
return repair_statement
