def execute(self):
if len(self.queries) == 0:
# Empty batch is a no-op
return
opener = "BEGIN " + (self.batch_type + " " if self.batch_type else "") + " BATCH"
if self.timestamp:
epoch = datetime(1970, 1, 1)
ts = long((self.timestamp - epoch).total_seconds() * 1000)
opener += " USING TIMESTAMP {}".format(ts)
query_list = [opener]
parameters = {}
ctx_counter = 0
for query in self.queries:
query.update_context_id(ctx_counter)
ctx = query.get_context()
ctx_counter += len(ctx)
query_list.append(" " + str(query))
parameters.update(ctx)
query_list.append("APPLY BATCH;")
execute("\n".join(query_list), parameters, self._consistency)
self.queries = []
def execute(self):
if len(self.queries) == 0:
# Empty batch is a no-op
return
opener = 'BEGIN ' + (self.batch_type + ' ' if self.batch_type else '') + ' BATCH'
if self.timestamp:
epoch = datetime(1970, 1, 1)
ts = long((self.timestamp - epoch).total_seconds() * 1000)
opener += ' USING TIMESTAMP {}'.format(ts)
query_list = [opener]
parameters = {}
ctx_counter = 0
for query in self.queries:
query.update_context_id(ctx_counter)
ctx = query.get_context()
ctx_counter += len(ctx)
query_list.append(' ' + str(query))
parameters.update(ctx)
query_list.append('APPLY BATCH;')
execute('\n'.join(query_list), parameters, self._consistency)
self.queries = []
def create_keyspace(name, strategy_class='SimpleStrategy', replication_factor=3, durable_writes=True, **replication_values):
"""
creates a keyspace
:param name: name of keyspace to create
:param strategy_class: keyspace replication strategy class
:param replication_factor: keyspace replication factor
:param durable_writes: 1.2 only, write log is bypassed if set to False
:param **replication_values: 1.2 only, additional values to ad to the replication data map
"""
cluster = get_cluster()
if name not in cluster.metadata.keyspaces:
#try the 1.2 method
replication_map = {
'class': strategy_class,
'replication_factor':replication_factor
}
replication_map.update(replication_values)
if strategy_class.lower() != 'simplestrategy':
# Although the Cassandra documentation states for `replication_factor`
# that it is "Required if class is SimpleStrategy; otherwise,
# not used." we get an error if it is present.
replication_map.pop('replication_factor', None)
query = """
CREATE KEYSPACE {}
WITH REPLICATION = {}
""".format(name, json.dumps(replication_map).replace('"', "'"))
if strategy_class != 'SimpleStrategy':
query += " AND DURABLE_WRITES = {}".format('true' if durable_writes else 'false')
execute(query)
def update_compaction(model):
logger.debug("Checking %s for compaction differences", model)
row = get_table_settings(model)
# check compaction_strategy_class
if not model.__compaction__:
return
do_update = not row['compaction_strategy_class'].endswith(model.__compaction__)
existing_options = row['compaction_strategy_options']
existing_options = json.loads(existing_options)
desired_options = get_compaction_options(model)
desired_options.pop('class', None)
for k,v in desired_options.items():
val = existing_options.pop(k, None)
if val != v:
do_update = True
# check compaction_strategy_options
if do_update:
options = get_compaction_options(model)
# jsonify
options = json.dumps(options).replace('"', "'")
cf_name = model.column_family_name()
query = "ALTER TABLE {} with compaction = {}".format(cf_name, options)
logger.debug(query)
execute(query)
def test_extra_field(self):
drop_table(self.TestModel)
sync_table(self.TestModel)
self.TestModel.create()
execute("ALTER TABLE {} add blah int".format(
self.TestModel.column_family_name(include_keyspace=True)))
self.TestModel.objects().all()
def trim(self, key, length, batch_interface=None):
'''
trim using Cassandra's tombstones black magic
retrieve the WRITETIME of the last item we want to keep
then delete everything written after that
this is still pretty inefficient since it needs to retrieve
length amount of items
WARNING: since activities created using Batch share the same timestamp
trim can trash up to (batch_size - 1) more activities than requested
'''
query = "SELECT WRITETIME(%s) as wt FROM %s.%s WHERE feed_id='%s' ORDER BY activity_id DESC LIMIT %s;"
trim_col = [
c for c in self.model._columns.keys()
if c not in self.model._primary_keys.keys()
][0]
parameters = (trim_col, self.model._get_keyspace(),
self.column_family_name, key, length + 1)
results = execute(query % parameters)
if len(results) < length:
return
trim_ts = (results[-1].wt + results[-2].wt) / 2
delete_query = "DELETE FROM %s.%s USING TIMESTAMP %s WHERE feed_id='%s';"
delete_params = (self.model._get_keyspace(), self.column_family_name,
trim_ts, key)
execute(delete_query % delete_params)
def update_compaction(model):
logger.debug("Checking %s for compaction differences", model)
row = get_table_settings(model)
# check compaction_strategy_class
if not model.__compaction__:
return
do_update = not row['compaction_strategy_class'].endswith(
model.__compaction__)
existing_options = row['compaction_strategy_options']
existing_options = json.loads(existing_options)
desired_options = get_compaction_options(model)
desired_options.pop('class', None)
for k, v in desired_options.items():
val = existing_options.pop(k, None)
if val != v:
do_update = True
# check compaction_strategy_options
if do_update:
options = get_compaction_options(model)
# jsonify
options = json.dumps(options).replace('"', "'")
cf_name = model.column_family_name()
query = "ALTER TABLE {} with compaction = {}".format(cf_name, options)
logger.debug(query)
execute(query)
def trim(self, key, length, batch_interface=None):
'''
trim using Cassandra's tombstones black magic
retrieve the WRITETIME of the last item we want to keep
then delete everything written after that
this is still pretty inefficient since it needs to retrieve
length amount of items
WARNING: since activities created using Batch share the same timestamp
trim can trash up to (batch_size - 1) more activities than requested
'''
query = "SELECT WRITETIME(%s) as wt FROM %s.%s WHERE feed_id='%s' ORDER BY activity_id DESC LIMIT %s;"
trim_col = [c for c in self.model._columns.keys(
) if c not in self.model._primary_keys.keys()][0]
parameters = (
trim_col, self.model._get_keyspace(), self.column_family_name, key, length + 1)
results = execute(query % parameters)
if len(results) < length:
return
trim_ts = (results[-1]['wt'] + results[-2]['wt']) / 2
delete_query = "DELETE FROM %s.%s USING TIMESTAMP %s WHERE feed_id='%s';"
delete_params = (
self.model._get_keyspace(), self.column_family_name, trim_ts, key)
execute(delete_query % delete_params)
def create_keyspace(name, strategy_class='SimpleStrategy', replication_factor=3, durable_writes=True, **replication_values):
"""
creates a keyspace
:param name: name of keyspace to create
:param strategy_class: keyspace replication strategy class
:param replication_factor: keyspace replication factor
:param durable_writes: 1.2 only, write log is bypassed if set to False
:param **replication_values: 1.2 only, additional values to ad to the replication data map
"""
with connection_manager() as con:
keyspaces = con.execute(
"""SELECT keyspace_name FROM system.schema_keyspaces""", {})
if name not in [r[0] for r in keyspaces]:
# try the 1.2 method
replication_map = {
'class': strategy_class,
'replication_factor': replication_factor
}
replication_map.update(replication_values)
query = """
CREATE KEYSPACE {}
WITH REPLICATION = {}
""".format(name, json.dumps(replication_map).replace('"', "'"))
if strategy_class != 'SimpleStrategy':
query += " AND DURABLE_WRITES = {}".format(
'true' if durable_writes else 'false')
execute(query)
def create_keyspace(name, strategy_class='SimpleStrategy', replication_factor=3, durable_writes=True, **replication_values):
"""
creates a keyspace
:param name: name of keyspace to create
:param strategy_class: keyspace replication strategy class
:param replication_factor: keyspace replication factor
:param durable_writes: 1.2 only, write log is bypassed if set to False
:param **replication_values: 1.2 only, additional values to ad to the replication data map
"""
with connection_manager() as con:
_, keyspaces = con.execute("""SELECT keyspace_name FROM system.schema_keyspaces""", {})
if name not in [r[0] for r in keyspaces]:
#try the 1.2 method
replication_map = {
'class': strategy_class,
'replication_factor':replication_factor
}
replication_map.update(replication_values)
if strategy_class.lower() != 'simplestrategy':
# Although the Cassandra documentation states for `replication_factor`
# that it is "Required if class is SimpleStrategy; otherwise,
# not used." we get an error if it is present.
replication_map.pop('replication_factor', None)
query = """
CREATE KEYSPACE {}
WITH REPLICATION = {}
""".format(name, json.dumps(replication_map).replace('"', "'"))
if strategy_class != 'SimpleStrategy':
query += " AND DURABLE_WRITES = {}".format('true' if durable_writes else 'false')
execute(query)
def execute(self):
if len(self.queries) == 0:
# Empty batch is a no-op
return
opener = 'BEGIN ' + (self.batch_type + ' ' if self.batch_type else '') + ' BATCH'
if self.timestamp:
if isinstance(self.timestamp, (int, long)):
ts = self.timestamp
elif isinstance(self.timestamp, timedelta):
ts = long((datetime.now() + self.timestamp - datetime.fromtimestamp(0)).total_seconds() * 1000000)
elif isinstance(self.timestamp, datetime):
ts = long((self.timestamp - datetime.fromtimestamp(0)).total_seconds() * 1000000)
else:
raise ValueError("Batch expects a long, a timedelta, or a datetime")
opener += ' USING TIMESTAMP {}'.format(ts)
query_list = [opener]
parameters = {}
ctx_counter = 0
for query in self.queries:
query.update_context_id(ctx_counter)
ctx = query.get_context()
ctx_counter += len(ctx)
query_list.append(' ' + str(query))
parameters.update(ctx)
query_list.append('APPLY BATCH;')
execute('\n'.join(query_list), parameters, self._consistency)
self.queries = []
def drop_table(model):
# don't try to delete non existant tables
meta = get_cluster().metadata
ks_name = model._get_keyspace()
raw_cf_name = model.column_family_name(include_keyspace=False)
try:
table = meta.keyspaces[ks_name].tables[raw_cf_name]
execute('drop table {};'.format(model.column_family_name(include_keyspace=True)))
except KeyError:
pass
def drop_table(model):
# don't try to delete non existant tables
ks_name = model._get_keyspace()
with connection_manager() as con:
tables = con.execute(
"SELECT columnfamily_name from system.schema_columnfamilies WHERE keyspace_name = %(ks_name)s",
{'ks_name': ks_name})
raw_cf_name = model.column_family_name(include_keyspace=False)
if raw_cf_name not in [t[0] for t in tables]:
return
cf_name = model.column_family_name()
execute('drop table {};'.format(cf_name))
def update_compaction(model):
"""Updates the compaction options for the given model if necessary.
:param model: The model to update.
:return: `True`, if the compaction options were modified in Cassandra,
`False` otherwise.
:rtype: bool
"""
logger.debug("Checking %s for compaction differences", model)
table = get_table_settings(model)
existing_options = table.options.copy()
existing_compaction_strategy = existing_options[
'compaction_strategy_class']
existing_options = json.loads(
existing_options['compaction_strategy_options'])
desired_options = get_compaction_options(model)
desired_compact_strategy = desired_options.get(
'class', SizeTieredCompactionStrategy)
desired_options.pop('class', None)
do_update = False
if desired_compact_strategy not in existing_compaction_strategy:
do_update = True
for k, v in desired_options.items():
val = existing_options.pop(k, None)
if val != v:
do_update = True
# check compaction_strategy_options
if do_update:
options = get_compaction_options(model)
# jsonify
options = json.dumps(options).replace('"', "'")
cf_name = model.column_family_name()
query = "ALTER TABLE {} with compaction = {}".format(cf_name, options)
logger.debug(query)
execute(query)
return True
return False
def drop_table(model):
# don't try to delete non existant tables
ks_name = model._get_keyspace()
with connection_manager() as con:
_, tables = con.execute(
"SELECT columnfamily_name from system.schema_columnfamilies WHERE keyspace_name = :ks_name",
{'ks_name': ks_name}
)
raw_cf_name = model.column_family_name(include_keyspace=False)
if raw_cf_name not in [t[0] for t in tables]:
return
cf_name = model.column_family_name()
execute('drop table {};'.format(cf_name))
def delete(self, columns=[]):
"""
Deletes the contents of a query
"""
#validate where clause
partition_key = self.model._primary_keys.values()[0]
if not any([c.column.db_field_name == partition_key.db_field_name for c in self._where]):
raise QueryException("The partition key must be defined on delete queries")
qs = ['DELETE FROM {}'.format(self.column_family_name)]
qs += ['WHERE {}'.format(self._where_clause())]
qs = ' '.join(qs)
if self._batch:
self._batch.add_query(qs, self._where_values())
else:
execute(qs, self._where_values())
def _execute_query(self):
if self._batch:
raise CQLEngineException("Only inserts, updates, and deletes are available in batch mode")
if self._result_cache is None:
self._result_cache = execute(self._select_query(), self._where_values())
field_names = set(sum([res._fields for res in self._result_cache], tuple()))
self._construct_result = self._get_result_constructor(field_names)
def update_compaction(model):
"""Updates the compaction options for the given model if necessary.
:param model: The model to update.
:return: `True`, if the compaction options were modified in Cassandra,
`False` otherwise.
:rtype: bool
"""
logger.debug("Checking %s for compaction differences", model)
table = get_table_settings(model)
existing_options = table.options.copy()
existing_compaction_strategy = existing_options['compaction_strategy_class']
existing_options = json.loads(existing_options['compaction_strategy_options'])
desired_options = get_compaction_options(model)
desired_compact_strategy = desired_options.get('class', SizeTieredCompactionStrategy)
desired_options.pop('class', None)
do_update = False
if desired_compact_strategy not in existing_compaction_strategy:
do_update = True
for k, v in desired_options.items():
val = existing_options.pop(k, None)
if val != v:
do_update = True
# check compaction_strategy_options
if do_update:
options = get_compaction_options(model)
# jsonify
options = json.dumps(options).replace('"', "'")
cf_name = model.column_family_name()
query = "ALTER TABLE {} with compaction = {}".format(cf_name, options)
logger.debug(query)
execute(query)
return True
return False
def _execute(self, q):
if self._batch:
return self._batch.add_query(q)
else:
tmp = execute(q, consistency_level=self._consistency)
if self._if_not_exists:
check_applied(tmp)
return tmp
def _execute_query(self):
if self._batch:
raise CQLEngineException(
"Only inserts, updates, and deletes are available in batch mode"
)
if self._result_cache is None:
columns, self._result_cache = execute(self._select_query(),
self._where_values())
self._construct_result = self._get_result_constructor(columns)
def update_compaction(model):
"""Updates the compaction options for the given model if necessary.
:param model: The model to update.
:return: `True`, if the compaction options were modified in Cassandra,
`False` otherwise.
:rtype: bool
"""
logger.debug("Checking %s for compaction differences", model)
row = get_table_settings(model)
# check compaction_strategy_class
if not model.__compaction__:
return
do_update = not row['compaction_strategy_class'].endswith(model.__compaction__)
existing_options = json.loads(row['compaction_strategy_options'])
# The min/max thresholds are stored differently in the system data dictionary
existing_options.update({
'min_threshold': str(row['min_compaction_threshold']),
'max_threshold': str(row['max_compaction_threshold']),
})
desired_options = get_compaction_options(model)
desired_options.pop('class', None)
for k, v in desired_options.items():
val = existing_options.pop(k, None)
if val != v:
do_update = True
# check compaction_strategy_options
if do_update:
options = get_compaction_options(model)
# jsonify
options = json.dumps(options).replace('"', "'")
cf_name = model.column_family_name()
query = "ALTER TABLE {} with compaction = {}".format(cf_name, options)
logger.debug(query)
execute(query)
return True
return False
def delete(self, columns=[]):
"""
Deletes the contents of a query
"""
#validate where clause
partition_key = self.model._primary_keys.values()[0]
if not any([
c.column.db_field_name == partition_key.db_field_name
for c in self._where
]):
raise QueryException(
"The partition key must be defined on delete queries")
qs = ['DELETE FROM {}'.format(self.column_family_name)]
qs += ['WHERE {}'.format(self._where_clause())]
qs = ' '.join(qs)
if self._batch:
self._batch.add_query(qs, self._where_values())
else:
execute(qs, self._where_values())
def delete(self):
""" Deletes one instance """
if self.instance is None:
raise CQLEngineException("DML Query intance attribute is None")
field_values = {}
qs = ['DELETE FROM {}'.format(self.column_family_name)]
qs += ['WHERE']
where_statements = []
for name, col in self.model._primary_keys.items():
field_id = uuid4().hex
field_values[field_id] = col.to_database(getattr(self.instance, name))
where_statements += ['"{}" = :{}'.format(col.db_field_name, field_id)]
qs += [' AND '.join(where_statements)]
qs = ' '.join(qs)
if self._batch:
self._batch.add_query(qs, field_values)
else:
execute(qs, field_values)
def _execute_query(self):
if self._batch:
raise CQLEngineException(
"Only inserts, updates, and deletes are available in batch mode"
)
if self._result_cache is None:
self._result_cache = execute(self._select_query(),
self._where_values())
field_names = set(
sum([res._fields for res in self._result_cache], tuple()))
self._construct_result = self._get_result_constructor(field_names)
def execute(self):
if len(self.queries) == 0:
# Empty batch is a no-op
# except for callbacks
self._execute_callbacks()
return
opener = 'BEGIN ' + (self.batch_type +
' ' if self.batch_type else '') + ' BATCH'
if self.timestamp:
if isinstance(self.timestamp, six.integer_types):
ts = self.timestamp
elif isinstance(self.timestamp, (datetime, timedelta)):
ts = self.timestamp
if isinstance(self.timestamp, timedelta):
ts += datetime.now() # Apply timedelta
ts = int(time.mktime(ts.timetuple()) * 1e+6 + ts.microsecond)
else:
raise ValueError(
"Batch expects a long, a timedelta, or a datetime")
opener += ' USING TIMESTAMP {}'.format(ts)
query_list = [opener]
parameters = {}
ctx_counter = 0
for query in self.queries:
query.update_context_id(ctx_counter)
ctx = query.get_context()
ctx_counter += len(ctx)
query_list.append(' ' + str(query))
parameters.update(ctx)
query_list.append('APPLY BATCH;')
execute('\n'.join(query_list), parameters, self._consistency)
self.queries = []
self._execute_callbacks()
def delete(self):
""" Deletes one instance """
if self.instance is None:
raise CQLEngineException("DML Query intance attribute is None")
field_values = {}
qs = ['DELETE FROM {}'.format(self.column_family_name)]
qs += ['WHERE']
where_statements = []
for name, col in self.model._primary_keys.items():
field_id = uuid4().hex
field_values[field_id] = getattr(self.instance, name)
where_statements += [
'"{}" = %({})s'.format(col.db_field_name, field_id)
]
qs += [' AND '.join(where_statements)]
qs = ' '.join(qs)
if self._batch:
self._batch.add_query(qs, field_values)
else:
execute(qs, field_values)
def execute(self):
if len(self.queries) == 0:
# Empty batch is a no-op
return
opener = 'BEGIN ' + (self.batch_type + ' ' if self.batch_type else '') + ' BATCH'
if self.timestamp:
epoch = datetime(1970, 1, 1)
ts = long((self.timestamp - epoch).total_seconds() * 1000)
opener += ' USING TIMESTAMP {}'.format(ts)
query_list = [opener]
parameters = {}
for query, params in self.queries:
query_list.append(' ' + query)
parameters.update(params)
query_list.append('APPLY BATCH;')
execute('\n'.join(query_list), parameters)
self.queries = []
def execute(self):
if len(self.queries) == 0:
# Empty batch is a no-op
# except for callbacks
self._execute_callbacks()
return
opener = 'BEGIN ' + (self.batch_type + ' ' if self.batch_type else '') + ' BATCH'
if self.timestamp:
if isinstance(self.timestamp, (int, long)):
ts = self.timestamp
elif isinstance(self.timestamp, (datetime, timedelta)):
ts = self.timestamp
if isinstance(self.timestamp, timedelta):
ts += datetime.now() # Apply timedelta
ts = long(time.mktime(ts.timetuple()) * 1e+6 + ts.microsecond)
else:
raise ValueError("Batch expects a long, a timedelta, or a datetime")
opener += ' USING TIMESTAMP {}'.format(ts)
query_list = [opener]
parameters = {}
ctx_counter = 0
for query in self.queries:
query.update_context_id(ctx_counter)
ctx = query.get_context()
ctx_counter += len(ctx)
query_list.append(' ' + str(query))
parameters.update(ctx)
query_list.append('APPLY BATCH;')
execute('\n'.join(query_list), parameters, self._consistency)
self.queries = []
self._execute_callbacks()
def execute(self):
if len(self.queries) == 0:
# Empty batch is a no-op
return
opener = 'BEGIN ' + (self.batch_type +
' ' if self.batch_type else '') + ' BATCH'
if self.timestamp:
epoch = datetime(1970, 1, 1)
ts = long((self.timestamp - epoch).total_seconds() * 1000)
opener += ' USING TIMESTAMP {}'.format(ts)
query_list = [opener]
parameters = {}
for query, params in self.queries:
query_list.append(' ' + query)
parameters.update(params)
query_list.append('APPLY BATCH;')
execute('\n'.join(query_list), parameters)
self.queries = []
def get_fields(model):
# returns all fields that aren't part of the PK
ks_name = model._get_keyspace()
col_family = model.column_family_name(include_keyspace=False)
field_types = ['regular', 'static']
query = "select * from system.schema_columns where keyspace_name = %s and columnfamily_name = %s"
tmp = execute(query, [ks_name, col_family])
# Tables containing only primary keys do not appear to create
# any entries in system.schema_columns, as only non-primary-key attributes
# appear to be inserted into the schema_columns table
try:
return [Field(x['column_name'], x['validator']) for x in tmp if x['type'] in field_types]
except KeyError:
return [Field(x['column_name'], x['validator']) for x in tmp]
def get_fields(model):
# returns all fields that aren't part of the PK
ks_name = model._get_keyspace()
col_family = model.column_family_name(include_keyspace=False)
query = "select * from system.schema_columns where keyspace_name = %s and columnfamily_name = %s"
tmp = execute(query, [ks_name, col_family])
# Tables containing only primary keys do not appear to create
# any entries in system.schema_columns, as only non-primary-key attributes
# appear to be inserted into the schema_columns table
try:
return [Field(x['column_name'], x['validator']) for x in tmp if x['type'] == 'regular']
except KeyError:
return [Field(x['column_name'], x['validator']) for x in tmp]
def count(self):
""" Returns the number of rows matched by this query """
if self._batch:
raise CQLEngineException("Only inserts, updates, and deletes are available in batch mode")
#TODO: check for previous query execution and return row count if it exists
if self._result_cache is None:
qs = ['SELECT COUNT(*)']
qs += ['FROM {}'.format(self.column_family_name)]
if self._where:
qs += ['WHERE {}'.format(self._where_clause())]
if self._allow_filtering:
qs += ['ALLOW FILTERING']
qs = ' '.join(qs)
_, result = execute(qs, self._where_values())
return result[0][0]
else:
return len(self._result_cache)
def count(self):
""" Returns the number of rows matched by this query """
if self._batch:
raise CQLEngineException(
"Only inserts, updates, and deletes are available in batch mode"
)
#TODO: check for previous query execution and return row count if it exists
if self._result_cache is None:
qs = ['SELECT COUNT(*)']
qs += ['FROM {}'.format(self.column_family_name)]
if self._where:
qs += ['WHERE {}'.format(self._where_clause())]
if self._allow_filtering:
qs += ['ALLOW FILTERING']
qs = ' '.join(qs)
result = execute(qs, self._where_values())
return result[0][0]
else:
return len(self._result_cache)
def _execute_query(self):
if self._batch:
raise CQLEngineException("Only inserts, updates, and deletes are available in batch mode")
if self._result_cache is None:
columns, self._result_cache = execute(self._select_query(), self._where_values())
self._construct_result = self._get_result_constructor(columns)
def sync_table(model, create_missing_keyspace=True):
if model.__abstract__:
raise CQLEngineException("cannot create table from abstract model")
# construct query string
cf_name = model.column_family_name()
raw_cf_name = model.column_family_name(include_keyspace=False)
ks_name = model._get_keyspace()
# create missing keyspace
if create_missing_keyspace:
create_keyspace(ks_name)
with connection_manager() as con:
tables = con.execute(
"SELECT columnfamily_name from system.schema_columnfamilies WHERE keyspace_name = %(ks_name)s",
{'ks_name': ks_name})
tables = [x.columnfamily_name for x in tables]
# check for an existing column family
if raw_cf_name not in tables:
qs = get_create_table(model)
try:
execute(qs)
except CQLEngineException as ex:
# 1.2 doesn't return cf names, so we have to examine the exception
# and ignore if it says the column family already exists
if "Cannot add already existing column family" not in unicode(ex):
raise
else:
# see if we're missing any columns
fields = get_fields(model)
field_names = [x.name for x in fields]
for name, col in model._columns.items():
if col.primary_key or col.partition_key:
continue # we can't mess with the PK
if col.db_field_name in field_names:
continue # skip columns already defined
# add missing column using the column def
query = "ALTER TABLE {} add {}".format(cf_name,
col.get_column_def())
logger.debug(query)
execute(query)
update_compaction(model)
# get existing index names, skip ones that already exist
with connection_manager() as con:
idx_names = con.execute(
"SELECT index_name from system.\"IndexInfo\" WHERE table_name=%(table_name)s",
{'table_name': raw_cf_name})
idx_names = [i.index_name for i in idx_names]
idx_names = filter(None, idx_names)
indexes = [c for n, c in model._columns.items() if c.index]
if indexes:
for column in indexes:
if column.db_index_name in idx_names:
continue
qs = [
'CREATE INDEX index_{}_{}'.format(raw_cf_name,
column.db_field_name)
]
qs += ['ON {}'.format(cf_name)]
qs += ['("{}")'.format(column.db_field_name)]
qs = ' '.join(qs)
try:
execute(qs)
except CQLEngineException:
# index already exists
pass
def delete_keyspace(name):
with connection_manager() as con:
_, keyspaces = con.execute(
"""SELECT keyspace_name FROM system.schema_keyspaces""", {})
if name in [r[0] for r in keyspaces]:
execute("DROP KEYSPACE {}".format(name))
def _execute(self, q):
if self._batch:
return self._batch.add_query(q)
else:
tmp = execute(q, consistency_level=self._consistency)
return tmp
def save(self):
"""
Creates / updates a row.
This is a blind insert call.
All validation and cleaning needs to happen
prior to calling this.
"""
if self.instance is None:
raise CQLEngineException("DML Query intance attribute is None")
assert type(self.instance) == self.model
# organize data
value_pairs = []
values = self.instance._as_dict()
# get defined fields and their column names
for name, col in self.model._columns.items():
val = values.get(name)
if val is None:
continue
value_pairs += [(col.db_field_name, val)]
# construct query string
field_names = zip(*value_pairs)[0]
field_ids = {n: uuid4().hex for n in field_names}
field_values = dict(value_pairs)
query_values = {field_ids[n]: field_values[n] for n in field_names}
qs = []
if self.instance._has_counter or self.instance._can_update():
qs += ["UPDATE {}".format(self.column_family_name)]
qs += ["SET"]
set_statements = []
# get defined fields and their column names
for name, col in self.model._columns.items():
if not col.is_primary_key:
val = values.get(name)
if val is None:
continue
if isinstance(col, (BaseContainerColumn, Counter)):
# remove value from query values, the column will
# handle it
query_values.pop(field_ids.get(name), None)
val_mgr = self.instance._values[name]
set_statements += col.get_update_statement(
val, val_mgr.previous_value, query_values)
else:
set_statements += [
'"{}" = %({})s'.format(col.db_field_name, field_ids[col.db_field_name])]
qs += [', '.join(set_statements)]
qs += ['WHERE']
where_statements = []
for name, col in self.model._primary_keys.items():
where_statements += ['"{}" = %({})s'.format(col.db_field_name,
field_ids[col.db_field_name])]
qs += [' AND '.join(where_statements)]
# clear the qs if there are no set statements and this is not a
# counter model
if not set_statements and not self.instance._has_counter:
qs = []
else:
qs += ["INSERT INTO {}".format(self.column_family_name)]
qs += ["({})".format(', '.join(['"{}"'.format(f)
for f in field_names]))]
qs += ['VALUES']
qs += ["({})".format(', '.join(['%(' + field_ids[f] + ')s' for f in field_names]))]
qs = ' '.join(qs)
# skip query execution if it's empty
# caused by pointless update queries
if qs:
if self._batch:
self._batch.add_query(qs, query_values)
else:
execute(qs, query_values)
# delete nulled columns and removed map keys
qs = ['DELETE']
query_values = {}
del_statements = []
for k, v in self.instance._values.items():
col = v.column
if v.deleted:
del_statements += ['"{}"'.format(col.db_field_name)]
elif isinstance(col, Map):
del_statements += col.get_delete_statement(
v.value, v.previous_value, query_values)
if del_statements:
qs += [', '.join(del_statements)]
qs += ['FROM {}'.format(self.column_family_name)]
qs += ['WHERE']
where_statements = []
for name, col in self.model._primary_keys.items():
field_id = uuid4().hex
query_values[field_id] = field_values[name]
where_statements += [
'"{}" = %({})s'.format(col.db_field_name, field_id)]
qs += [' AND '.join(where_statements)]
qs = ' '.join(qs)
if self._batch:
self._batch.add_query(qs, query_values)
else:
execute(qs, query_values)
def sync_table(model, create_missing_keyspace=True):
if model.__abstract__:
raise CQLEngineException("cannot create table from abstract model")
#construct query string
cf_name = model.column_family_name()
raw_cf_name = model.column_family_name(include_keyspace=False)
ks_name = model._get_keyspace()
#create missing keyspace
if create_missing_keyspace:
create_keyspace(ks_name)
with connection_manager() as con:
tables = con.execute(
"SELECT columnfamily_name from system.schema_columnfamilies WHERE keyspace_name = :ks_name",
{'ks_name': ks_name}
)
tables = [x[0] for x in tables.results]
#check for an existing column family
if raw_cf_name not in tables:
qs = get_create_table(model)
try:
execute(qs)
except CQLEngineException as ex:
# 1.2 doesn't return cf names, so we have to examine the exception
# and ignore if it says the column family already exists
if "Cannot add already existing column family" not in unicode(ex):
raise
else:
# see if we're missing any columns
fields = get_fields(model)
field_names = [x.name for x in fields]
for name, col in model._columns.items():
if col.primary_key or col.partition_key: continue # we can't mess with the PK
if col.db_field_name in field_names: continue # skip columns already defined
# add missing column using the column def
query = "ALTER TABLE {} add {}".format(cf_name, col.get_column_def())
logger.debug(query)
execute(query)
update_compaction(model)
#get existing index names, skip ones that already exist
with connection_manager() as con:
_, idx_names = con.execute(
"SELECT index_name from system.\"IndexInfo\" WHERE table_name=:table_name",
{'table_name': raw_cf_name}
)
idx_names = [i[0] for i in idx_names]
idx_names = filter(None, idx_names)
indexes = [c for n,c in model._columns.items() if c.index]
if indexes:
for column in indexes:
if column.db_index_name in idx_names: continue
qs = ['CREATE INDEX index_{}_{}'.format(raw_cf_name, column.db_field_name)]
qs += ['ON {}'.format(cf_name)]
qs += ['("{}")'.format(column.db_field_name)]
qs = ' '.join(qs)
try:
execute(qs)
except CQLEngineException:
# index already exists
pass
def _execute(self, q):
if self._batch:
return self._batch.add_query(q)
else:
return execute(q, consistency_level=self._consistency)
def delete_keyspace(name):
cluster = get_cluster()
if name in cluster.metadata.keyspaces:
execute("DROP KEYSPACE {}".format(name))
def sync_table(model, create_missing_keyspace=True):
"""
Inspects the model and creates / updates the corresponding table and columns.
Note that the attributes removed from the model are not deleted on the database.
They become effectively ignored by (will not show up on) the model.
:param create_missing_keyspace: (Defaults to True) Flags to us that we need to create missing keyspace
mentioned in the model automatically.
:type create_missing_keyspace: bool
"""
if not issubclass(model, Model):
raise CQLEngineException("Models must be derived from base Model.")
if model.__abstract__:
raise CQLEngineException("cannot create table from abstract model")
#construct query string
cf_name = model.column_family_name()
raw_cf_name = model.column_family_name(include_keyspace=False)
ks_name = model._get_keyspace()
#create missing keyspace
if create_missing_keyspace:
create_keyspace(ks_name)
cluster = get_cluster()
keyspace = cluster.metadata.keyspaces[ks_name]
tables = keyspace.tables
#check for an existing column family
if raw_cf_name not in tables:
qs = get_create_table(model)
try:
execute(qs)
except CQLEngineException as ex:
# 1.2 doesn't return cf names, so we have to examine the exception
# and ignore if it says the column family already exists
if "Cannot add already existing column family" not in unicode(ex):
raise
else:
# see if we're missing any columns
fields = get_fields(model)
field_names = [x.name for x in fields]
for name, col in model._columns.items():
if col.primary_key or col.partition_key: continue # we can't mess with the PK
if col.db_field_name in field_names: continue # skip columns already defined
# add missing column using the column def
query = "ALTER TABLE {} add {}".format(cf_name, col.get_column_def())
logger.debug(query)
execute(query)
update_compaction(model)
table = cluster.metadata.keyspaces[ks_name].tables[raw_cf_name]
indexes = [c for n,c in model._columns.items() if c.index]
for column in indexes:
if table.columns[column.db_field_name].index:
continue
qs = ['CREATE INDEX index_{}_{}'.format(raw_cf_name, column.db_field_name)]
qs += ['ON {}'.format(cf_name)]
qs += ['("{}")'.format(column.db_field_name)]
qs = ' '.join(qs)
execute(qs)
def save(self):
"""
Creates / updates a row.
This is a blind insert call.
All validation and cleaning needs to happen
prior to calling this.
"""
if self.instance is None:
raise CQLEngineException("DML Query intance attribute is None")
assert type(self.instance) == self.model
#organize data
value_pairs = []
values = self.instance._as_dict()
#get defined fields and their column names
for name, col in self.model._columns.items():
val = values.get(name)
if val is None: continue
value_pairs += [(col.db_field_name, val)]
#construct query string
field_names = zip(*value_pairs)[0]
field_ids = {n: uuid4().hex for n in field_names}
field_values = dict(value_pairs)
query_values = {field_ids[n]: field_values[n] for n in field_names}
qs = []
if self.instance._has_counter or self.instance._can_update():
qs += ["UPDATE {}".format(self.column_family_name)]
qs += ["SET"]
set_statements = []
#get defined fields and their column names
for name, col in self.model._columns.items():
if not col.is_primary_key:
val = values.get(name)
if val is None:
continue
if isinstance(col, (BaseContainerColumn, Counter)):
#remove value from query values, the column will handle it
query_values.pop(field_ids.get(name), None)
val_mgr = self.instance._values[name]
set_statements += col.get_update_statement(
val, val_mgr.previous_value, query_values)
else:
set_statements += [
'"{}" = %({})s'.format(
col.db_field_name,
field_ids[col.db_field_name])
]
qs += [', '.join(set_statements)]
qs += ['WHERE']
where_statements = []
for name, col in self.model._primary_keys.items():
where_statements += [
'"{}" = %({})s'.format(col.db_field_name,
field_ids[col.db_field_name])
]
qs += [' AND '.join(where_statements)]
# clear the qs if there are no set statements and this is not a counter model
if not set_statements and not self.instance._has_counter:
qs = []
else:
qs += ["INSERT INTO {}".format(self.column_family_name)]
qs += [
"({})".format(', '.join(
['"{}"'.format(f) for f in field_names]))
]
qs += ['VALUES']
qs += [
"({})".format(', '.join(
['%(' + field_ids[f] + ')s' for f in field_names]))
]
qs = ' '.join(qs)
# skip query execution if it's empty
# caused by pointless update queries
if qs:
if self._batch:
self._batch.add_query(qs, query_values)
else:
execute(qs, query_values)
# delete nulled columns and removed map keys
qs = ['DELETE']
query_values = {}
del_statements = []
for k, v in self.instance._values.items():
col = v.column
if v.deleted:
del_statements += ['"{}"'.format(col.db_field_name)]
elif isinstance(col, Map):
del_statements += col.get_delete_statement(
v.value, v.previous_value, query_values)
if del_statements:
qs += [', '.join(del_statements)]
qs += ['FROM {}'.format(self.column_family_name)]
qs += ['WHERE']
where_statements = []
for name, col in self.model._primary_keys.items():
field_id = uuid4().hex
query_values[field_id] = field_values[name]
where_statements += [
'"{}" = %({})s'.format(col.db_field_name, field_id)
]
qs += [' AND '.join(where_statements)]
qs = ' '.join(qs)
if self._batch:
self._batch.add_query(qs, query_values)
else:
execute(qs, query_values)
def sync_table(model):
"""
Inspects the model and creates / updates the corresponding table and columns.
Note that the attributes removed from the model are not deleted on the database.
They become effectively ignored by (will not show up on) the model.
:param create_missing_keyspace: (Defaults to True) Flags to us that we need to create missing keyspace
mentioned in the model automatically.
:type create_missing_keyspace: bool
"""
if not issubclass(model, Model):
raise CQLEngineException("Models must be derived from base Model.")
if model.__abstract__:
raise CQLEngineException("cannot create table from abstract model")
#construct query string
cf_name = model.column_family_name()
raw_cf_name = model.column_family_name(include_keyspace=False)
ks_name = model._get_keyspace()
cluster = get_cluster()
keyspace = cluster.metadata.keyspaces[ks_name]
tables = keyspace.tables
#check for an existing column family
if raw_cf_name not in tables:
qs = get_create_table(model)
try:
execute(qs)
except CQLEngineException as ex:
# 1.2 doesn't return cf names, so we have to examine the exception
# and ignore if it says the column family already exists
if "Cannot add already existing column family" not in unicode(ex):
raise
else:
# see if we're missing any columns
fields = get_fields(model)
field_names = [x.name for x in fields]
for name, col in model._columns.items():
if col.primary_key or col.partition_key:
continue # we can't mess with the PK
if col.db_field_name in field_names:
continue # skip columns already defined
# add missing column using the column def
query = "ALTER TABLE {} add {}".format(cf_name,
col.get_column_def())
logger.debug(query)
execute(query)
update_compaction(model)
table = cluster.metadata.keyspaces[ks_name].tables[raw_cf_name]
indexes = [c for n, c in model._columns.items() if c.index]
for column in indexes:
if table.columns[column.db_field_name].index:
continue
qs = [
'CREATE INDEX index_{}_{}'.format(raw_cf_name,
column.db_field_name)
]
qs += ['ON {}'.format(cf_name)]
qs += ['("{}")'.format(column.db_field_name)]
qs = ' '.join(qs)
execute(qs)
def test_extra_field(self):
drop_table(self.TestModel)
sync_table(self.TestModel)
self.TestModel.create()
execute("ALTER TABLE {} add blah int".format(self.TestModel.column_family_name(include_keyspace=True)))
self.TestModel.objects().all()
def delete_keyspace(name):
cluster = get_cluster()
if name in cluster.metadata.keyspaces:
execute("DROP KEYSPACE {}".format(name))
def delete_keyspace(name):
with connection_manager() as con:
_, keyspaces = con.execute("""SELECT keyspace_name FROM system.schema_keyspaces""", {})
if name in [r[0] for r in keyspaces]:
execute("DROP KEYSPACE {}".format(name))
