def test_sync_table_works_with_primary_keys_only_tables(self):
# This is "create table":
sync_table(PrimaryKeysOnlyModel)
# let's make sure settings persisted correctly:
assert PrimaryKeysOnlyModel.__compaction__ == LeveledCompactionStrategy
# blows up with DoesNotExist if table does not exist
table_settings = management.get_table_settings(PrimaryKeysOnlyModel)
# let make sure the flag we care about
assert LeveledCompactionStrategy in table_settings.options['compaction_strategy_class']
# Now we are "updating" the table:
# setting up something to change
PrimaryKeysOnlyModel.__compaction__ = SizeTieredCompactionStrategy
# primary-keys-only tables do not create entries in system.schema_columns
# table. Only non-primary keys are added to that table.
# Our code must deal with that eventuality properly (not crash)
# on subsequent runs of sync_table (which runs get_fields internally)
get_fields(PrimaryKeysOnlyModel)
sync_table(PrimaryKeysOnlyModel)
table_settings = management.get_table_settings(PrimaryKeysOnlyModel)
assert SizeTieredCompactionStrategy in table_settings.options['compaction_strategy_class']
def test_sync_table_works_with_primary_keys_only_tables(self):
# This is "create table":
sync_table(PrimaryKeysOnlyModel)
# let's make sure settings persisted correctly:
assert PrimaryKeysOnlyModel.__compaction__ == LeveledCompactionStrategy
# blows up with DoesNotExist if table does not exist
table_settings = management.get_table_settings(PrimaryKeysOnlyModel)
# let make sure the flag we care about
assert LeveledCompactionStrategy in table_settings.options[
'compaction_strategy_class']
# Now we are "updating" the table:
# setting up something to change
PrimaryKeysOnlyModel.__compaction__ = SizeTieredCompactionStrategy
# primary-keys-only tables do not create entries in system.schema_columns
# table. Only non-primary keys are added to that table.
# Our code must deal with that eventuality properly (not crash)
# on subsequent runs of sync_table (which runs get_fields internally)
get_fields(PrimaryKeysOnlyModel)
sync_table(PrimaryKeysOnlyModel)
table_settings = management.get_table_settings(PrimaryKeysOnlyModel)
assert SizeTieredCompactionStrategy in table_settings.options[
'compaction_strategy_class']
def test_add_column(self):
create_table(FirstModel)
fields = get_fields(FirstModel)
# this should contain the second key
self.assertEqual(len(fields), 2)
# get schema
create_table(SecondModel)
fields = get_fields(FirstModel)
self.assertEqual(len(fields), 3)
create_table(ThirdModel)
fields = get_fields(FirstModel)
self.assertEqual(len(fields), 4)
create_table(FourthModel)
fields = get_fields(FirstModel)
self.assertEqual(len(fields), 4)
def verify(*models, **kwargs):
ignore_extra = kwargs.get('ignore_extra', {})
results = {}
by_keyspace = {}
by_cf = {}
results = {}
for model in models:
ks_name = model._get_keyspace()
try:
by_keyspace[ks_name].add(model)
except KeyError:
by_keyspace[ks_name] = set([model])
cf_name = model.column_family_name(include_keyspace=False)
by_cf[cf_name] = model
results[model] = VerifyResult(model)
for keyspace, models in by_keyspace.items():
query_result = cqlengine.connection.get_session().execute(
"SELECT columnfamily_name, key_aliases, key_validator, column_aliases, comparator from system.schema_columnfamilies WHERE keyspace_name = %(ks_name)s",
{'ks_name': ks_name})
tables = {}
for result in query_result:
columnfamily_name = result['columnfamily_name']
partition_keys = result['key_aliases']
partition_key_types = result['key_validator']
primary_keys = result['column_aliases']
primary_key_types = result['comparator']
partition_keys = json.loads(partition_keys)
if len(partition_keys) > 1:
partition_key_types = partition_key_types[len('org.apache.cassandra.db.marshal.CompositeType('):-1].split(',')[:len(partition_keys)]
else:
partition_key_types = [partition_key_types]
primary_keys = json.loads(primary_keys)
primary_key_types = primary_key_types[len('org.apache.cassandra.db.marshal.CompositeType('):].split(',')[:len(primary_keys)]
item = {
'cf': columnfamily_name,
'partition_keys': partition_keys,
'partition_key_types': partition_key_types,
'primary_keys': primary_keys,
'primary_key_types': primary_key_types
}
tables[columnfamily_name] = item
for model in models:
cf_name = model.column_family_name(include_keyspace=False)
db_field_names = {col.db_field_name: col for name, col in model._columns.items()}
result = results[model]
# Check that model's cf is in db's tables.
if cf_name not in tables:
result.is_missing = True
else:
table_info = tables[cf_name]
fields = get_fields(model)
fields = {field.name: field.type for field in fields}
for name, field_type in fields.iteritems():
# If field is missing, that's an error.
if name not in db_field_names:
result.extra.add(name)
# If field is present, check the type.
else:
col = db_field_names[name]
if isinstance(col, columns.Map):
if not field_type.startswith('org.apache.cassandra.db.marshal.MapType'):
result.different.add(col.column_name)
elif isinstance(col, columns.List):
if not field_type.startswith('org.apache.cassandra.db.marshal.ListType'):
result.different.add(col.column_name)
elif isinstance(col, columns.Set):
if not field_type.startswith('org.apache.cassandra.db.marshal.SetType'):
result.different.add(col.column_name)
else:
local_metadata = _type_to_metadata(col.db_type)
if local_metadata != field_type:
result.different.add(col.column_name)
for name, kind in zip(table_info['partition_keys'], table_info['partition_key_types']):
if name not in db_field_names:
result.extra.add(name)
else:
col = db_field_names[name]
local_metadata = _type_to_metadata(col.db_type)
if local_metadata != kind:
result.different.add(col.column_name)
for name, kind in zip(table_info['primary_keys'], table_info['primary_key_types']):
if name not in db_field_names:
result.extra.add(name)
else:
col = db_field_names[name]
local_metadata = _type_to_metadata(col.db_type)
if col.clustering_order == 'desc':
local_metadata = u'org.apache.cassandra.db.marshal.ReversedType({})'.format(local_metadata)
if local_metadata != kind:
result.different.add(col.column_name)
for name, col in db_field_names.items():
# Handle primary keys from table-level data.
if col.primary_key:
if col.partition_key:
if name not in table_info['partition_keys']:
result.missing.add(col.column_name)
else:
local_metadata = _type_to_metadata(col.db_type)
i = table_info['partition_keys'].index(name)
if local_metadata != table_info['partition_key_types'][i]:
result.different.add(col.column_name)
else:
if name not in table_info['primary_keys']:
result.missing.add(col.column_name)
else:
local_metadata = _type_to_metadata(col.db_type)
if col.clustering_order == 'desc':
local_metadata = u'org.apache.cassandra.db.marshal.ReversedType({})'.format(local_metadata)
i = table_info['primary_keys'].index(name)
if local_metadata != table_info['primary_key_types'][i]:
result.different.add(col.column_name)
# Primary keys are not listed in fields.
if not col.primary_key and name not in fields:
result.missing.add(col.column_name)
for cf in tables:
if cf not in by_cf and cf not in ignore_extra:
result = VerifyResult(cf)
result.is_extra = True
results[cf] = result
model_indexes = {}
for model in models:
this_model_indexes = {col.db_field_name: col for name, col in model._columns.items() if col.index}
if this_model_indexes:
model_indexes[model.column_family_name(include_keyspace=False)] = this_model_indexes
query_results = cqlengine.connection.get_session().execute(
"SELECT index_name from system.\"IndexInfo\" WHERE table_name=%(table_name)s",
{'table_name': model._get_keyspace()})
cassandra_indexes = {}
for result_dict in query_results:
idx = result_dict['index_name']
try:
cf, index_name = idx.split('.')
look_for = 'index_{}_'.format(cf)
index_name = index_name[len(look_for):]
except ValueError:
cf = None
index_name = None
if cf:
try:
cassandra_indexes[cf].add(index_name)
except KeyError:
cassandra_indexes[cf] = set([index_name])
for cf, index_names in cassandra_indexes.items():
if cf not in model_indexes:
if cf not in by_cf:
result = VerifyResult(cf)
result.is_extra = True
results[cf] = result
else:
model = by_cf[cf]
result = results[model]
result.extra_indexes.add(index_name)
else:
this_model_indexes = model_indexes[cf]
if index_name not in this_model_indexes:
model = by_cf[cf]
result = results[model]
result.extra_indexes.add(index_name)
for cf, this_model_indexes in model_indexes.items():
for index_name in this_model_indexes.keys():
if cf not in cassandra_indexes or index_name not in cassandra_indexes[cf]:
model = by_cf[cf]
result = results[model]
result.missing_indexes.add(index_name)
results = {model: result for model, result in results.items() if result.has_errors()}
return results.values()