def _update_options(model, connection=None):
"""Updates the table options for the given model if necessary.
:param model: The model to update.
:param connection: Name of the connection to use
:return: `True`, if the options were modified in Cassandra,
`False` otherwise.
:rtype: bool
"""
ks_name = model._get_keyspace()
msg = format_log_context("Checking %s for option differences",
keyspace=ks_name,
connection=connection)
log.debug(msg, model)
model_options = model.__options__ or {}
table_meta = _get_table_metadata(model, connection=connection)
# go to CQL string first to normalize meta from different versions
existing_option_strings = set(
table_meta._make_option_strings(table_meta.options))
existing_options = _options_map_from_strings(existing_option_strings)
model_option_strings = metadata.TableMetadataV3._make_option_strings(
model_options)
model_options = _options_map_from_strings(model_option_strings)
update_options = {}
for name, value in model_options.items():
try:
existing_value = existing_options[name]
except KeyError:
msg = format_log_context(
"Invalid table option: '%s'; known options: %s",
keyspace=ks_name,
connection=connection)
raise KeyError(msg % (name, existing_options.keys()))
if isinstance(existing_value, six.string_types):
if value != existing_value:
update_options[name] = value
else:
try:
for k, v in value.items():
if existing_value[k] != v:
update_options[name] = value
break
except KeyError:
update_options[name] = value
if update_options:
options = ' AND '.join(
metadata.TableMetadataV3._make_option_strings(update_options))
query = "ALTER TABLE {0} WITH {1}".format(model.column_family_name(),
options)
execute(query, connection=connection)
return True
return False
def __create_keyspace(name, durable_writes, strategy_class, strategy_options, connection=None):
cluster = get_cluster(connection)
if name not in cluster.metadata.keyspaces:
log.info(format_log_context("Creating keyspace %s", connection=connection), name)
ks_meta = metadata.KeyspaceMetadata(name, durable_writes, strategy_class, strategy_options)
execute(ks_meta.as_cql_query(), connection=connection)
else:
log.info(format_log_context("Not creating keyspace %s because it already exists", connection=connection), name)
def __create_keyspace(name, durable_writes, strategy_class, strategy_options, connection=None):
cluster = get_cluster(connection)
if name not in cluster.metadata.keyspaces:
log.info(format_log_context("Creating keyspace %s", connection=connection), name)
ks_meta = metadata.KeyspaceMetadata(name, durable_writes, strategy_class, strategy_options)
execute(ks_meta.as_cql_query(), connection=connection)
else:
log.info(format_log_context("Not creating keyspace %s because it already exists", connection=connection), name)
def _sync_type(ks_name, type_model, omit_subtypes=None, connection=None):
syncd_sub_types = omit_subtypes or set()
for field in type_model._fields.values():
udts = []
columns.resolve_udts(field, udts)
for udt in [u for u in udts if u not in syncd_sub_types]:
_sync_type(ks_name, udt, syncd_sub_types, connection=connection)
syncd_sub_types.add(udt)
type_name = type_model.type_name()
type_name_qualified = "%s.%s" % (ks_name, type_name)
cluster = get_cluster(connection)
keyspace = cluster.metadata.keyspaces[ks_name]
defined_types = keyspace.user_types
if type_name not in defined_types:
log.debug(format_log_context("sync_type creating new type %s", keyspace=ks_name, connection=connection),
type_name_qualified)
cql = get_create_type(type_model, ks_name)
execute(cql, connection=connection)
cluster.refresh_user_type_metadata(ks_name, type_name)
type_model.register_for_keyspace(ks_name, connection=connection)
else:
type_meta = defined_types[type_name]
defined_fields = type_meta.field_names
model_fields = set()
for field in type_model._fields.values():
model_fields.add(field.db_field_name)
if field.db_field_name not in defined_fields:
execute("ALTER TYPE {0} ADD {1}".format(type_name_qualified, field.get_column_def()),
connection=connection)
else:
field_type = type_meta.field_types[defined_fields.index(field.db_field_name)]
if field_type != field.db_type:
msg = format_log_context(
'Existing user type {0} has field "{1}" with a type ({2}) differing from the model user type ({3}).'
' UserType should be updated.', keyspace=ks_name, connection=connection)
msg = msg.format(type_name_qualified, field.db_field_name, field_type, field.db_type)
warnings.warn(msg)
log.warning(msg)
type_model.register_for_keyspace(ks_name, connection=connection)
if len(defined_fields) == len(model_fields):
log.info(
format_log_context("Type %s did not require synchronization", keyspace=ks_name, connection=connection),
type_name_qualified)
return
db_fields_not_in_model = model_fields.symmetric_difference(defined_fields)
if db_fields_not_in_model:
msg = format_log_context("Type %s has fields not referenced by model: %s", keyspace=ks_name,
connection=connection)
log.info(msg, type_name_qualified, db_fields_not_in_model)
def _sync_type(ks_name, type_model, omit_subtypes=None, connection=None):
syncd_sub_types = omit_subtypes or set()
for field in type_model._fields.values():
udts = []
columns.resolve_udts(field, udts)
for udt in [u for u in udts if u not in syncd_sub_types]:
_sync_type(ks_name, udt, syncd_sub_types, connection=connection)
syncd_sub_types.add(udt)
type_name = type_model.type_name()
type_name_qualified = "%s.%s" % (ks_name, type_name)
cluster = get_cluster(connection)
keyspace = cluster.metadata.keyspaces[ks_name]
defined_types = keyspace.user_types
if type_name not in defined_types:
log.debug(format_log_context("sync_type creating new type %s", keyspace=ks_name, connection=connection), type_name_qualified)
cql = get_create_type(type_model, ks_name)
execute(cql, connection=connection)
cluster.refresh_user_type_metadata(ks_name, type_name)
type_model.register_for_keyspace(ks_name, connection=connection)
else:
type_meta = defined_types[type_name]
defined_fields = type_meta.field_names
model_fields = set()
for field in type_model._fields.values():
model_fields.add(field.db_field_name)
if field.db_field_name not in defined_fields:
execute("ALTER TYPE {0} ADD {1}".format(type_name_qualified, field.get_column_def()), connection=connection)
else:
field_type = type_meta.field_types[defined_fields.index(field.db_field_name)]
if field_type != field.db_type:
msg = format_log_context('Existing user type {0} has field "{1}" with a type ({2}) differing from the model user type ({3}).'
' UserType should be updated.', keyspace=ks_name, connection=connection)
msg = msg.format(type_name_qualified, field.db_field_name, field_type, field.db_type)
warnings.warn(msg)
log.warning(msg)
type_model.register_for_keyspace(ks_name, connection=connection)
if len(defined_fields) == len(model_fields):
log.info(format_log_context("Type %s did not require synchronization", keyspace=ks_name, connection=connection), type_name_qualified)
return
db_fields_not_in_model = model_fields.symmetric_difference(defined_fields)
if db_fields_not_in_model:
msg = format_log_context("Type %s has fields not referenced by model: %s", keyspace=ks_name, connection=connection)
log.info(msg, type_name_qualified, db_fields_not_in_model)
def _update_options(model, connection=None):
"""Updates the table options for the given model if necessary.
:param model: The model to update.
:param connection: Name of the connection to use
:return: `True`, if the options were modified in Cassandra,
`False` otherwise.
:rtype: bool
"""
ks_name = model._get_keyspace()
msg = format_log_context("Checking %s for option differences", keyspace=ks_name, connection=connection)
log.debug(msg, model)
model_options = model.__options__ or {}
table_meta = _get_table_metadata(model, connection=connection)
# go to CQL string first to normalize meta from different versions
existing_option_strings = set(table_meta._make_option_strings(table_meta.options))
existing_options = _options_map_from_strings(existing_option_strings)
model_option_strings = metadata.TableMetadataV3._make_option_strings(model_options)
model_options = _options_map_from_strings(model_option_strings)
update_options = {}
for name, value in model_options.items():
try:
existing_value = existing_options[name]
except KeyError:
msg = format_log_context("Invalid table option: '%s'; known options: %s", keyspace=ks_name, connection=connection)
raise KeyError(msg % (name, existing_options.keys()))
if isinstance(existing_value, six.string_types):
if value != existing_value:
update_options[name] = value
else:
try:
for k, v in value.items():
if existing_value[k] != v:
update_options[name] = value
break
except KeyError:
update_options[name] = value
if update_options:
options = ' AND '.join(metadata.TableMetadataV3._make_option_strings(update_options))
query = "ALTER TABLE {0} WITH {1}".format(model.column_family_name(), options)
execute(query, connection=connection)
return True
return False
def _sync_table(model, connection=None):
if not _allow_schema_modification():
return
if not issubclass(model, Model):
raise CQLEngineException("Models must be derived from base Model.")
if model.__abstract__:
raise CQLEngineException("cannot create table from abstract model")
cf_name = model.column_family_name()
raw_cf_name = model._raw_column_family_name()
ks_name = model._get_keyspace()
connection = connection or model._get_connection()
cluster = get_cluster(connection)
try:
keyspace = cluster.metadata.keyspaces[ks_name]
except KeyError:
msg = format_log_context("Keyspace '{0}' for model {1} does not exist.", connection=connection)
raise CQLEngineException(msg.format(ks_name, model))
tables = keyspace.tables
syncd_types = set()
for col in model._columns.values():
udts = []
columns.resolve_udts(col, udts)
for udt in [u for u in udts if u not in syncd_types]:
_sync_type(ks_name, udt, syncd_types, connection=connection)
if raw_cf_name not in tables:
log.debug(format_log_context("sync_table creating new table %s", keyspace=ks_name, connection=connection), cf_name)
qs = _get_create_table(model)
try:
execute(qs, connection=connection)
except CQLEngineException as ex:
# 1.2 doesn't return cf names, so we have to examine the exception
# and ignore if it says the column family already exists
if "Cannot add already existing column family" not in unicode(ex):
raise
else:
log.debug(format_log_context("sync_table checking existing table %s", keyspace=ks_name, connection=connection), cf_name)
table_meta = tables[raw_cf_name]
_validate_pk(model, table_meta)
table_columns = table_meta.columns
model_fields = set()
for model_name, col in model._columns.items():
db_name = col.db_field_name
model_fields.add(db_name)
if db_name in table_columns:
col_meta = table_columns[db_name]
if col_meta.cql_type != col.db_type:
msg = format_log_context('Existing table {0} has column "{1}" with a type ({2}) differing from the model type ({3}).'
' Model should be updated.', keyspace=ks_name, connection=connection)
msg = msg.format(cf_name, db_name, col_meta.cql_type, col.db_type)
warnings.warn(msg)
log.warning(msg)
continue
if col.primary_key or col.primary_key:
msg = format_log_context("Cannot add primary key '{0}' (with db_field '{1}') to existing table {2}", keyspace=ks_name, connection=connection)
raise CQLEngineException(msg.format(model_name, db_name, cf_name))
query = "ALTER TABLE {0} add {1}".format(cf_name, col.get_column_def())
execute(query, connection=connection)
db_fields_not_in_model = model_fields.symmetric_difference(table_columns)
if db_fields_not_in_model:
msg = format_log_context("Table {0} has fields not referenced by model: {1}", keyspace=ks_name, connection=connection)
log.info(msg.format(cf_name, db_fields_not_in_model))
_update_options(model, connection=connection)
table = cluster.metadata.keyspaces[ks_name].tables[raw_cf_name]
indexes = [c for n, c in model._columns.items() if c.index]
# TODO: support multiple indexes in C* 3.0+
for column in indexes:
index_name = _get_index_name_by_column(table, column.db_field_name)
if index_name:
continue
qs = ['CREATE INDEX']
qs += ['ON {0}'.format(cf_name)]
qs += ['("{0}")'.format(column.db_field_name)]
qs = ' '.join(qs)
execute(qs, connection=connection)
def _sync_table(model, connection=None):
if not _allow_schema_modification():
return
if not issubclass(model, Model):
raise CQLEngineException("Models must be derived from base Model.")
if model.__abstract__:
raise CQLEngineException("cannot create table from abstract model")
cf_name = model.column_family_name()
raw_cf_name = model._raw_column_family_name()
ks_name = model._get_keyspace()
connection = connection or model._get_connection()
cluster = get_cluster(connection)
try:
keyspace = cluster.metadata.keyspaces[ks_name]
except KeyError:
msg = format_log_context("Keyspace '{0}' for model {1} does not exist.", connection=connection)
raise CQLEngineException(msg.format(ks_name, model))
tables = keyspace.tables
syncd_types = set()
for col in model._columns.values():
udts = []
columns.resolve_udts(col, udts)
for udt in [u for u in udts if u not in syncd_types]:
_sync_type(ks_name, udt, syncd_types, connection=connection)
if raw_cf_name not in tables:
log.debug(format_log_context("sync_table creating new table %s", keyspace=ks_name, connection=connection), cf_name)
qs = _get_create_table(model)
try:
execute(qs, connection=connection)
except CQLEngineException as ex:
# 1.2 doesn't return cf names, so we have to examine the exception
# and ignore if it says the column family already exists
if "Cannot add already existing column family" not in unicode(ex):
raise
else:
log.debug(format_log_context("sync_table checking existing table %s", keyspace=ks_name, connection=connection), cf_name)
table_meta = tables[raw_cf_name]
_validate_pk(model, table_meta)
table_columns = table_meta.columns
model_fields = set()
for model_name, col in model._columns.items():
db_name = col.db_field_name
model_fields.add(db_name)
if db_name in table_columns:
col_meta = table_columns[db_name]
if col_meta.cql_type != col.db_type:
msg = format_log_context('Existing table {0} has column "{1}" with a type ({2}) differing from the model type ({3}).'
' Model should be updated.', keyspace=ks_name, connection=connection)
msg = msg.format(cf_name, db_name, col_meta.cql_type, col.db_type)
warnings.warn(msg)
log.warning(msg)
continue
if col.primary_key or col.primary_key:
msg = format_log_context("Cannot add primary key '{0}' (with db_field '{1}') to existing table {2}", keyspace=ks_name, connection=connection)
raise CQLEngineException(msg.format(model_name, db_name, cf_name))
query = "ALTER TABLE {0} add {1}".format(cf_name, col.get_column_def())
execute(query, connection=connection)
db_fields_not_in_model = model_fields.symmetric_difference(table_columns)
if db_fields_not_in_model:
msg = format_log_context("Table {0} has fields not referenced by model: {1}", keyspace=ks_name, connection=connection)
log.info(msg.format(cf_name, db_fields_not_in_model))
_update_options(model, connection=connection)
table = cluster.metadata.keyspaces[ks_name].tables[raw_cf_name]
indexes = [c for n, c in model._columns.items() if c.index]
# TODO: support multiple indexes in C* 3.0+
for column in indexes:
index_name = _get_index_name_by_column(table, column.db_field_name)
if index_name:
continue
qs = ['CREATE INDEX']
qs += ['ON {0}'.format(cf_name)]
qs += ['("{0}")'.format(column.db_field_name)]
qs = ' '.join(qs)
execute(qs, connection=connection)
