def get_new_connection(self, connection_settings): contact_points = connection_settings.pop( 'contact_points', self.default_settings['CONTACT_POINTS'] ) keyspace = connection_settings.pop( 'keyspace', self.settings_dict['DEFAULT_KEYSPACE'] ) self.keyspace = keyspace try: connection.get_connection() except CQLEngineException: connection.setup( contact_points, keyspace, **connection_settings ) self.session = connection.get_session() self.cluster = connection.get_cluster() self.session.default_timeout = None # Should be in config. return CassandraCursor(self.session)
def test_create_drop_succeeeds(self): cluster = get_cluster() keyspace_ss = 'test_ks_ss' self.assertFalse(keyspace_ss in cluster.metadata.keyspaces) management.create_keyspace_simple(keyspace_ss, 2) self.assertTrue(keyspace_ss in cluster.metadata.keyspaces) management.drop_keyspace(keyspace_ss) self.assertFalse(keyspace_ss in cluster.metadata.keyspaces) with warnings.catch_warnings(record=True) as w: management.create_keyspace(keyspace_ss, strategy_class="SimpleStrategy", replication_factor=1) self.assertEqual(len(w), 1) self.assertEqual(w[-1].category, DeprecationWarning) self.assertTrue(keyspace_ss in cluster.metadata.keyspaces) management.drop_keyspace(keyspace_ss) self.assertFalse(keyspace_ss in cluster.metadata.keyspaces) keyspace_nts = 'test_ks_nts' self.assertFalse(keyspace_nts in cluster.metadata.keyspaces) management.create_keyspace_simple(keyspace_nts, 2) self.assertTrue(keyspace_nts in cluster.metadata.keyspaces) with warnings.catch_warnings(record=True) as w: management.delete_keyspace(keyspace_nts) self.assertEqual(len(w), 1) self.assertEqual(w[-1].category, DeprecationWarning) self.assertFalse(keyspace_nts in cluster.metadata.keyspaces)
def _get_table_metadata(model, connection=None): # returns the table as provided by the native driver for a given model cluster = get_cluster(connection) ks = model._get_keyspace() table = model._raw_column_family_name() table = cluster.metadata.keyspaces[ks].tables[table] return table
def get_new_connection(self, connection_settings): contact_points = connection_settings.pop( 'contact_points', self.default_settings['CONTACT_POINTS'] ) keyspace = connection_settings.pop( 'keyspace', self.settings_dict['DEFAULT_KEYSPACE'] ) self.keyspace = keyspace self.session = connection.get_session() if not(self.session is None or self.session.is_shutdown): return CassandraCursor(self.session) connection.setup( contact_points, keyspace, **connection_settings ) self.cluster = connection.get_cluster() self.session = connection.get_session() return CassandraCursor(self.session)
def get_table_settings(model): # returns the table as provided by the native driver for a given model cluster = get_cluster() ks = model._get_keyspace() table = model._raw_column_family_name() table = cluster.metadata.keyspaces[ks].tables[table] return table
def _get_partition_keys(self): try: table_meta = get_cluster().metadata.keyspaces[self.keyspace].tables[self.name] self.__partition_keys = OrderedDict((pk.name, Column(primary_key=True, partition_key=True, db_field=pk.name)) for pk in table_meta.partition_key) except Exception as e: raise CQLEngineException("Failed inspecting partition keys for {0}." "Ensure cqlengine is connected before attempting this with NamedTable.".format(self.column_family_name()))
def __create_keyspace(name, durable_writes, strategy_class, strategy_options, connection=None): cluster = get_cluster(connection) if name not in cluster.metadata.keyspaces: log.info(format_log_context("Creating keyspace %s", connection=connection), name) ks_meta = metadata.KeyspaceMetadata(name, durable_writes, strategy_class, strategy_options) execute(ks_meta.as_cql_query(), connection=connection) else: log.info(format_log_context("Not creating keyspace %s because it already exists", connection=connection), name)
def _sync_type(ks_name, type_model, omit_subtypes=None, connection=None): syncd_sub_types = omit_subtypes or set() for field in type_model._fields.values(): udts = [] columns.resolve_udts(field, udts) for udt in [u for u in udts if u not in syncd_sub_types]: _sync_type(ks_name, udt, syncd_sub_types, connection=connection) syncd_sub_types.add(udt) type_name = type_model.type_name() type_name_qualified = "%s.%s" % (ks_name, type_name) cluster = get_cluster(connection) keyspace = cluster.metadata.keyspaces[ks_name] defined_types = keyspace.user_types if type_name not in defined_types: log.debug(format_log_context("sync_type creating new type %s", keyspace=ks_name, connection=connection), type_name_qualified) cql = get_create_type(type_model, ks_name) execute(cql, connection=connection) cluster.refresh_user_type_metadata(ks_name, type_name) type_model.register_for_keyspace(ks_name, connection=connection) else: type_meta = defined_types[type_name] defined_fields = type_meta.field_names model_fields = set() for field in type_model._fields.values(): model_fields.add(field.db_field_name) if field.db_field_name not in defined_fields: execute("ALTER TYPE {0} ADD {1}".format(type_name_qualified, field.get_column_def()), connection=connection) else: field_type = type_meta.field_types[defined_fields.index(field.db_field_name)] if field_type != field.db_type: msg = format_log_context( 'Existing user type {0} has field "{1}" with a type ({2}) differing from the model user type ({3}).' ' UserType should be updated.', keyspace=ks_name, connection=connection) msg = msg.format(type_name_qualified, field.db_field_name, field_type, field.db_type) warnings.warn(msg) log.warning(msg) type_model.register_for_keyspace(ks_name, connection=connection) if len(defined_fields) == len(model_fields): log.info( format_log_context("Type %s did not require synchronization", keyspace=ks_name, connection=connection), type_name_qualified) return db_fields_not_in_model = model_fields.symmetric_difference(defined_fields) if db_fields_not_in_model: msg = format_log_context("Type %s has fields not referenced by model: %s", keyspace=ks_name, connection=connection) log.info(msg, type_name_qualified, db_fields_not_in_model)
def create_keyspace(name, strategy_class, replication_factor, durable_writes=True, **replication_values): """ *Deprecated - use :func:`create_keyspace_simple` or :func:`create_keyspace_network_topology` instead* Creates a keyspace If the keyspace already exists, it will not be modified. **This function should be used with caution, especially in production environments. Take care to execute schema modifications in a single context (i.e. not concurrently with other clients).** *There are plans to guard schema-modifying functions with an environment-driven conditional.* :param str name: name of keyspace to create :param str strategy_class: keyspace replication strategy class (:attr:`~.SimpleStrategy` or :attr:`~.NetworkTopologyStrategy` :param int replication_factor: keyspace replication factor, used with :attr:`~.SimpleStrategy` :param bool durable_writes: Write log is bypassed if set to False :param \*\*replication_values: Additional values to ad to the replication options map """ if not _allow_schema_modification(): return msg = "Deprecated. Use create_keyspace_simple or create_keyspace_network_topology instead" warnings.warn(msg, DeprecationWarning) log.warning(msg) cluster = get_cluster() if name not in cluster.metadata.keyspaces: # try the 1.2 method replication_map = { 'class': strategy_class, 'replication_factor': replication_factor } replication_map.update(replication_values) if strategy_class.lower() != 'simplestrategy': # Although the Cassandra documentation states for `replication_factor` # that it is "Required if class is SimpleStrategy; otherwise, # not used." we get an error if it is present. replication_map.pop('replication_factor', None) query = """ CREATE KEYSPACE {} WITH REPLICATION = {} """.format(name, json.dumps(replication_map).replace('"', "'")) if strategy_class != 'SimpleStrategy': query += " AND DURABLE_WRITES = {}".format( 'true' if durable_writes else 'false') execute(query)
def _get_partition_keys(self): try: table_meta = get_cluster(self._get_connection()).metadata.keyspaces[self.keyspace].tables[self.name] self.__partition_keys = OrderedDict( (pk.name, Column(primary_key=True, partition_key=True, db_field=pk.name)) for pk in table_meta.partition_key) except Exception as e: raise CQLEngineException("Failed inspecting partition keys for {0}." "Ensure cqlengine is connected before attempting this with NamedTable.".format( self.column_family_name()))
def _execute_statement(model, statement, consistency_level, timeout): params = statement.get_context() s = SimpleStatement(str(statement), consistency_level=consistency_level, fetch_size=statement.fetch_size) if model._partition_key_index: key_values = statement.partition_key_values(model._partition_key_index) if not any(v is None for v in key_values): parts = model._routing_key_from_values(key_values, connection.get_cluster().protocol_version) s.routing_key = parts s.keyspace = model._get_keyspace() return connection.execute(s, params, timeout=timeout)
def __init__(self, seeds, keyspace): self.seeds = seeds self.keyspace = keyspace # TODO for metrics # setup_cass(self.seeds, self.keyspace, # consistency=ConsistencyLevel.TWO, lazy_connect=False, # retry_connect=True, metrics_enabled=True) setup_cass(self.seeds, self.keyspace, consistency=ConsistencyLevel.TWO, lazy_connect=False, retry_connect=True) self.session = get_session() set_session(self.session) self.cluster = get_cluster()
def __init__(self, config): self.config = config self.__cass_url = config.get("cassandra", "host") self.__cass_username = config.get("cassandra", "username") self.__cass_password = config.get("cassandra", "password") self.__cass_keyspace = config.get("cassandra", "keyspace") self.__cass_local_DC = config.get("cassandra", "local_datacenter") self.__cass_protocol_version = config.getint("cassandra", "protocol_version") self.__cass_dc_policy = config.get("cassandra", "dc_policy") try: self.__cass_port = config.getint("cassandra", "port") except NoOptionError: self.__cass_port = 9042 with INIT_LOCK: try: connection.get_cluster() except CQLEngineException: self.__open()
def setUpClass(cls): cls.original_cluster = connection.get_cluster() cls.keyspace1 = 'ctest1' cls.keyspace2 = 'ctest2' super(ConnectionTest, cls).setUpClass() cls.setup_cluster = Cluster(protocol_version=PROTOCOL_VERSION) cls.setup_session = cls.setup_cluster.connect() ddl = "CREATE KEYSPACE {0} WITH replication = {{'class': 'SimpleStrategy', 'replication_factor': '{1}'}}".format(cls.keyspace1, 1) execute_with_long_wait_retry(cls.setup_session, ddl) ddl = "CREATE KEYSPACE {0} WITH replication = {{'class': 'SimpleStrategy', 'replication_factor': '{1}'}}".format(cls.keyspace2, 1) execute_with_long_wait_retry(cls.setup_session, ddl)
def _create_keyspace(name, durable_writes, strategy_class, strategy_options): if not _allow_schema_modification(): return cluster = get_cluster() if name not in cluster.metadata.keyspaces: log.info("Creating keyspace %s ", name) ks_meta = metadata.KeyspaceMetadata(name, durable_writes, strategy_class, strategy_options) execute(ks_meta.as_cql_query()) else: log.info("Not creating keyspace %s because it already exists", name)
def __init__(self, seeds, keyspace): self.seeds = seeds self.keyspace = keyspace # TODO configure ConsitencyLevel setup_cass(self.seeds, self.keyspace, consistency=ConsistencyLevel.TWO, lazy_connect=False, retry_connect=True, metrics_enabled=True) #setup_cass(self.seeds, self.keyspace, consistency=ConsistencyLevel.ONE, lazy_connect=False, retry_connect=True) self.session = get_session() set_session(self.session) self.cluster = get_cluster() self.logger = logging.getLogger('pet_race_job')
def setup(self): with self.lock: self.session = connection.get_session() if not (self.session is None or self.session.is_shutdown): # already connected return for option, value in self.session_options.items(): setattr(Session, option, value) connection.setup(self.hosts, self.keyspace, **self.connection_options) self.session = connection.get_session() self.cluster = connection.get_cluster()
def setUpClass(cls): cls.original_cluster = connection.get_cluster() cls.keyspace1 = 'ctest1' cls.keyspace2 = 'ctest2' super(ConnectionTest, cls).setUpClass() cls.setup_cluster = Cluster(protocol_version=PROTOCOL_VERSION) cls.setup_session = cls.setup_cluster.connect() ddl = "CREATE KEYSPACE {0} WITH replication = {{'class': 'SimpleStrategy', 'replication_factor': '{1}'}}".format( cls.keyspace1, 1) execute_with_long_wait_retry(cls.setup_session, ddl) ddl = "CREATE KEYSPACE {0} WITH replication = {{'class': 'SimpleStrategy', 'replication_factor': '{1}'}}".format( cls.keyspace2, 1) execute_with_long_wait_retry(cls.setup_session, ddl)
def _sync_type(ks_name, type_model, omit_subtypes=None, connection=None): syncd_sub_types = omit_subtypes or set() for field in type_model._fields.values(): udts = [] columns.resolve_udts(field, udts) for udt in [u for u in udts if u not in syncd_sub_types]: _sync_type(ks_name, udt, syncd_sub_types, connection=connection) syncd_sub_types.add(udt) type_name = type_model.type_name() type_name_qualified = "%s.%s" % (ks_name, type_name) cluster = get_cluster(connection) keyspace = cluster.metadata.keyspaces[ks_name] defined_types = keyspace.user_types if type_name not in defined_types: log.debug(format_log_context("sync_type creating new type %s", keyspace=ks_name, connection=connection), type_name_qualified) cql = get_create_type(type_model, ks_name) execute(cql, connection=connection) cluster.refresh_user_type_metadata(ks_name, type_name) type_model.register_for_keyspace(ks_name, connection=connection) else: type_meta = defined_types[type_name] defined_fields = type_meta.field_names model_fields = set() for field in type_model._fields.values(): model_fields.add(field.db_field_name) if field.db_field_name not in defined_fields: execute("ALTER TYPE {0} ADD {1}".format(type_name_qualified, field.get_column_def()), connection=connection) else: field_type = type_meta.field_types[defined_fields.index(field.db_field_name)] if field_type != field.db_type: msg = format_log_context('Existing user type {0} has field "{1}" with a type ({2}) differing from the model user type ({3}).' ' UserType should be updated.', keyspace=ks_name, connection=connection) msg = msg.format(type_name_qualified, field.db_field_name, field_type, field.db_type) warnings.warn(msg) log.warning(msg) type_model.register_for_keyspace(ks_name, connection=connection) if len(defined_fields) == len(model_fields): log.info(format_log_context("Type %s did not require synchronization", keyspace=ks_name, connection=connection), type_name_qualified) return db_fields_not_in_model = model_fields.symmetric_difference(defined_fields) if db_fields_not_in_model: msg = format_log_context("Type %s has fields not referenced by model: %s", keyspace=ks_name, connection=connection) log.info(msg, type_name_qualified, db_fields_not_in_model)
def test_db_field_override(self): """ Tests for db_field override Tests to ensure that udt's in models can specify db_field for a particular field and that it will be honored. @since 3.1.0 @jira_ticket PYTHON-346 @expected_result The actual cassandra column will use the db_field specified. @test_category data_types:udt """ class db_field_different(UserType): age = columns.Integer(db_field='a') name = columns.Text(db_field='n') class TheModel(Model): id = columns.Integer(primary_key=True) info = columns.UserDefinedType(db_field_different) sync_table(TheModel) cluster = connection.get_cluster() type_meta = cluster.metadata.keyspaces[TheModel._get_keyspace( )].user_types[db_field_different.type_name()] type_fields = (db_field_different.age.column, db_field_different.name.column) self.assertEqual(len(type_meta.field_names), len(type_fields)) for f in type_fields: self.assertIn(f.db_field_name, type_meta.field_names) id = 0 age = 42 name = 'John' info = db_field_different(age=age, name=name) TheModel.create(id=id, info=info) self.assertEqual(1, TheModel.objects.count()) john = TheModel.objects().first() self.assertEqual(john.id, id) info = john.info self.assertIsInstance(info, db_field_different) self.assertEqual(info.age, age) self.assertEqual(info.name, name) # also excercise the db_Field mapping self.assertEqual(info.a, age) self.assertEqual(info.n, name)
def create_keyspace(name, strategy_class, replication_factor, durable_writes=True, **replication_values): """ *Deprecated - use :func:`create_keyspace_simple` or :func:`create_keyspace_network_topology` instead* Creates a keyspace If the keyspace already exists, it will not be modified. **This function should be used with caution, especially in production environments. Take care to execute schema modifications in a single context (i.e. not concurrently with other clients).** *There are plans to guard schema-modifying functions with an environment-driven conditional.* :param str name: name of keyspace to create :param str strategy_class: keyspace replication strategy class (:attr:`~.SimpleStrategy` or :attr:`~.NetworkTopologyStrategy` :param int replication_factor: keyspace replication factor, used with :attr:`~.SimpleStrategy` :param bool durable_writes: Write log is bypassed if set to False :param \*\*replication_values: Additional values to ad to the replication options map """ if not _allow_schema_modification(): return msg = "Deprecated. Use create_keyspace_simple or create_keyspace_network_topology instead" warnings.warn(msg, DeprecationWarning) log.warning(msg) cluster = get_cluster() if name not in cluster.metadata.keyspaces: # try the 1.2 method replication_map = { 'class': strategy_class, 'replication_factor': replication_factor } replication_map.update(replication_values) if strategy_class.lower() != 'simplestrategy': # Although the Cassandra documentation states for `replication_factor` # that it is "Required if class is SimpleStrategy; otherwise, # not used." we get an error if it is present. replication_map.pop('replication_factor', None) query = """ CREATE KEYSPACE {} WITH REPLICATION = {} """.format(metadata.protect_name(name), json.dumps(replication_map).replace('"', "'")) if strategy_class != 'SimpleStrategy': query += " AND DURABLE_WRITES = {}".format('true' if durable_writes else 'false') execute(query)
def test_db_field_override(self): """ Tests for db_field override Tests to ensure that udt's in models can specify db_field for a particular field and that it will be honored. @since 3.1.0 @jira_ticket PYTHON-346 @expected_result The actual cassandra column will use the db_field specified. @test_category data_types:udt """ class db_field_different(UserType): age = columns.Integer(db_field='a') name = columns.Text(db_field='n') class TheModel(Model): id = columns.Integer(primary_key=True) info = columns.UserDefinedType(db_field_different) sync_table(TheModel) self.addCleanup(drop_table, TheModel) cluster = connection.get_cluster() type_meta = cluster.metadata.keyspaces[TheModel._get_keyspace()].user_types[db_field_different.type_name()] type_fields = (db_field_different.age.column, db_field_different.name.column) self.assertEqual(len(type_meta.field_names), len(type_fields)) for f in type_fields: self.assertIn(f.db_field_name, type_meta.field_names) id = 0 age = 42 name = 'John' info = db_field_different(age=age, name=name) TheModel.create(id=id, info=info) self.assertEqual(1, TheModel.objects.count()) john = TheModel.objects.first() self.assertEqual(john.id, id) info = john.info self.assertIsInstance(info, db_field_different) self.assertEqual(info.age, age) self.assertEqual(info.name, name) # also excercise the db_Field mapping self.assertEqual(info.a, age) self.assertEqual(info.n, name)
def _drop_table(model): if not _allow_schema_modification(): return # don't try to delete non existant tables meta = get_cluster().metadata ks_name = model._get_keyspace() raw_cf_name = model._raw_column_family_name() try: meta.keyspaces[ks_name].tables[raw_cf_name] execute('DROP TABLE {0};'.format(model.column_family_name())) except KeyError: pass
def drop_keyspace(name): """ Drops a keyspace, if it exists. *There are plans to guard schema-modifying functions with an environment-driven conditional.* **This function should be used with caution, especially in production environments. Take care to execute schema modifications in a single context (i.e. not concurrently with other clients).** :param str name: name of keyspace to drop """ if not _allow_schema_modification(): return cluster = get_cluster() if name in cluster.metadata.keyspaces: execute("DROP KEYSPACE {0}".format(metadata.protect_name(name)))
def drop_keyspace(name): """ Drops a keyspace, if it exists. *There are plans to guard schema-modifying functions with an environment-driven conditional.* **This function should be used with caution, especially in production environments. Take care to execute schema modifications in a single context (i.e. not concurrently with other clients).** :param str name: name of keyspace to drop """ if not _allow_schema_modification(): return cluster = get_cluster() if name in cluster.metadata.keyspaces: execute("DROP KEYSPACE {}".format(metadata.protect_name(name)))
def _sync_type(ks_name, type_model, omit_subtypes=None): syncd_sub_types = omit_subtypes or set() for field in type_model._fields.values(): udts = [] columns.resolve_udts(field, udts) for udt in [u for u in udts if u not in syncd_sub_types]: _sync_type(ks_name, udt, syncd_sub_types) syncd_sub_types.add(udt) type_name = type_model.type_name() type_name_qualified = "%s.%s" % (ks_name, type_name) cluster = get_cluster() keyspace = cluster.metadata.keyspaces[ks_name] defined_types = keyspace.user_types if type_name not in defined_types: log.debug("sync_type creating new type %s", type_name_qualified) cql = get_create_type(type_model, ks_name) execute(cql) cluster.refresh_user_type_metadata(ks_name, type_name) type_model.register_for_keyspace(ks_name) else: defined_fields = defined_types[type_name].field_names model_fields = set() for field in type_model._fields.values(): model_fields.add(field.db_field_name) if field.db_field_name not in defined_fields: execute("ALTER TYPE {0} ADD {1}".format( type_name_qualified, field.get_column_def())) type_model.register_for_keyspace(ks_name) if len(defined_fields) == len(model_fields): log.info("Type %s did not require synchronization", type_name_qualified) return db_fields_not_in_model = model_fields.symmetric_difference( defined_fields) if db_fields_not_in_model: log.info("Type %s has fields not referenced by model: %s", type_name_qualified, db_fields_not_in_model)
def test_create_drop_succeeeds(self): cluster = get_cluster() keyspace_ss = 'test_ks_ss' self.assertNotIn(keyspace_ss, cluster.metadata.keyspaces) management.create_keyspace_simple(keyspace_ss, 2) self.assertIn(keyspace_ss, cluster.metadata.keyspaces) management.drop_keyspace(keyspace_ss) self.assertNotIn(keyspace_ss, cluster.metadata.keyspaces) keyspace_nts = 'test_ks_nts' self.assertNotIn(keyspace_nts, cluster.metadata.keyspaces) management.create_keyspace_network_topology(keyspace_nts, {'dc1': 1}) self.assertIn(keyspace_nts, cluster.metadata.keyspaces) management.drop_keyspace(keyspace_nts) self.assertNotIn(keyspace_nts, cluster.metadata.keyspaces)
def _sync_type(ks_name, type_model, omit_subtypes=None): syncd_sub_types = omit_subtypes or set() for field in type_model._fields.values(): udts = [] columns.resolve_udts(field, udts) for udt in [u for u in udts if u not in syncd_sub_types]: _sync_type(ks_name, udt, syncd_sub_types) syncd_sub_types.add(udt) type_name = type_model.type_name() type_name_qualified = "%s.%s" % (ks_name, type_name) cluster = get_cluster() keyspace = cluster.metadata.keyspaces[ks_name] defined_types = keyspace.user_types if type_name not in defined_types: log.debug("sync_type creating new type %s", type_name_qualified) cql = get_create_type(type_model, ks_name) execute(cql) cluster.refresh_user_type_metadata(ks_name, type_name) type_model.register_for_keyspace(ks_name) else: defined_fields = defined_types[type_name].field_names model_fields = set() for field in type_model._fields.values(): model_fields.add(field.db_field_name) if field.db_field_name not in defined_fields: execute("ALTER TYPE {} ADD {}".format(type_name_qualified, field.get_column_def())) type_model.register_for_keyspace(ks_name) if len(defined_fields) == len(model_fields): log.info("Type %s did not require synchronization", type_name_qualified) return db_fields_not_in_model = model_fields.symmetric_difference(defined_fields) if db_fields_not_in_model: log.info("Type %s has fields not referenced by model: %s", type_name_qualified, db_fields_not_in_model)
def drop_table(model): """ Drops the table indicated by the model, if it exists. **This function should be used with caution, especially in production environments. Take care to execute schema modifications in a single context (i.e. not concurrently with other clients).** *There are plans to guard schema-modifying functions with an environment-driven conditional.* """ if not _allow_schema_modification(): return # don't try to delete non existant tables meta = get_cluster().metadata ks_name = model._get_keyspace() raw_cf_name = model._raw_column_family_name() try: meta.keyspaces[ks_name].tables[raw_cf_name] execute('DROP TABLE {0};'.format(model.column_family_name())) except KeyError: pass
def drop_table(model): """ Drops the table indicated by the model, if it exists. **This function should be used with caution, especially in production environments. Take care to execute schema modifications in a single context (i.e. not concurrently with other clients).** *There are plans to guard schema-modifying functions with an environment-driven conditional.* """ if not _allow_schema_modification(): return # don't try to delete non existant tables meta = get_cluster().metadata ks_name = model._get_keyspace() raw_cf_name = model._raw_column_family_name() try: meta.keyspaces[ks_name].tables[raw_cf_name] execute('DROP TABLE {};'.format(model.column_family_name())) except KeyError: pass
def test_db_field_override(self): class db_field_different(UserType): age = columns.Integer(db_field='a') name = columns.Text(db_field='n') class TheModel(Model): id = columns.Integer(primary_key=True) info = columns.UserDefinedType(db_field_different) sync_table(TheModel) cluster = connection.get_cluster() type_meta = cluster.metadata.keyspaces[TheModel._get_keyspace()].user_types[db_field_different.type_name()] type_fields = (db_field_different.age.column, db_field_different.name.column) self.assertEqual(len(type_meta.field_names), len(type_fields)) for f in type_fields: self.assertIn(f.db_field_name, type_meta.field_names) id = 0 age = 42 name = 'John' info = db_field_different(age=age, name=name) TheModel.create(id=id, info=info) self.assertEqual(1, TheModel.objects.count()) john = TheModel.objects().first() self.assertEqual(john.id, id) info = john.info self.assertIsInstance(info, db_field_different) self.assertEqual(info.age, age) self.assertEqual(info.name, name) # also excercise the db_Field mapping self.assertEqual(info.a, age) self.assertEqual(info.n, name)
def cluster(self): if self.connection: return get_cluster(self.connection_name) return
from cassandra.cqlengine import columns from cassandra.auth import PlainTextAuthProvider from cassandra.cluster import Cluster from cassandra.cqlengine import connection from cassandra.cqlengine.management import sync_table import config as settings import pandas as pd hosts = settings.DB_CONTACT_POINTS authProvider = PlainTextAuthProvider(username=settings.DB_USER, password=settings.DB_PASSWORD) # cluster = {'protocol_version': settings.CASSAN_PROTOCOL_VERSION, 'auth_provider': authProvider} clusterParamDict = {'auth_provider': authProvider} keyspace = settings.DB_KEYSPACE connection.setup(hosts, keyspace, **clusterParamDict) cluster = connection.get_cluster() session = connection.get_session() session.set_keyspace(settings.DB_KEYSPACE) # session.default_fetch_size = 50000 session.default_timeout = 120 class tbl_hello_world(Model): task_id = columns.BigInt(partition_key=True, primary_key=True) start_time = columns.Text() end_time = columns.Text() is_risk = columns.Boolean() predictable_characterization = columns.Text() predictable_identification = columns.Boolean() fault_scenario = columns.Text() root_cause = columns.Text()
def _drop_keyspace(name, connection=None): cluster = get_cluster(connection) if name in cluster.metadata.keyspaces: execute("DROP KEYSPACE {0}".format(metadata.protect_name(name)), connection=connection)
def sync_table(model): """ Inspects the model and creates / updates the corresponding table and columns. Any User Defined Types used in the table are implicitly synchronized. This function can only add fields that are not part of the primary key. Note that the attributes removed from the model are not deleted on the database. They become effectively ignored by (will not show up on) the model. **This function should be used with caution, especially in production environments. Take care to execute schema modifications in a single context (i.e. not concurrently with other clients).** *There are plans to guard schema-modifying functions with an environment-driven conditional.* """ if not _allow_schema_modification(): return if not issubclass(model, Model): raise CQLEngineException("Models must be derived from base Model.") if model.__abstract__: raise CQLEngineException("cannot create table from abstract model") cf_name = model.column_family_name() raw_cf_name = model._raw_column_family_name() ks_name = model._get_keyspace() cluster = get_cluster() try: keyspace = cluster.metadata.keyspaces[ks_name] except KeyError: raise CQLEngineException("Keyspace '{0}' for model {1} does not exist.".format(ks_name, model)) tables = keyspace.tables syncd_types = set() for col in model._columns.values(): udts = [] columns.resolve_udts(col, udts) for udt in [u for u in udts if u not in syncd_types]: _sync_type(ks_name, udt, syncd_types) if raw_cf_name not in tables: log.debug("sync_table creating new table %s", cf_name) qs = _get_create_table(model) try: execute(qs) except CQLEngineException as ex: # 1.2 doesn't return cf names, so we have to examine the exception # and ignore if it says the column family already exists if "Cannot add already existing column family" not in unicode(ex): raise else: log.debug("sync_table checking existing table %s", cf_name) table_meta = tables[raw_cf_name] _validate_pk(model, table_meta) table_columns = table_meta.columns model_fields = set() for model_name, col in model._columns.items(): db_name = col.db_field_name model_fields.add(db_name) if db_name in table_columns: col_meta = table_columns[db_name] if col_meta.cql_type != col.db_type: msg = 'Existing table {0} has column "{1}" with a type ({2}) differing from the model type ({3}).' \ ' Model should be updated.'.format(cf_name, db_name, col_meta.cql_type, col.db_type) warnings.warn(msg) log.warning(msg) continue if col.primary_key or col.primary_key: raise CQLEngineException("Cannot add primary key '{0}' (with db_field '{1}') to existing table {2}".format(model_name, db_name, cf_name)) query = "ALTER TABLE {0} add {1}".format(cf_name, col.get_column_def()) execute(query) db_fields_not_in_model = model_fields.symmetric_difference(table_columns) if db_fields_not_in_model: log.info("Table {0} has fields not referenced by model: {1}".format(cf_name, db_fields_not_in_model)) _update_options(model) table = cluster.metadata.keyspaces[ks_name].tables[raw_cf_name] indexes = [c for n, c in model._columns.items() if c.index] # TODO: support multiple indexes in C* 3.0+ for column in indexes: index_name = _get_index_name_by_column(table, column.db_field_name) if index_name: continue qs = ['CREATE INDEX'] qs += ['ON {0}'.format(cf_name)] qs += ['("{0}")'.format(column.db_field_name)] qs = ' '.join(qs) execute(qs)
def sync_table(model): """ Inspects the model and creates / updates the corresponding table and columns. Any User Defined Types used in the table are implicitly synchronized. This function can only add fields that are not part of the primary key. Note that the attributes removed from the model are not deleted on the database. They become effectively ignored by (will not show up on) the model. **This function should be used with caution, especially in production environments. Take care to execute schema modifications in a single context (i.e. not concurrently with other clients).** *There are plans to guard schema-modifying functions with an environment-driven conditional.* """ if not _allow_schema_modification(): return if not issubclass(model, Model): raise CQLEngineException("Models must be derived from base Model.") if model.__abstract__: raise CQLEngineException("cannot create table from abstract model") cf_name = model.column_family_name() raw_cf_name = model._raw_column_family_name() ks_name = model._get_keyspace() cluster = get_cluster() keyspace = cluster.metadata.keyspaces[ks_name] tables = keyspace.tables syncd_types = set() for col in model._columns.values(): udts = [] columns.resolve_udts(col, udts) for udt in [u for u in udts if u not in syncd_types]: _sync_type(ks_name, udt, syncd_types) # check for an existing column family if raw_cf_name not in tables: log.debug("sync_table creating new table %s", cf_name) qs = get_create_table(model) try: execute(qs) except CQLEngineException as ex: # 1.2 doesn't return cf names, so we have to examine the exception # and ignore if it says the column family already exists if "Cannot add already existing column family" not in unicode(ex): raise else: log.debug("sync_table checking existing table %s", cf_name) # see if we're missing any columns fields = get_fields(model) field_names = [x.name for x in fields] model_fields = set() # # TODO: does this work with db_name?? for name, col in model._columns.items(): if col.primary_key or col.partition_key: continue # we can't mess with the PK model_fields.add(name) if col.db_field_name in field_names: continue # skip columns already defined # add missing column using the column def query = "ALTER TABLE {} add {}".format(cf_name, col.get_column_def()) execute(query) db_fields_not_in_model = model_fields.symmetric_difference(field_names) if db_fields_not_in_model: log.info("Table %s has fields not referenced by model: %s", cf_name, db_fields_not_in_model) update_compaction(model) table = cluster.metadata.keyspaces[ks_name].tables[raw_cf_name] indexes = [c for n, c in model._columns.items() if c.index] for column in indexes: if table.columns[column.db_field_name].index: continue qs = ['CREATE INDEX index_{}_{}'.format(raw_cf_name, column.db_field_name)] qs += ['ON {}'.format(cf_name)] qs += ['("{}")'.format(column.db_field_name)] qs = ' '.join(qs) execute(qs)
def sync_table(model): """ Inspects the model and creates / updates the corresponding table and columns. Any User Defined Types used in the table are implicitly synchronized. This function can only add fields that are not part of the primary key. Note that the attributes removed from the model are not deleted on the database. They become effectively ignored by (will not show up on) the model. **This function should be used with caution, especially in production environments. Take care to execute schema modifications in a single context (i.e. not concurrently with other clients).** *There are plans to guard schema-modifying functions with an environment-driven conditional.* """ if not _allow_schema_modification(): return if not issubclass(model, Model): raise CQLEngineException("Models must be derived from base Model.") if model.__abstract__: raise CQLEngineException("cannot create table from abstract model") cf_name = model.column_family_name() raw_cf_name = model._raw_column_family_name() ks_name = model._get_keyspace() cluster = get_cluster() try: keyspace = cluster.metadata.keyspaces[ks_name] except KeyError: raise CQLEngineException( "Keyspace '{0}' for model {1} does not exist.".format( ks_name, model)) tables = keyspace.tables syncd_types = set() for col in model._columns.values(): udts = [] columns.resolve_udts(col, udts) for udt in [u for u in udts if u not in syncd_types]: _sync_type(ks_name, udt, syncd_types) if raw_cf_name not in tables: log.debug("sync_table creating new table %s", cf_name) qs = _get_create_table(model) try: execute(qs) except CQLEngineException as ex: # 1.2 doesn't return cf names, so we have to examine the exception # and ignore if it says the column family already exists if "Cannot add already existing column family" not in unicode(ex): raise else: log.debug("sync_table checking existing table %s", cf_name) table_meta = tables[raw_cf_name] _validate_pk(model, table_meta) table_columns = table_meta.columns model_fields = set() for model_name, col in model._columns.items(): db_name = col.db_field_name model_fields.add(db_name) if db_name in table_columns: col_meta = table_columns[db_name] if col_meta.cql_type != col.db_type: msg = 'Existing table {0} has column "{1}" with a type ({2}) differing from the model type ({3}).' \ ' Model should be updated.'.format(cf_name, db_name, col_meta.cql_type, col.db_type) warnings.warn(msg) log.warning(msg) continue if col.primary_key or col.primary_key: raise CQLEngineException( "Cannot add primary key '{0}' (with db_field '{1}') to existing table {2}" .format(model_name, db_name, cf_name)) query = "ALTER TABLE {0} add {1}".format(cf_name, col.get_column_def()) execute(query) db_fields_not_in_model = model_fields.symmetric_difference( table_columns) if db_fields_not_in_model: log.info( "Table {0} has fields not referenced by model: {1}".format( cf_name, db_fields_not_in_model)) _update_options(model) table = cluster.metadata.keyspaces[ks_name].tables[raw_cf_name] indexes = [c for n, c in model._columns.items() if c.index] # TODO: support multiple indexes in C* 3.0+ for column in indexes: index_name = _get_index_name_by_column(table, column.db_field_name) if index_name: continue qs = ['CREATE INDEX'] qs += ['ON {0}'.format(cf_name)] qs += ['("{0}")'.format(column.db_field_name)] qs = ' '.join(qs) execute(qs)
def _sync_table(model): if not _allow_schema_modification(): return if not issubclass(model, Model): raise CQLEngineException("Models must be derived from base Model.") if model.__abstract__: raise CQLEngineException("cannot create table from abstract model") cf_name = model.column_family_name() raw_cf_name = model._raw_column_family_name() ks_name = model._get_keyspace() cluster = get_cluster() try: keyspace = cluster.metadata.keyspaces[ks_name] except KeyError: raise CQLEngineException("Keyspace '{0}' for model {1} does not exist.".format(ks_name, model)) tables = keyspace.tables syncd_types = set() for col in model._columns.values(): udts = [] columns.resolve_udts(col, udts) for udt in [u for u in udts if u not in syncd_types]: _sync_type(ks_name, udt, syncd_types) if raw_cf_name not in tables: log.debug("sync_table creating new table %s", cf_name) qs = _get_create_table(model) try: execute(qs) except CQLEngineException as ex: # 1.2 doesn't return cf names, so we have to examine the exception # and ignore if it says the column family already exists if "Cannot add already existing column family" not in unicode(ex): raise else: log.debug("sync_table checking existing table %s", cf_name) table_meta = tables[raw_cf_name] _validate_pk(model, table_meta) table_columns = table_meta.columns model_fields = set() for model_name, col in model._columns.items(): db_name = col.db_field_name model_fields.add(db_name) if db_name in table_columns: col_meta = table_columns[db_name] if col_meta.cql_type != col.db_type: msg = 'Existing table {0} has column "{1}" with a type ({2}) differing from the model type ({3}).' \ ' Model should be updated.'.format(cf_name, db_name, col_meta.cql_type, col.db_type) warnings.warn(msg) log.warning(msg) continue if col.primary_key or col.primary_key: raise CQLEngineException("Cannot add primary key '{0}' (with db_field '{1}') to existing table {2}".format(model_name, db_name, cf_name)) query = "ALTER TABLE {0} add {1}".format(cf_name, col.get_column_def()) execute(query) db_fields_not_in_model = model_fields.symmetric_difference(table_columns) if db_fields_not_in_model: log.info("Table {0} has fields not referenced by model: {1}".format(cf_name, db_fields_not_in_model)) _update_options(model) table = cluster.metadata.keyspaces[ks_name].tables[raw_cf_name] indexes = [c for n, c in model._columns.items() if c.index] # TODO: support multiple indexes in C* 3.0+ for column in indexes: index_name = _get_index_name_by_column(table, column.db_field_name) if index_name: continue qs = ['CREATE INDEX'] qs += ['ON {0}'.format(cf_name)] qs += ['("{0}")'.format(column.db_field_name)] qs = ' '.join(qs) execute(qs)
def test_get_cluster_fails_without_existing_connection(self): """ Users can't get the default cluster without having a default connection set. """ with self.assertRaisesRegexp(connection.CQLEngineException, self.no_registered_connection_msg): connection.get_cluster(connection=None)
def cluster(self): return connection.get_cluster()
def _sync_table(model, connection=None): if not _allow_schema_modification(): return if not issubclass(model, Model): raise CQLEngineException("Models must be derived from base Model.") if model.__abstract__: raise CQLEngineException("cannot create table from abstract model") cf_name = model.column_family_name() raw_cf_name = model._raw_column_family_name() ks_name = model._get_keyspace() connection = connection or model._get_connection() cluster = get_cluster(connection) try: keyspace = cluster.metadata.keyspaces[ks_name] except KeyError: msg = format_log_context("Keyspace '{0}' for model {1} does not exist.", connection=connection) raise CQLEngineException(msg.format(ks_name, model)) tables = keyspace.tables syncd_types = set() for col in model._columns.values(): udts = [] columns.resolve_udts(col, udts) for udt in [u for u in udts if u not in syncd_types]: _sync_type(ks_name, udt, syncd_types, connection=connection) if raw_cf_name not in tables: log.debug(format_log_context("sync_table creating new table %s", keyspace=ks_name, connection=connection), cf_name) qs = _get_create_table(model) try: execute(qs, connection=connection) except CQLEngineException as ex: # 1.2 doesn't return cf names, so we have to examine the exception # and ignore if it says the column family already exists if "Cannot add already existing column family" not in unicode(ex): raise else: log.debug(format_log_context("sync_table checking existing table %s", keyspace=ks_name, connection=connection), cf_name) table_meta = tables[raw_cf_name] _validate_pk(model, table_meta) table_columns = table_meta.columns model_fields = set() for model_name, col in model._columns.items(): db_name = col.db_field_name model_fields.add(db_name) if db_name in table_columns: col_meta = table_columns[db_name] if col_meta.cql_type != col.db_type: msg = format_log_context('Existing table {0} has column "{1}" with a type ({2}) differing from the model type ({3}).' ' Model should be updated.', keyspace=ks_name, connection=connection) msg = msg.format(cf_name, db_name, col_meta.cql_type, col.db_type) warnings.warn(msg) log.warning(msg) continue if col.primary_key or col.primary_key: msg = format_log_context("Cannot add primary key '{0}' (with db_field '{1}') to existing table {2}", keyspace=ks_name, connection=connection) raise CQLEngineException(msg.format(model_name, db_name, cf_name)) query = "ALTER TABLE {0} add {1}".format(cf_name, col.get_column_def()) execute(query, connection=connection) db_fields_not_in_model = model_fields.symmetric_difference(table_columns) if db_fields_not_in_model: msg = format_log_context("Table {0} has fields not referenced by model: {1}", keyspace=ks_name, connection=connection) log.info(msg.format(cf_name, db_fields_not_in_model)) _update_options(model, connection=connection) table = cluster.metadata.keyspaces[ks_name].tables[raw_cf_name] indexes = [c for n, c in model._columns.items() if c.index] # TODO: support multiple indexes in C* 3.0+ for column in indexes: index_name = _get_index_name_by_column(table, column.db_field_name) if index_name: continue qs = ['CREATE INDEX'] qs += ['ON {0}'.format(cf_name)] qs += ['("{0}")'.format(column.db_field_name)] qs = ' '.join(qs) execute(qs, connection=connection)
def sync_table(model): """ Inspects the model and creates / updates the corresponding table and columns. Any User Defined Types used in the table are implicitly synchronized. This function can only add fields that are not part of the primary key. Note that the attributes removed from the model are not deleted on the database. They become effectively ignored by (will not show up on) the model. **This function should be used with caution, especially in production environments. Take care to execute schema modifications in a single context (i.e. not concurrently with other clients).** *There are plans to guard schema-modifying functions with an environment-driven conditional.* """ if not _allow_schema_modification(): return if not issubclass(model, Model): raise CQLEngineException("Models must be derived from base Model.") if model.__abstract__: raise CQLEngineException("cannot create table from abstract model") cf_name = model.column_family_name() raw_cf_name = model._raw_column_family_name() ks_name = model._get_keyspace() cluster = get_cluster() keyspace = cluster.metadata.keyspaces[ks_name] tables = keyspace.tables syncd_types = set() for col in model._columns.values(): udts = [] columns.resolve_udts(col, udts) for udt in [u for u in udts if u not in syncd_types]: _sync_type(ks_name, udt, syncd_types) # check for an existing column family if raw_cf_name not in tables: log.debug("sync_table creating new table %s", cf_name) qs = _get_create_table(model) try: execute(qs) except CQLEngineException as ex: # 1.2 doesn't return cf names, so we have to examine the exception # and ignore if it says the column family already exists if "Cannot add already existing column family" not in unicode(ex): raise else: log.debug("sync_table checking existing table %s", cf_name) # see if we're missing any columns field_names = _get_non_pk_field_names(tables[raw_cf_name]) model_fields = set() # # TODO: does this work with db_name?? for name, col in model._columns.items(): if col.primary_key or col.partition_key: continue # we can't mess with the PK model_fields.add(name) if col.db_field_name in field_names: continue # skip columns already defined # add missing column using the column def query = "ALTER TABLE {0} add {1}".format(cf_name, col.get_column_def()) execute(query) db_fields_not_in_model = model_fields.symmetric_difference(field_names) if db_fields_not_in_model: log.info("Table %s has fields not referenced by model: %s", cf_name, db_fields_not_in_model) _update_options(model) table = cluster.metadata.keyspaces[ks_name].tables[raw_cf_name] indexes = [c for n, c in model._columns.items() if c.index] # TODO: support multiple indexes in C* 3.0+ for column in indexes: index_name = _get_index_name_by_column(table, column.db_field_name) if index_name: continue qs = ['CREATE INDEX'] qs += ['ON {0}'.format(cf_name)] qs += ['("{0}")'.format(column.db_field_name)] qs = ' '.join(qs) execute(qs)