def test_can_insert_nested_udts(self): class Depth_0(UserType): age = columns.Integer() name = columns.Text() class Depth_1(UserType): value = columns.UserDefinedType(Depth_0) class Depth_2(UserType): value = columns.UserDefinedType(Depth_1) class Depth_3(UserType): value = columns.UserDefinedType(Depth_2) class DepthModel(Model): id = columns.Integer(primary_key=True) v_0 = columns.UserDefinedType(Depth_0) v_1 = columns.UserDefinedType(Depth_1) v_2 = columns.UserDefinedType(Depth_2) v_3 = columns.UserDefinedType(Depth_3) sync_table(DepthModel) udts = [Depth_0(age=42, name="John")] udts.append(Depth_1(value=udts[0])) udts.append(Depth_2(value=udts[1])) udts.append(Depth_3(value=udts[2])) DepthModel.create(id=0, v_0=udts[0], v_1=udts[1], v_2=udts[2], v_3=udts[3]) output = DepthModel.objects().first() self.assertEqual(udts[0], output.v_0) self.assertEqual(udts[1], output.v_1) self.assertEqual(udts[2], output.v_2) self.assertEqual(udts[3], output.v_3)
def test_udt_validate_with_default(self): """ Test to verify restrictions are honored and that validate is called on the default value @since 3.10 @jira_ticket PYTHON-505 @expected_result a validation error is arisen due to the name being too long @test_category data_types:object_mapper """ class UserValidateDefault(UserType): age = columns.Integer() name = columns.Text(max_length=2, default="Robert") class UserModelValidateDefault(Model): id = columns.Integer(primary_key=True) info = columns.UserDefinedType(UserValidateDefault) sync_table(UserModelValidateDefault) self.addCleanup(drop_table, UserModelValidateDefault) user = UserValidateDefault(age=1) item = UserModelValidateDefault(id=1, info=user) with self.assertRaises(ValidationError): item.save()
def _reset_data(self): for ks in self.keyspaces: drop_keyspace(ks, connections=self.conns) for ks in self.keyspaces: create_keyspace_simple(ks, 1, connections=self.conns) sync_table(TestModel, keyspaces=self.keyspaces, connections=self.conns)
def sync_db(self): """Sync all defined tables. All defined models must be imported before this method is called """ models = get_subclasses(self.Model) for model in models: sync_table(model)
def test_can_insert_partial_udts(self): class User(UserType): age = columns.Integer() name = columns.Text() gender = columns.Text() class UserModel(Model): id = columns.Integer(primary_key=True) info = columns.UserDefinedType(User) sync_table(UserModel) user = User(age=42, name="John") UserModel.create(id=0, info=user) john_info = UserModel.objects().first().info self.assertEqual(42, john_info.age) self.assertEqual("John", john_info.name) self.assertIsNone(john_info.gender) user = User(age=42) UserModel.create(id=0, info=user) john_info = UserModel.objects().first().info self.assertEqual(42, john_info.age) self.assertIsNone(john_info.name) self.assertIsNone(john_info.gender)
def test_can_insert_udts_with_all_datatypes(self): class AllDatatypes(UserType): a = columns.Ascii() b = columns.BigInt() c = columns.Blob() d = columns.Boolean() e = columns.Date() f = columns.DateTime() g = columns.Decimal() h = columns.Float(double_precision=True) i = columns.Inet() j = columns.Integer() k = columns.Text() l = columns.TimeUUID() m = columns.UUID() n = columns.VarInt() class AllDatatypesModel(Model): id = columns.Integer(primary_key=True) data = columns.UserDefinedType(AllDatatypes) sync_table(AllDatatypesModel) input = AllDatatypes(a='ascii', b=2 ** 63 - 1, c=bytearray(b'hello world'), d=True, e=date(1970, 1, 1), f=datetime.utcfromtimestamp(872835240), g=Decimal('12.3E+7'), h=3.4028234663852886e+38, i='123.123.123.123', j=2147483647, k='text', l= UUID('FE2B4360-28C6-11E2-81C1-0800200C9A66'), m=UUID('067e6162-3b6f-4ae2-a171-2470b63dff00'), n=int(str(2147483647) + '000')) alldata = AllDatatypesModel.create(id=0, data=input) self.assertEqual(1, AllDatatypesModel.objects.count()) output = AllDatatypesModel.objects().first().data for i in range(ord('a'), ord('a') + 14): self.assertEqual(input[chr(i)], output[chr(i)])
def test_can_insert_udts_with_nones(self): """ Test for inserting all column types as empty into a UserType as None's test_can_insert_udts_with_nones tests that each cqlengine column type can be inserted into a UserType as None's. It first creates a UserType that has each cqlengine column type, and a corresponding table/Model. It then creates a UserType instance where all the fields are None's and inserts the UserType as an instance of the Model. Finally, it verifies that each column read from the UserType from Cassandra is None. @since 2.5.0 @jira_ticket PYTHON-251 @expected_result The UserType is inserted with each column type, and the resulting read yields None's for each column. @test_category data_types:udt """ sync_table(AllDatatypesModel) self.addCleanup(drop_table, AllDatatypesModel) input = AllDatatypes(a=None, b=None, c=None, d=None, e=None, f=None, g=None, h=None, i=None, j=None, k=None, l=None, m=None, n=None) AllDatatypesModel.create(id=0, data=input) self.assertEqual(1, AllDatatypesModel.objects.count()) output = AllDatatypesModel.objects.first().data self.assertEqual(input, output)
def test_model_over_write(self): """ Test to ensure overwriting of primary keys in model inheritance is allowed This is currently only an issue in PyPy. When PYTHON-504 is introduced this should be updated error out and warn the user @since 3.6.0 @jira_ticket PYTHON-576 @expected_result primary keys can be overwritten via inheritance @test_category object_mapper """ class TimeModelBase(Model): uuid = columns.TimeUUID(primary_key=True) class DerivedTimeModel(TimeModelBase): __table_name__ = 'derived_time' uuid = columns.TimeUUID(primary_key=True, partition_key=True) value = columns.Text(required=False) # In case the table already exists in keyspace drop_table(DerivedTimeModel) sync_table(DerivedTimeModel) uuid_value = uuid1() uuid_value2 = uuid1() DerivedTimeModel.create(uuid=uuid_value, value="first") DerivedTimeModel.create(uuid=uuid_value2, value="second") DerivedTimeModel.objects.filter(uuid=uuid_value)
def test_can_insert_udts_with_nones(self): """ Test for inserting all column types as empty into a UserType as None's test_can_insert_udts_with_nones tests that each cqlengine column type can be inserted into a UserType as None's. It first creates a UserType that has each cqlengine column type, and a corresponding table/Model. It then creates a UserType instance where all the fields are None's and inserts the UserType as an instance of the Model. Finally, it verifies that each column read from the UserType from Cassandra is None. @since 2.5.0 @jira_ticket PYTHON-251 @expected_result The UserType is inserted with each column type, and the resulting read yields None's for each column. @test_category data_types:udt """ class AllDatatypes(UserType): a = columns.Ascii() b = columns.BigInt() c = columns.Blob() d = columns.Boolean() e = columns.DateTime() f = columns.Decimal() g = columns.Double() h = columns.Float() i = columns.Inet() j = columns.Integer() k = columns.Text() l = columns.TimeUUID() m = columns.UUID() n = columns.VarInt() class AllDatatypesModel(Model): id = columns.Integer(primary_key=True) data = columns.UserDefinedType(AllDatatypes) sync_table(AllDatatypesModel) input = AllDatatypes( a=None, b=None, c=None, d=None, e=None, f=None, g=None, h=None, i=None, j=None, k=None, l=None, m=None, n=None, ) AllDatatypesModel.create(id=0, data=input) self.assertEqual(1, AllDatatypesModel.objects.count()) output = AllDatatypesModel.objects().first().data self.assertEqual(input, output)
def test_defaultFetchSize(self): # Populate Table People.objects.create(last_name="Smith", first_name="John", birthday=datetime.now()) People.objects.create(last_name="Bestwater", first_name="Alan", birthday=datetime.now()) People.objects.create(last_name="Smith", first_name="Greg", birthday=datetime.now()) People.objects.create(last_name="Smith", first_name="Adam", birthday=datetime.now()) # Check query constructions expected_fields = ['first_name', 'birthday'] self.assertEqual(People.filter(last_name="Smith")._select_fields(), expected_fields) # Validate correct fields are fetched smiths = list(People.filter(last_name="Smith")) self.assertEqual(len(smiths), 3) self.assertTrue(smiths[0].last_name is not None) # Modify table with new value sync_table(People2) # populate new format People2.objects.create(last_name="Smith", first_name="Chris", middle_name="Raymond", birthday=datetime.now()) People2.objects.create(last_name="Smith", first_name="Andrew", middle_name="Micheal", birthday=datetime.now()) # validate query construction expected_fields = ['first_name', 'middle_name', 'birthday'] self.assertEqual(People2.filter(last_name="Smith")._select_fields(), expected_fields) # validate correct items are returneds smiths = list(People2.filter(last_name="Smith")) self.assertEqual(len(smiths), 5) self.assertTrue(smiths[0].last_name is not None)
def setUpClass(cls): if PROTOCOL_VERSION < 4 or CASSANDRA_VERSION < "3.0": return cls.db_klass, cls.python_klass = UserDefinedType, User cls.first_value = User( age=1, date_param=datetime.utcnow(), map_param={1: time(2, 12, 7, 50), 2: util.Time(time(2, 12, 7, 49))}, list_param=[datetime(1, 1, 2), datetime(1, 1, 3)], set_param=set((datetime(1, 1, 3), util.Date(datetime(1, 1, 1)))), tuple_param=(datetime(1, 1, 3), 2, False, 1, 2.324, uuid4()) ) cls.second_value = User( age=1, date_param=datetime.utcnow(), map_param={1: time(2, 12, 7, 50), 2: util.Time(time(2, 12, 7, 49))}, list_param=[datetime(1, 1, 2), datetime(1, 2, 3)], set_param=None, tuple_param=(datetime(1, 1, 2), 2, False, 1, 2.324, uuid4()) ) cls.third_value = User( age=2, date_param=None, map_param={1: time(2, 12, 7, 51), 2: util.Time(time(2, 12, 7, 49))}, list_param=[datetime(1, 1, 2), datetime(1, 1, 4)], set_param=set((datetime(1, 1, 3), util.Date(datetime(1, 1, 2)))), tuple_param=(None, 3, False, None, 2.3214, uuid4()) ) cls.model_class = UserModel sync_table(cls.model_class)
def setUpClass(cls): super(TestIndexedPolymorphicQuery, cls).setUpClass() management.sync_table(IndexedPoly1) management.sync_table(IndexedPoly2) cls.p1 = IndexedPoly1.create(data1='pickle') cls.p2 = IndexedPoly2.create(partition=cls.p1.partition, data2='bacon')
def sync_cassandra(): from cassandra.cqlengine.management import sync_table, create_keyspace from benchmark.feeds import UserFeed, TimelineFeed create_keyspace('stream_framework_bench', 'SimpleStrategy', 3) for feed_class in [UserFeed, TimelineFeed]: timeline = feed_class.get_timeline_storage() sync_table(timeline.model)
def sync(self, alias): engine = get_engine_from_db_alias(alias) if engine != 'django_cassandra_engine': raise CommandError('Database {} is not cassandra!'.format(alias)) connection = connections[alias] connection.connect() options = connection.settings_dict.get('OPTIONS', {}) keyspace = connection.settings_dict['NAME'] replication_opts = options.get('replication', {}) strategy_class = replication_opts.pop('strategy_class', 'SimpleStrategy') replication_factor = replication_opts.pop('replication_factor', 1) self.stdout.write('Creating keyspace {}..'.format(keyspace)) if strategy_class == 'SimpleStrategy': create_keyspace_simple(keyspace, replication_factor) else: create_keyspace_network_topology(keyspace, replication_opts) for app_name, app_models \ in connection.introspection.cql_models.items(): for model in app_models: self.stdout.write('Syncing %s.%s' % (app_name, model.__name__)) sync_table(model)
def test_default_values(self): """ Test that default types are set on object creation for UDTs @since 3.7.0 @jira_ticket PYTHON-606 @expected_result Default values should be set. @test_category data_types:udt """ class NestedUdt(UserType): test_id = columns.UUID(default=uuid4) something = columns.Text() default_text = columns.Text(default="default text") class OuterModel(Model): name = columns.Text(primary_key=True) first_name = columns.Text() nested = columns.List(columns.UserDefinedType(NestedUdt)) simple = columns.UserDefinedType(NestedUdt) sync_table(OuterModel) self.addCleanup(drop_table, OuterModel) t = OuterModel.create(name='test1') t.nested = [NestedUdt(something='test')] t.simple = NestedUdt(something="") t.save() self.assertIsNotNone(t.nested[0].test_id) self.assertEqual(t.nested[0].default_text, "default text") self.assertIsNotNone(t.simple.test_id) self.assertEqual(t.simple.default_text, "default text")
def test_table_definition(self): """ Tests that creating a table with capitalized column names succeeds """ sync_table(LowercaseKeyModel) sync_table(CapitalizedKeyModel) drop_table(LowercaseKeyModel) drop_table(CapitalizedKeyModel)
def test_udts_with_unicode(self): """ Test for inserting models with unicode and udt columns. test_udts_with_unicode constructs a model with a user defined type. It then attempts to insert that model with a unicode primary key. It will also attempt to upsert a udt that contains unicode text. @since 3.0.0 @jira_ticket PYTHON-353 @expected_result No exceptions thrown @test_category data_types:udt """ ascii_name = 'normal name' unicode_name = u'Fran\u00E7ois' class User(UserType): age = columns.Integer() name = columns.Text() class UserModelText(Model): id = columns.Text(primary_key=True) info = columns.UserDefinedType(User) sync_table(UserModelText) # Two udt instances one with a unicode one with ascii user_template_ascii = User(age=25, name=ascii_name) user_template_unicode = User(age=25, name=unicode_name) UserModelText.create(id=ascii_name, info=user_template_unicode) UserModelText.create(id=unicode_name, info=user_template_ascii) UserModelText.create(id=unicode_name, info=user_template_unicode)
def test_keywords_as_names(self): create_keyspace_simple('keyspace', 1) class table(Model): __keyspace__ = 'keyspace' select = columns.Integer(primary_key=True) table = columns.Text() # create should work drop_table(table) sync_table(table) created = table.create(select=0, table='table') selected = table.objects(select=0)[0] self.assertEqual(created.select, selected.select) self.assertEqual(created.table, selected.table) # alter should work class table(Model): __keyspace__ = 'keyspace' select = columns.Integer(primary_key=True) table = columns.Text() where = columns.Text() sync_table(table) created = table.create(select=1, table='table') selected = table.objects(select=1)[0] self.assertEqual(created.select, selected.select) self.assertEqual(created.table, selected.table) self.assertEqual(created.where, selected.where) drop_keyspace('keyspace')
def test_can_insert_double_and_float(self): """ Test for inserting single-precision and double-precision values into a Float and Double columns @since 2.6.0 @changed 3.0.0 removed deprecated Float(double_precision) parameter @jira_ticket PYTHON-246 @expected_result Each floating point column type is able to hold their respective precision values. @test_category data_types:primitive """ class FloatingPointModel(Model): id = columns.Integer(primary_key=True) f = columns.Float() d = columns.Double() sync_table(FloatingPointModel) FloatingPointModel.create(id=0, f=2.39) output = FloatingPointModel.objects().first() self.assertEqual(2.390000104904175, output.f) # float loses precision FloatingPointModel.create(id=0, f=3.4028234663852886e+38, d=2.39) output = FloatingPointModel.objects().first() self.assertEqual(3.4028234663852886e+38, output.f) self.assertEqual(2.39, output.d) # double retains precision FloatingPointModel.create(id=0, d=3.4028234663852886e+38) output = FloatingPointModel.objects().first() self.assertEqual(3.4028234663852886e+38, output.d)
def setUpClass(cls): if PROTOCOL_VERSION < 4: return super(TestQuerying, cls).setUpClass() drop_table(TestQueryModel) sync_table(TestQueryModel)
def test_set_table_properties(self): sync_table(ModelWithTableProperties) expected = {'bloom_filter_fp_chance': 0.76328, 'comment': 'TxfguvBdzwROQALmQBOziRMbkqVGFjqcJfVhwGR', 'gc_grace_seconds': 2063, 'read_repair_chance': 0.17985, # For some reason 'dclocal_read_repair_chance' in CQL is called # just 'local_read_repair_chance' in the schema table. # Source: https://issues.apache.org/jira/browse/CASSANDRA-6717 # TODO: due to a bug in the native driver i'm not seeing the local read repair chance show up # 'local_read_repair_chance': 0.50811, } if CASSANDRA_VERSION <= '2.0.0': expected['caching'] = CACHING_ALL expected['replicate_on_write'] = False if CASSANDRA_VERSION == '2.0.0': expected['populate_io_cache_on_flush'] = True expected['index_interval'] = 98706 if CASSANDRA_VERSION >= '2.0.0': expected['default_time_to_live'] = 4756 expected['memtable_flush_period_in_ms'] = 43681 options = management.get_table_settings(ModelWithTableProperties).options self.assertEqual(dict([(k, options.get(k)) for k in expected.keys()]), expected)
def test_can_insert_udts_with_nulls(self): class AllDatatypes(UserType): a = columns.Ascii() b = columns.BigInt() c = columns.Blob() d = columns.Boolean() e = columns.Date() f = columns.DateTime() g = columns.Decimal() h = columns.Float() i = columns.Inet() j = columns.Integer() k = columns.Text() l = columns.TimeUUID() m = columns.UUID() n = columns.VarInt() class AllDatatypesModel(Model): id = columns.Integer(primary_key=True) data = columns.UserDefinedType(AllDatatypes) sync_table(AllDatatypesModel) input = AllDatatypes(a=None, b=None, c=None, d=None, e=None, f=None, g=None, h=None, i=None, j=None, k=None, l=None, m=None, n=None) AllDatatypesModel.create(id=0, data=input) self.assertEqual(1, AllDatatypesModel.objects.count()) output = AllDatatypesModel.objects().first().data self.assertEqual(input, output)
def test_sync_table_works_with_primary_keys_only_tables(self): # This is "create table": sync_table(PrimaryKeysOnlyModel) # let's make sure settings persisted correctly: assert PrimaryKeysOnlyModel.__compaction__ == LeveledCompactionStrategy # blows up with DoesNotExist if table does not exist table_settings = management.get_table_settings(PrimaryKeysOnlyModel) # let make sure the flag we care about assert LeveledCompactionStrategy in table_settings.options['compaction_strategy_class'] # Now we are "updating" the table: # setting up something to change PrimaryKeysOnlyModel.__compaction__ = SizeTieredCompactionStrategy # primary-keys-only tables do not create entries in system.schema_columns # table. Only non-primary keys are added to that table. # Our code must deal with that eventuality properly (not crash) # on subsequent runs of sync_table (which runs get_fields internally) get_fields(PrimaryKeysOnlyModel) sync_table(PrimaryKeysOnlyModel) table_settings = management.get_table_settings(PrimaryKeysOnlyModel) assert SizeTieredCompactionStrategy in table_settings.options['compaction_strategy_class']
def setUpClass(cls): if PROTOCOL_VERSION < 4: raise unittest.SkipTest("Date query tests require native protocol 4+, currently using: {0}".format(PROTOCOL_VERSION)) super(TestQuerying, cls).setUpClass() drop_table(TestQueryModel) sync_table(TestQueryModel)
def test_can_insert_udts_with_all_datatypes(self): """ Test for inserting all column types into a UserType test_can_insert_udts_with_all_datatypes tests that each cqlengine column type can be inserted into a UserType. It first creates a UserType that has each cqlengine column type, and a corresponding table/Model. It then creates a UserType instance where all the fields have corresponding data, and inserts the UserType as an instance of the Model. Finally, it verifies that each column read from the UserType from Cassandra is the same as the input parameters. @since 2.5.0 @jira_ticket PYTHON-251 @expected_result The UserType is inserted with each column type, and the resulting read yields proper data for each column. @test_category data_types:udt """ sync_table(AllDatatypesModel) self.addCleanup(drop_table, AllDatatypesModel) input = AllDatatypes(a='ascii', b=2 ** 63 - 1, c=bytearray(b'hello world'), d=True, e=datetime.utcfromtimestamp(872835240), f=Decimal('12.3E+7'), g=2.39, h=3.4028234663852886e+38, i='123.123.123.123', j=2147483647, k='text', l=UUID('FE2B4360-28C6-11E2-81C1-0800200C9A66'), m=UUID('067e6162-3b6f-4ae2-a171-2470b63dff00'), n=int(str(2147483647) + '000')) AllDatatypesModel.create(id=0, data=input) self.assertEqual(1, AllDatatypesModel.objects.count()) output = AllDatatypesModel.objects.first().data for i in range(ord('a'), ord('a') + 14): self.assertEqual(input[chr(i)], output[chr(i)])
def main(): connection.default() # Management functions would normally be used in development, and possibly for deployments. # They are typically not part of a core application. log.info("### creating keyspace...") management.create_keyspace_simple(KEYSPACE, 1) log.info("### syncing model...") management.sync_table(FamilyMembers) # default uuid is assigned simmons = FamilyMembers.create(surname='Simmons', name='Gene', birth_year=1949, sex='m') # add members to his family later FamilyMembers.create(id=simmons.id, surname='Simmons', name='Nick', birth_year=1989, sex='m') sophie = FamilyMembers.create(id=simmons.id, surname='Simmons', name='Sophie', sex='f') nick = FamilyMembers.objects(id=simmons.id, surname='Simmons', name='Nick') try: nick.iff(birth_year=1988).update(birth_year=1989) except LWTException: print "precondition not met" # showing validation try: FamilyMembers.create(id=simmons.id, surname='Tweed', name='Shannon', birth_year=1957, sex='f') except ValidationError: log.exception('INTENTIONAL VALIDATION EXCEPTION; Failed creating instance:') FamilyMembers.create(id=simmons.id, surname='Tweed', name='Shannon', sex='f') log.info("### add multiple as part of a batch") # If creating many at one time, can use a batch to minimize round-trips hogan_id = uuid4() with BatchQuery() as b: FamilyMembers.batch(b).create(id=hogan_id, surname='Hogan', name='Hulk', sex='m') FamilyMembers.batch(b).create(id=hogan_id, surname='Hogan', name='Linda', sex='f') FamilyMembers.batch(b).create(id=hogan_id, surname='Hogan', name='Nick', sex='m') FamilyMembers.batch(b).create(id=hogan_id, surname='Hogan', name='Brooke', sex='f') log.info("### All members") for m in FamilyMembers.all(): print m, m.birth_year, m.sex log.info("### Select by partition key") for m in FamilyMembers.objects(id=simmons.id): print m, m.birth_year, m.sex log.info("### Constrain on clustering key") for m in FamilyMembers.objects(id=simmons.id, surname=simmons.surname): print m, m.birth_year, m.sex log.info("### Constrain on clustering key") kids = FamilyMembers.objects(id=simmons.id, surname=simmons.surname, name__in=['Nick', 'Sophie']) log.info("### Delete a record") FamilyMembers(id=hogan_id, surname='Hogan', name='Linda').delete() for m in FamilyMembers.objects(id=hogan_id): print m, m.birth_year, m.sex management.drop_keyspace(KEYSPACE)
def test_collection_with_default(self): """ Test the updates work as expected when an object is deleted @since 3.9 @jira_ticket PYTHON-657 @expected_result the non updated column is None and the updated column has the set value @test_category object_mapper """ sync_table(ModelWithDefaultCollection) item = ModelWithDefaultCollection.create(id=1, mf={1: 1}, dummy=1).save() self.assertEqual(ModelWithDefaultCollection.objects().all().get()._as_dict(), {'id': 1, 'dummy': 1, 'mf': {1: 1}}) item.update(mf={2: 2}) self.assertEqual(ModelWithDefaultCollection.objects().all().get()._as_dict(), {'id': 1, 'dummy': 1, 'mf': {2: 2}}) item.update(mf=None) self.assertEqual(ModelWithDefaultCollection.objects().all().get()._as_dict(), {'id': 1, 'dummy': 1, 'mf': {}}) item = ModelWithDefaultCollection.create(id=2, dummy=2).save() self.assertEqual(ModelWithDefaultCollection.objects().all().get(id=2)._as_dict(), {'id': 2, 'dummy': 2, 'mf': {2: 2}}) item.update(mf={1: 1, 4: 4}) self.assertEqual(ModelWithDefaultCollection.objects().all().get(id=2)._as_dict(), {'id': 2, 'dummy': 2, 'mf': {1: 1, 4: 4}}) drop_table(ModelWithDefaultCollection)
def setUpClass(cls): super(TestIndexedInheritanceQuery, cls).setUpClass() management.sync_table(IndexedInherit1) management.sync_table(IndexedInherit2) cls.p1 = IndexedInherit1.create(data1='pickle') cls.p2 = IndexedInherit2.create(partition=cls.p1.partition, data2='bacon')
def test_all_size_tiered_options(self): class AllSizeTieredOptionsModel(Model): __compaction__ = SizeTieredCompactionStrategy __compaction_bucket_low__ = .3 __compaction_bucket_high__ = 2 __compaction_min_threshold__ = 2 __compaction_max_threshold__ = 64 __compaction_tombstone_compaction_interval__ = 86400 cid = columns.UUID(primary_key=True) name = columns.Text() drop_table(AllSizeTieredOptionsModel) sync_table(AllSizeTieredOptionsModel) options = get_table_settings(AllSizeTieredOptionsModel).options['compaction_strategy_options'] options = json.loads(options) expected = {u'min_threshold': u'2', u'bucket_low': u'0.3', u'tombstone_compaction_interval': u'86400', u'bucket_high': u'2', u'max_threshold': u'64'} self.assertDictEqual(options, expected)
def test_bogus_option_update(self): sync_table(ModelWithTableProperties) option = 'no way will this ever be an option' try: ModelWithTableProperties.__options__[option] = 'what was I thinking?' self.assertRaisesRegexp(KeyError, "Invalid table option.*%s.*" % option, sync_table, ModelWithTableProperties) finally: ModelWithTableProperties.__options__.pop(option, None)
def test_static_columns(self): if PROTOCOL_VERSION < 2: raise unittest.SkipTest("Native protocol 2+ required, currently using: {0}".format(PROTOCOL_VERSION)) class StaticModel(Model): id = columns.Integer(primary_key=True) c = columns.Integer(primary_key=True) name = columns.Text(static=True) drop_table(StaticModel) session = get_session() with mock.patch.object(session, "execute", wraps=session.execute) as m: sync_table(StaticModel) self.assertGreater(m.call_count, 0) statement = m.call_args[0][0].query_string self.assertIn('"name" text static', statement) # if we sync again, we should not apply an alter w/ a static sync_table(StaticModel) with mock.patch.object(session, "execute", wraps=session.execute) as m2: sync_table(StaticModel) self.assertEqual(len(m2.call_args_list), 0)
def build_user_nodes(self): try: Utils.get_Connection_SNA4Slack() sync_table(SlackArchive) instances = SlackArchive.objects.filter(teamName=self.team_name) for row in instances: if row[CHANNEL_NAME] in channel_record.keys(): for user in channel_record[row[CHANNEL_NAME]]: if self.graph.has_edge(row[SENDER_COLUMN], user): self.graph[row[SENDER_COLUMN]][user][ EDGE_WEIGHT_LABEL] += 1 else: self.graph.add_edge(row[SENDER_COLUMN], user, weight=1) tsvalues = SlackArchive.objects.filter(timeStamp=self.timeStamp) for row in instances: self.graph.add_node(row[SENDER_COLUMN]) for val in tsvalues: self.graph.nodes.add_value(val) except Exception as e: print "exception found during node building: " + str(e)
def test_nested_udts_inserts(self): """ Test for inserting collections of user types using cql engine. test_nested_udts_inserts Constructs a model that contains a list of usertypes. It will then attempt to insert them. The expectation is that no exception is thrown during insert. For sanity sake we also validate that our input and output values match. This combination of model, and UT produces a syntax error in 2.5.1 due to improper quoting around the names collection. @since 2.6.0 @jira_ticket PYTHON-311 @expected_result No syntax exception thrown @test_category data_types:udt """ class Name(UserType): type_name__ = "header" name = columns.Text() value = columns.Text() class Container(Model): id = columns.UUID(primary_key=True, default=uuid4) names = columns.List(columns.UserDefinedType(Name)) # Construct the objects and insert them names = [] for i in range(0, 10): names.append(Name(name="name{0}".format(i), value="value{0}".format(i))) # Create table, insert data sync_table(Container) Container.create(id=UUID('FE2B4360-28C6-11E2-81C1-0800200C9A66'), names=names) # Validate input and output matches self.assertEqual(1, Container.objects.count()) names_output = Container.objects().first().names self.assertEqual(names_output, names)
def sync_db(db): keyspaces = [ 'book_keyspace', ] conns = ['node_0', 'node_1'] # registers your connections # ... # create all keyspaces on all connections for ks in keyspaces: management.create_keyspace_simple(name=ks, replication_factor=2, durable_writes=True, connections=conns) # define your Automobile model # ... # sync your models management.sync_table(db, keyspaces=keyspaces, connections=conns)
def test_sync_warnings(self): """ Test to insure when inconsistent changes are made to a table, or type as part of a sync call that the proper logging messages are surfaced @since 3.2 @jira_ticket PYTHON-260 @expected_result warnings are logged @test_category object_mapper """ mock_handler = MockLoggingHandler() logger = logging.getLogger(management.__name__) logger.addHandler(mock_handler) sync_table(BaseInconsistent) sync_table(ChangedInconsistent) self.assertTrue('differing from the model type' in mock_handler.messages.get('warning')[0]) if CASSANDRA_VERSION >= Version('2.1'): sync_type(DEFAULT_KEYSPACE, BaseInconsistentType) mock_handler.reset() sync_type(DEFAULT_KEYSPACE, ChangedInconsistentType) self.assertTrue('differing from the model user type' in mock_handler.messages.get('warning')[0]) logger.removeHandler(mock_handler)
def test_all_size_tiered_options(self): class AllSizeTieredOptionsModel(Model): __options__ = { 'compaction': { 'class': 'org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy', 'bucket_low': '.3', 'bucket_high': '2', 'min_threshold': '2', 'max_threshold': '64', 'tombstone_compaction_interval': '86400' } } cid = columns.UUID(primary_key=True) name = columns.Text() drop_table(AllSizeTieredOptionsModel) sync_table(AllSizeTieredOptionsModel) table_meta = _get_table_metadata(AllSizeTieredOptionsModel) self._verify_options(table_meta, AllSizeTieredOptionsModel.__options__)
def setUpClass(cls): super(BaseColumnIOTest, cls).setUpClass() # if the test column hasn't been defined, bail out if not cls.column: return # create a table with the given column class IOTestModel(Model): table_name = cls.column.db_type + "_io_test_model_{0}".format( uuid4().hex[:8]) pkey = cls.column(primary_key=True) data = cls.column() cls._generated_model = IOTestModel sync_table(cls._generated_model) # tupleify the tested values if not isinstance(cls.pkey_val, tuple): cls.pkey_val = cls.pkey_val, if not isinstance(cls.data_val, tuple): cls.data_val = cls.data_val,
def test_is_not_null_execution(self): """ Verify that CQL statements have correct syntax when executed If we wanted them to return something meaningful and not a InvalidRequest we'd have to create an index in search for the column we are using IsNotNull @since 2.5 @jira_ticket PYTHON-968 @expected_result InvalidRequest is arisen @test_category cqlengine """ sync_table(TestQueryUpdateModel) self.addCleanup(drop_table, TestQueryUpdateModel) # Raises InvalidRequest instead of dse.protocol.SyntaxException with self.assertRaises(InvalidRequest): list(TestQueryUpdateModel.filter(IsNotNull("text"))) with self.assertRaises(InvalidRequest): list(TestQueryUpdateModel.filter(IsNotNull("text"), partition=uuid4()))
def build_nodes(self): Utils.get_Connection_SNA4Slack() sync_table(SlackArchive) # instances = csv.DictReader(csvfile) instances = SlackArchive.objects.filter(teamName=self.team_name) memo = set() node_id = 0 for row in instances: self.graph.add_node(row[CHANNEL_NAME]) self.graph.add_edge(self.team_name, row[CHANNEL_NAME], weight=1) if (row[CHANNEL_NAME], row[SENDER_COLUMN]) not in memo: self.graph.add_node(node_id) self.graph.node[node_id]["name"] = row[SENDER_COLUMN] memo.add((row[CHANNEL_NAME], row[SENDER_COLUMN])) self.graph.add_edge(row[CHANNEL_NAME], node_id, weight=1) self.graph.node[node_id][USER_PROFILE_PIC] = row[ USER_PROFILE_PIC] node_id += 1
def load_csv(): # create the table using the FlightModel sync_table(FlightModel) # load the data from a csv df = pd.read_csv("data/flights_from_pg.csv", header=None) df.columns = [ 'id', 'year', 'day_of_month', 'fl_date', 'airline_id', 'carrier', 'fl_num', 'origin_airport_id', 'origin', 'origin_city_name', 'origin_state_abr', 'dest', 'dest_city_name', 'dest_state_abr', 'dep_time', 'arr_time', 'actual_elapsed_time', 'air_time', 'distance' ] # MUNGE: dep_time and arr_time have 2400 values - change those to 0000 df.dep_time[df.dep_time == 2400] = 0 df.arr_time[df.arr_time == 2400] = 0 # add the date parts to the departure and arrival times padtime = lambda x: "%04d" % x df.dep_time = df.fl_date + " " + df.dep_time.apply(padtime) df.arr_time = df.fl_date + " " + df.arr_time.apply(padtime) print(df.ix[1]) #(1048576, 1) # convert all the timestamp types to pandas datetimes df.fl_date = pd.to_datetime(df.fl_date) df.dep_time = pd.to_datetime(df.dep_time) df.arr_time = pd.to_datetime(df.arr_time) print(df.shape) df.apply(lambda r: FlightModel.create(**r.to_dict()), axis=1) # cqlsh> select count(*) from flughafen.flights; # count # --------- # 1048576 a_airports = len( list(filter(lambda x: x[0] == 'A', list(df.origin.unique())))) print("Number of airport codes starting with A: %d" % a_airports)
def __init__(self): sync_table(User) # create users_info table if not exits sync_table( PositionLatent) # create users_position_latent table if not exits sync_table(PositionRaw) # create users_position_raw table if not exits typer.secho("➲ [Database initialized successfully]", fg=typer.colors.WHITE, bold=True)
def sync_tables(keyspace): from cassandra.cqlengine import management management.sync_table(model=Entities, keyspaces=keyspace) management.sync_table(model=Environments, keyspaces=keyspace) management.sync_table(model=Devices, keyspaces=keyspace)
def setup_storage(settings=None): """Create cassandra models.""" from caliopen_storage.core import core_registry # Make discovery happen from caliopen_main.user.core import User from caliopen_main.contact.objects.contact import Contact from caliopen_main.message.objects.message import Message from caliopen_main.common.objects.tag import ResourceTag from caliopen_main.device.core import Device from caliopen_main.notification.core import Notification, NotificationTtl from cassandra.cqlengine.management import sync_table, \ create_keyspace_simple keyspace = Configuration('global').get('cassandra.keyspace') if not keyspace: raise Exception('Configuration missing for cassandra keyspace') # XXX tofix : define strategy and replication_factor in configuration create_keyspace_simple(keyspace, 1) for name, kls in core_registry.items(): log.info('Creating cassandra model %s' % name) if hasattr(kls._model_class, 'pk'): # XXX find a better way to detect model from udt sync_table(kls._model_class)
def test_alter_options(self): class AlterTable(Model): __options__ = { 'compaction': { 'class': 'org.apache.cassandra.db.compaction.LeveledCompactionStrategy', 'sstable_size_in_mb': '64' } } user_id = columns.UUID(primary_key=True) name = columns.Text() drop_table(AlterTable) sync_table(AlterTable) table_meta = _get_table_metadata(AlterTable) self.assertRegexpMatches(table_meta.export_as_string(), ".*'sstable_size_in_mb': '64'.*") AlterTable.__options__['compaction']['sstable_size_in_mb'] = '128' sync_table(AlterTable) table_meta = _get_table_metadata(AlterTable) self.assertRegexpMatches(table_meta.export_as_string(), ".*'sstable_size_in_mb': '128'.*")
def test_compaction_not_altered_without_changes_sizetiered(self): class SizeTieredCompactionChangesDetectionTest(Model): __options__ = { 'compaction': { 'class': 'org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy', 'bucket_high': '20', 'bucket_low': '10', 'max_threshold': '200', 'min_threshold': '100', 'min_sstable_size': '1000', 'tombstone_threshold': '0.125', 'tombstone_compaction_interval': '3600' } } pk = columns.Integer(primary_key=True) drop_table(SizeTieredCompactionChangesDetectionTest) sync_table(SizeTieredCompactionChangesDetectionTest) self.assertFalse( _update_options(SizeTieredCompactionChangesDetectionTest))
def test_can_update_udts_with_nones(self): class User(UserType): age = columns.Integer() name = columns.Text() class UserModel(Model): id = columns.Integer(primary_key=True) info = columns.UserDefinedType(User) sync_table(UserModel) user = User(age=42, name="John") created_user = UserModel.create(id=0, info=user) john_info = UserModel.objects().first().info self.assertEqual(42, john_info.age) self.assertEqual("John", john_info.name) created_user.info = None created_user.update() john_info = UserModel.objects().first().info self.assertIsNone(john_info)
def create_event_data_table(): try: setup(['127.0.0.1'], 'sparkifydb', retry_connect=True) sync_table(model.table.EventDataModel) sync_table(model.table.UserAndSessionModel) sync_table(model.table.UserAndSongModel) except Exception as e: print(e) exit(1)
def setUpClass(cls): super(TestUnindexedInheritanceQuery, cls).setUpClass() management.sync_table(UnindexedInherit1) management.sync_table(UnindexedInherit2) management.sync_table(UnindexedInherit3) cls.p1 = UnindexedInherit1.create(data1='pickle') cls.p2 = UnindexedInherit2.create(partition=cls.p1.partition, data2='bacon') cls.p3 = UnindexedInherit3.create(partition=cls.p1.partition, data3='turkey')
def test_can_insert_model_with_all_protocol_v4_column_types(self): """ Test for inserting all protocol v4 column types into a Model test_can_insert_model_with_all_protocol_v4_column_types tests that each cqlengine protocol v4 column type can be inserted into a Model. It first creates a Model that has each cqlengine protocol v4 column type. It then creates a Model instance where all the fields have corresponding data, which performs the insert into the Cassandra table. Finally, it verifies that each column read from the Model from Cassandra is the same as the input parameters. @since 2.6.0 @jira_ticket PYTHON-245 @expected_result The Model is inserted with each protocol v4 column type, and the resulting read yields proper data for each column. @test_category data_types:primitive """ if PROTOCOL_VERSION < 4: raise unittest.SkipTest("Protocol v4 datatypes require native protocol 4+, currently using: {0}".format(PROTOCOL_VERSION)) class v4DatatypesModel(Model): id = columns.Integer(primary_key=True) a = columns.Date() b = columns.SmallInt() c = columns.Time() d = columns.TinyInt() sync_table(v4DatatypesModel) input = [Date(date(1970, 1, 1)), 32523, Time(time(16, 47, 25, 7)), 123] v4DatatypesModel.create(id=0, a=date(1970, 1, 1), b=32523, c=time(16, 47, 25, 7), d=123) self.assertEqual(1, v4DatatypesModel.objects.count()) output = v4DatatypesModel.objects().first() for i, i_char in enumerate(range(ord('a'), ord('a') + 3)): self.assertEqual(input[i], output[chr(i_char)])
def test_connection_session_switch(self): """ Test to ensure that when the default keyspace is changed in a session and that session, is set in the connection class, that the new defaul keyspace is honored. @since 3.1 @jira_ticket PYTHON-486 @expected_result CQLENGINE adopts whatever keyspace is passed in vai the set_session method as default @test_category object_mapper """ connection.set_session(self.session1) sync_table(TestConnectModel) TCM1 = TestConnectModel.create(id=1, keyspace=self.keyspace1) connection.set_session(self.session2) sync_table(TestConnectModel) TCM2 = TestConnectModel.create(id=1, keyspace=self.keyspace2) connection.set_session(self.session1) self.assertEqual(1, TestConnectModel.objects.count()) self.assertEqual(TestConnectModel.objects.first(), TCM1) connection.set_session(self.session2) self.assertEqual(1, TestConnectModel.objects.count()) self.assertEqual(TestConnectModel.objects.first(), TCM2)
def test_lots_of_queries(self): import resource import objgraph class LoadTest(Model): k = columns.Integer(primary_key=True) v = columns.Integer() sync_table(LoadTest) gc.collect() objgraph.show_most_common_types() print("Starting...") for i in range(1000000): if i % 25000 == 0: # print memory statistic print("Memory usage: %s" % (resource.getrusage(resource.RUSAGE_SELF).ru_maxrss)) LoadTest.create(k=i, v=i) objgraph.show_most_common_types() raise Exception("you shouldn't be here")
def setUp(self): class MyModel(Model): id = columns.Text(max_length=63, primary_key=True, default=lambda: str(uuid.uuid4())) value = columns.Text(max_length=63) self.model = MyModel connection.setup(['127.0.0.1'], self.keyspace) create_keyspace(self.keyspace, strategy_class='SimpleStrategy', replication_factor=1) sync_table(MyModel) class MyManager(CQLManager): model = self.model fields = ( 'id', 'value', ) create_fields = ('value', ) update_fields = ('value', ) self.manager_class = MyManager
def init_database(cluster, keyspace, db_user, db_pass): try: auth_provider = PlainTextAuthProvider(username=db_user, password=db_pass) connection.setup(cluster, keyspace, auth_provider=auth_provider) sync_table(User) sync_table(Tweet) sync_table(Relation) except Exception: raise
def connect_cassandra(role): auth_provider = PlainTextAuthProvider(username=str(role), password=str(role)) connection.setup(['127.0.0.1'], "sensors", auth_provider=auth_provider, protocol_version=3) sync_table(Users) sync_table(TemperatureSensor) sync_table(PressureSensor) return True
def setUpClass(cls): super(TestUnindexedPolymorphicQuery, cls).setUpClass() management.sync_table(UnindexedPoly1) management.sync_table(UnindexedPoly2) management.sync_table(UnindexedPoly3) cls.p1 = UnindexedPoly1.create(data1='pickle') cls.p2 = UnindexedPoly2.create(partition=cls.p1.partition, data2='bacon') cls.p3 = UnindexedPoly3.create(partition=cls.p1.partition, data3='turkey')
def test_add_column(self): sync_table(FirstModel) fields = get_fields(FirstModel) # this should contain the second key self.assertEqual(len(fields), 2) # get schema sync_table(SecondModel) fields = get_fields(FirstModel) self.assertEqual(len(fields), 3) sync_table(ThirdModel) fields = get_fields(FirstModel) self.assertEqual(len(fields), 4) sync_table(FourthModel) fields = get_fields(FirstModel) self.assertEqual(len(fields), 4)
def test_add_column(self): sync_table(FirstModel) fields = _get_non_pk_field_names(_get_table_metadata(FirstModel)) # this should contain the second key self.assertEqual(len(fields), 2) # get schema sync_table(SecondModel) fields = _get_non_pk_field_names(_get_table_metadata(FirstModel)) self.assertEqual(len(fields), 3) sync_table(ThirdModel) fields = _get_non_pk_field_names(_get_table_metadata(FirstModel)) self.assertEqual(len(fields), 4) sync_table(FourthModel) fields = _get_non_pk_field_names(_get_table_metadata(FirstModel)) self.assertEqual(len(fields), 4)
def main(argv): game = None if argv[0] == 'csgo': game = Game.CSGO elif argv[0] == 'dota2': game = Game.DOTA2 item_names = steam_market.get_item_names(game) print('Number of items is %d' % len(item_names)) set_cassandra_hosts(['127.0.0.1']) set_cassandra_keyspace('scraper') connection.setup(cassandra_hosts, cassandra_keyspace_name, protocol_version=3) sync_table(Item) sync_table(ItemCsgo) sync_table(ItemDota2) timer = time.time() segment = int(len(item_names) / 8) print('Segment size is %d' % segment) workers = [] for i in range(0, 8): if i < 7: workers.append(Worker('worker ' + str(i + 1), Game.CSGO, item_names[i * segment:(i+1) * segment])) else: workers.append(Worker('worker 8', Game.CSGO, item_names[i * segment:len(item_names)])) for i in range(0, 8): workers[i].start() for i in range(0, 8): workers[i].join() print('Number of items %d' % ItemCsgo.objects.count()) timer = time.time() - timer print(timer) for i in range(0, 60): print('Sleeping %d seconds' % i) time.sleep(1) workers = [] market.config.CHECKED_ITEMS = 0 timer_history = time.time() for i in range(0, 8): if i < 7: workers.append(HistoryWorker('worker ' + str(i + 1), Game.CSGO, item_names[i * segment:(i + 1) * segment])) else: workers.append(HistoryWorker('worker 8', Game.CSGO, item_names[i * segment:len(item_names)])) for i in range(0, 8): workers[i].start() for i in range(0, 8): workers[i].join() timer_history = time.time() - timer_history print(timer_history)
async def setup_cassandra(app, loop): logger.warning('Initializing Database') cluster = Cluster(['cassandra_seed_node']) session = cluster.connect() session.execute( """CREATE KEYSPACE IF NOT EXISTS soda WITH replication = {'class': 'SimpleStrategy', 'replication_factor': '1'} AND durable_writes = true;""" ) session.execute('USE soda') register_connection(str(session), session=session) set_default_connection(str(session)) sync_table(User, keyspaces=['soda']) sync_table(Record, keyspaces=['soda']) sync_table(Project, keyspaces=['soda']) aiosession(session) app.db = session return app