def test_can_insert_nested_udts(self):
class Depth_0(UserType):
age = columns.Integer()
name = columns.Text()
class Depth_1(UserType):
value = columns.UserDefinedType(Depth_0)
class Depth_2(UserType):
value = columns.UserDefinedType(Depth_1)
class Depth_3(UserType):
value = columns.UserDefinedType(Depth_2)
class DepthModel(Model):
id = columns.Integer(primary_key=True)
v_0 = columns.UserDefinedType(Depth_0)
v_1 = columns.UserDefinedType(Depth_1)
v_2 = columns.UserDefinedType(Depth_2)
v_3 = columns.UserDefinedType(Depth_3)
sync_table(DepthModel)
udts = [Depth_0(age=42, name="John")]
udts.append(Depth_1(value=udts[0]))
udts.append(Depth_2(value=udts[1]))
udts.append(Depth_3(value=udts[2]))
DepthModel.create(id=0, v_0=udts[0], v_1=udts[1], v_2=udts[2], v_3=udts[3])
output = DepthModel.objects().first()
self.assertEqual(udts[0], output.v_0)
self.assertEqual(udts[1], output.v_1)
self.assertEqual(udts[2], output.v_2)
self.assertEqual(udts[3], output.v_3)
def test_udt_validate_with_default(self):
"""
Test to verify restrictions are honored and that validate is called
on the default value
@since 3.10
@jira_ticket PYTHON-505
@expected_result a validation error is arisen due to the name being
too long
@test_category data_types:object_mapper
"""
class UserValidateDefault(UserType):
age = columns.Integer()
name = columns.Text(max_length=2, default="Robert")
class UserModelValidateDefault(Model):
id = columns.Integer(primary_key=True)
info = columns.UserDefinedType(UserValidateDefault)
sync_table(UserModelValidateDefault)
self.addCleanup(drop_table, UserModelValidateDefault)
user = UserValidateDefault(age=1)
item = UserModelValidateDefault(id=1, info=user)
with self.assertRaises(ValidationError):
item.save()
def _reset_data(self):
for ks in self.keyspaces:
drop_keyspace(ks, connections=self.conns)
for ks in self.keyspaces:
create_keyspace_simple(ks, 1, connections=self.conns)
sync_table(TestModel, keyspaces=self.keyspaces, connections=self.conns)
def sync_db(self):
"""Sync all defined tables. All defined models must be imported before
this method is called
"""
models = get_subclasses(self.Model)
for model in models:
sync_table(model)
def test_can_insert_partial_udts(self):
class User(UserType):
age = columns.Integer()
name = columns.Text()
gender = columns.Text()
class UserModel(Model):
id = columns.Integer(primary_key=True)
info = columns.UserDefinedType(User)
sync_table(UserModel)
user = User(age=42, name="John")
UserModel.create(id=0, info=user)
john_info = UserModel.objects().first().info
self.assertEqual(42, john_info.age)
self.assertEqual("John", john_info.name)
self.assertIsNone(john_info.gender)
user = User(age=42)
UserModel.create(id=0, info=user)
john_info = UserModel.objects().first().info
self.assertEqual(42, john_info.age)
self.assertIsNone(john_info.name)
self.assertIsNone(john_info.gender)
def test_can_insert_udts_with_all_datatypes(self):
class AllDatatypes(UserType):
a = columns.Ascii()
b = columns.BigInt()
c = columns.Blob()
d = columns.Boolean()
e = columns.Date()
f = columns.DateTime()
g = columns.Decimal()
h = columns.Float(double_precision=True)
i = columns.Inet()
j = columns.Integer()
k = columns.Text()
l = columns.TimeUUID()
m = columns.UUID()
n = columns.VarInt()
class AllDatatypesModel(Model):
id = columns.Integer(primary_key=True)
data = columns.UserDefinedType(AllDatatypes)
sync_table(AllDatatypesModel)
input = AllDatatypes(a='ascii', b=2 ** 63 - 1, c=bytearray(b'hello world'), d=True, e=date(1970, 1, 1),
f=datetime.utcfromtimestamp(872835240),
g=Decimal('12.3E+7'), h=3.4028234663852886e+38, i='123.123.123.123', j=2147483647,
k='text', l= UUID('FE2B4360-28C6-11E2-81C1-0800200C9A66'),
m=UUID('067e6162-3b6f-4ae2-a171-2470b63dff00'), n=int(str(2147483647) + '000'))
alldata = AllDatatypesModel.create(id=0, data=input)
self.assertEqual(1, AllDatatypesModel.objects.count())
output = AllDatatypesModel.objects().first().data
for i in range(ord('a'), ord('a') + 14):
self.assertEqual(input[chr(i)], output[chr(i)])
def test_can_insert_udts_with_nones(self):
"""
Test for inserting all column types as empty into a UserType as None's
test_can_insert_udts_with_nones tests that each cqlengine column type can be inserted into a UserType as None's.
It first creates a UserType that has each cqlengine column type, and a corresponding table/Model. It then creates
a UserType instance where all the fields are None's and inserts the UserType as an instance of the Model. Finally,
it verifies that each column read from the UserType from Cassandra is None.
@since 2.5.0
@jira_ticket PYTHON-251
@expected_result The UserType is inserted with each column type, and the resulting read yields None's for each column.
@test_category data_types:udt
"""
sync_table(AllDatatypesModel)
self.addCleanup(drop_table, AllDatatypesModel)
input = AllDatatypes(a=None, b=None, c=None, d=None, e=None, f=None, g=None, h=None, i=None, j=None, k=None,
l=None, m=None, n=None)
AllDatatypesModel.create(id=0, data=input)
self.assertEqual(1, AllDatatypesModel.objects.count())
output = AllDatatypesModel.objects.first().data
self.assertEqual(input, output)
def test_model_over_write(self):
"""
Test to ensure overwriting of primary keys in model inheritance is allowed
This is currently only an issue in PyPy. When PYTHON-504 is introduced this should
be updated error out and warn the user
@since 3.6.0
@jira_ticket PYTHON-576
@expected_result primary keys can be overwritten via inheritance
@test_category object_mapper
"""
class TimeModelBase(Model):
uuid = columns.TimeUUID(primary_key=True)
class DerivedTimeModel(TimeModelBase):
__table_name__ = 'derived_time'
uuid = columns.TimeUUID(primary_key=True, partition_key=True)
value = columns.Text(required=False)
# In case the table already exists in keyspace
drop_table(DerivedTimeModel)
sync_table(DerivedTimeModel)
uuid_value = uuid1()
uuid_value2 = uuid1()
DerivedTimeModel.create(uuid=uuid_value, value="first")
DerivedTimeModel.create(uuid=uuid_value2, value="second")
DerivedTimeModel.objects.filter(uuid=uuid_value)
def test_can_insert_udts_with_nones(self):
"""
Test for inserting all column types as empty into a UserType as None's
test_can_insert_udts_with_nones tests that each cqlengine column type can be inserted into a UserType as None's.
It first creates a UserType that has each cqlengine column type, and a corresponding table/Model. It then creates
a UserType instance where all the fields are None's and inserts the UserType as an instance of the Model. Finally,
it verifies that each column read from the UserType from Cassandra is None.
@since 2.5.0
@jira_ticket PYTHON-251
@expected_result The UserType is inserted with each column type, and the resulting read yields None's for each column.
@test_category data_types:udt
"""
class AllDatatypes(UserType):
a = columns.Ascii()
b = columns.BigInt()
c = columns.Blob()
d = columns.Boolean()
e = columns.DateTime()
f = columns.Decimal()
g = columns.Double()
h = columns.Float()
i = columns.Inet()
j = columns.Integer()
k = columns.Text()
l = columns.TimeUUID()
m = columns.UUID()
n = columns.VarInt()
class AllDatatypesModel(Model):
id = columns.Integer(primary_key=True)
data = columns.UserDefinedType(AllDatatypes)
sync_table(AllDatatypesModel)
input = AllDatatypes(
a=None,
b=None,
c=None,
d=None,
e=None,
f=None,
g=None,
h=None,
i=None,
j=None,
k=None,
l=None,
m=None,
n=None,
)
AllDatatypesModel.create(id=0, data=input)
self.assertEqual(1, AllDatatypesModel.objects.count())
output = AllDatatypesModel.objects().first().data
self.assertEqual(input, output)
def test_defaultFetchSize(self):
# Populate Table
People.objects.create(last_name="Smith", first_name="John", birthday=datetime.now())
People.objects.create(last_name="Bestwater", first_name="Alan", birthday=datetime.now())
People.objects.create(last_name="Smith", first_name="Greg", birthday=datetime.now())
People.objects.create(last_name="Smith", first_name="Adam", birthday=datetime.now())
# Check query constructions
expected_fields = ['first_name', 'birthday']
self.assertEqual(People.filter(last_name="Smith")._select_fields(), expected_fields)
# Validate correct fields are fetched
smiths = list(People.filter(last_name="Smith"))
self.assertEqual(len(smiths), 3)
self.assertTrue(smiths[0].last_name is not None)
# Modify table with new value
sync_table(People2)
# populate new format
People2.objects.create(last_name="Smith", first_name="Chris", middle_name="Raymond", birthday=datetime.now())
People2.objects.create(last_name="Smith", first_name="Andrew", middle_name="Micheal", birthday=datetime.now())
# validate query construction
expected_fields = ['first_name', 'middle_name', 'birthday']
self.assertEqual(People2.filter(last_name="Smith")._select_fields(), expected_fields)
# validate correct items are returneds
smiths = list(People2.filter(last_name="Smith"))
self.assertEqual(len(smiths), 5)
self.assertTrue(smiths[0].last_name is not None)
def setUpClass(cls):
if PROTOCOL_VERSION < 4 or CASSANDRA_VERSION < "3.0":
return
cls.db_klass, cls.python_klass = UserDefinedType, User
cls.first_value = User(
age=1,
date_param=datetime.utcnow(),
map_param={1: time(2, 12, 7, 50), 2: util.Time(time(2, 12, 7, 49))},
list_param=[datetime(1, 1, 2), datetime(1, 1, 3)],
set_param=set((datetime(1, 1, 3), util.Date(datetime(1, 1, 1)))),
tuple_param=(datetime(1, 1, 3), 2, False, 1, 2.324, uuid4())
)
cls.second_value = User(
age=1,
date_param=datetime.utcnow(),
map_param={1: time(2, 12, 7, 50), 2: util.Time(time(2, 12, 7, 49))},
list_param=[datetime(1, 1, 2), datetime(1, 2, 3)],
set_param=None,
tuple_param=(datetime(1, 1, 2), 2, False, 1, 2.324, uuid4())
)
cls.third_value = User(
age=2,
date_param=None,
map_param={1: time(2, 12, 7, 51), 2: util.Time(time(2, 12, 7, 49))},
list_param=[datetime(1, 1, 2), datetime(1, 1, 4)],
set_param=set((datetime(1, 1, 3), util.Date(datetime(1, 1, 2)))),
tuple_param=(None, 3, False, None, 2.3214, uuid4())
)
cls.model_class = UserModel
sync_table(cls.model_class)
def setUpClass(cls):
super(TestIndexedPolymorphicQuery, cls).setUpClass()
management.sync_table(IndexedPoly1)
management.sync_table(IndexedPoly2)
cls.p1 = IndexedPoly1.create(data1='pickle')
cls.p2 = IndexedPoly2.create(partition=cls.p1.partition, data2='bacon')
def sync_cassandra():
from cassandra.cqlengine.management import sync_table, create_keyspace
from benchmark.feeds import UserFeed, TimelineFeed
create_keyspace('stream_framework_bench', 'SimpleStrategy', 3)
for feed_class in [UserFeed, TimelineFeed]:
timeline = feed_class.get_timeline_storage()
sync_table(timeline.model)
def sync(self, alias):
engine = get_engine_from_db_alias(alias)
if engine != 'django_cassandra_engine':
raise CommandError('Database {} is not cassandra!'.format(alias))
connection = connections[alias]
connection.connect()
options = connection.settings_dict.get('OPTIONS', {})
keyspace = connection.settings_dict['NAME']
replication_opts = options.get('replication', {})
strategy_class = replication_opts.pop('strategy_class',
'SimpleStrategy')
replication_factor = replication_opts.pop('replication_factor', 1)
self.stdout.write('Creating keyspace {}..'.format(keyspace))
if strategy_class == 'SimpleStrategy':
create_keyspace_simple(keyspace, replication_factor)
else:
create_keyspace_network_topology(keyspace, replication_opts)
for app_name, app_models \
in connection.introspection.cql_models.items():
for model in app_models:
self.stdout.write('Syncing %s.%s' % (app_name, model.__name__))
sync_table(model)
def test_default_values(self):
"""
Test that default types are set on object creation for UDTs
@since 3.7.0
@jira_ticket PYTHON-606
@expected_result Default values should be set.
@test_category data_types:udt
"""
class NestedUdt(UserType):
test_id = columns.UUID(default=uuid4)
something = columns.Text()
default_text = columns.Text(default="default text")
class OuterModel(Model):
name = columns.Text(primary_key=True)
first_name = columns.Text()
nested = columns.List(columns.UserDefinedType(NestedUdt))
simple = columns.UserDefinedType(NestedUdt)
sync_table(OuterModel)
self.addCleanup(drop_table, OuterModel)
t = OuterModel.create(name='test1')
t.nested = [NestedUdt(something='test')]
t.simple = NestedUdt(something="")
t.save()
self.assertIsNotNone(t.nested[0].test_id)
self.assertEqual(t.nested[0].default_text, "default text")
self.assertIsNotNone(t.simple.test_id)
self.assertEqual(t.simple.default_text, "default text")
def test_table_definition(self):
""" Tests that creating a table with capitalized column names succeeds """
sync_table(LowercaseKeyModel)
sync_table(CapitalizedKeyModel)
drop_table(LowercaseKeyModel)
drop_table(CapitalizedKeyModel)
def test_udts_with_unicode(self):
"""
Test for inserting models with unicode and udt columns.
test_udts_with_unicode constructs a model with a user defined type. It then attempts to insert that model with
a unicode primary key. It will also attempt to upsert a udt that contains unicode text.
@since 3.0.0
@jira_ticket PYTHON-353
@expected_result No exceptions thrown
@test_category data_types:udt
"""
ascii_name = 'normal name'
unicode_name = u'Fran\u00E7ois'
class User(UserType):
age = columns.Integer()
name = columns.Text()
class UserModelText(Model):
id = columns.Text(primary_key=True)
info = columns.UserDefinedType(User)
sync_table(UserModelText)
# Two udt instances one with a unicode one with ascii
user_template_ascii = User(age=25, name=ascii_name)
user_template_unicode = User(age=25, name=unicode_name)
UserModelText.create(id=ascii_name, info=user_template_unicode)
UserModelText.create(id=unicode_name, info=user_template_ascii)
UserModelText.create(id=unicode_name, info=user_template_unicode)
def test_keywords_as_names(self):
create_keyspace_simple('keyspace', 1)
class table(Model):
__keyspace__ = 'keyspace'
select = columns.Integer(primary_key=True)
table = columns.Text()
# create should work
drop_table(table)
sync_table(table)
created = table.create(select=0, table='table')
selected = table.objects(select=0)[0]
self.assertEqual(created.select, selected.select)
self.assertEqual(created.table, selected.table)
# alter should work
class table(Model):
__keyspace__ = 'keyspace'
select = columns.Integer(primary_key=True)
table = columns.Text()
where = columns.Text()
sync_table(table)
created = table.create(select=1, table='table')
selected = table.objects(select=1)[0]
self.assertEqual(created.select, selected.select)
self.assertEqual(created.table, selected.table)
self.assertEqual(created.where, selected.where)
drop_keyspace('keyspace')
def test_can_insert_double_and_float(self):
"""
Test for inserting single-precision and double-precision values into a Float and Double columns
@since 2.6.0
@changed 3.0.0 removed deprecated Float(double_precision) parameter
@jira_ticket PYTHON-246
@expected_result Each floating point column type is able to hold their respective precision values.
@test_category data_types:primitive
"""
class FloatingPointModel(Model):
id = columns.Integer(primary_key=True)
f = columns.Float()
d = columns.Double()
sync_table(FloatingPointModel)
FloatingPointModel.create(id=0, f=2.39)
output = FloatingPointModel.objects().first()
self.assertEqual(2.390000104904175, output.f) # float loses precision
FloatingPointModel.create(id=0, f=3.4028234663852886e+38, d=2.39)
output = FloatingPointModel.objects().first()
self.assertEqual(3.4028234663852886e+38, output.f)
self.assertEqual(2.39, output.d) # double retains precision
FloatingPointModel.create(id=0, d=3.4028234663852886e+38)
output = FloatingPointModel.objects().first()
self.assertEqual(3.4028234663852886e+38, output.d)
def setUpClass(cls):
if PROTOCOL_VERSION < 4:
return
super(TestQuerying, cls).setUpClass()
drop_table(TestQueryModel)
sync_table(TestQueryModel)
def test_set_table_properties(self):
sync_table(ModelWithTableProperties)
expected = {'bloom_filter_fp_chance': 0.76328,
'comment': 'TxfguvBdzwROQALmQBOziRMbkqVGFjqcJfVhwGR',
'gc_grace_seconds': 2063,
'read_repair_chance': 0.17985,
# For some reason 'dclocal_read_repair_chance' in CQL is called
# just 'local_read_repair_chance' in the schema table.
# Source: https://issues.apache.org/jira/browse/CASSANDRA-6717
# TODO: due to a bug in the native driver i'm not seeing the local read repair chance show up
# 'local_read_repair_chance': 0.50811,
}
if CASSANDRA_VERSION <= '2.0.0':
expected['caching'] = CACHING_ALL
expected['replicate_on_write'] = False
if CASSANDRA_VERSION == '2.0.0':
expected['populate_io_cache_on_flush'] = True
expected['index_interval'] = 98706
if CASSANDRA_VERSION >= '2.0.0':
expected['default_time_to_live'] = 4756
expected['memtable_flush_period_in_ms'] = 43681
options = management.get_table_settings(ModelWithTableProperties).options
self.assertEqual(dict([(k, options.get(k)) for k in expected.keys()]),
expected)
def test_can_insert_udts_with_nulls(self):
class AllDatatypes(UserType):
a = columns.Ascii()
b = columns.BigInt()
c = columns.Blob()
d = columns.Boolean()
e = columns.Date()
f = columns.DateTime()
g = columns.Decimal()
h = columns.Float()
i = columns.Inet()
j = columns.Integer()
k = columns.Text()
l = columns.TimeUUID()
m = columns.UUID()
n = columns.VarInt()
class AllDatatypesModel(Model):
id = columns.Integer(primary_key=True)
data = columns.UserDefinedType(AllDatatypes)
sync_table(AllDatatypesModel)
input = AllDatatypes(a=None, b=None, c=None, d=None, e=None, f=None, g=None, h=None, i=None, j=None, k=None, l=None, m=None, n=None)
AllDatatypesModel.create(id=0, data=input)
self.assertEqual(1, AllDatatypesModel.objects.count())
output = AllDatatypesModel.objects().first().data
self.assertEqual(input, output)
def test_sync_table_works_with_primary_keys_only_tables(self):
# This is "create table":
sync_table(PrimaryKeysOnlyModel)
# let's make sure settings persisted correctly:
assert PrimaryKeysOnlyModel.__compaction__ == LeveledCompactionStrategy
# blows up with DoesNotExist if table does not exist
table_settings = management.get_table_settings(PrimaryKeysOnlyModel)
# let make sure the flag we care about
assert LeveledCompactionStrategy in table_settings.options['compaction_strategy_class']
# Now we are "updating" the table:
# setting up something to change
PrimaryKeysOnlyModel.__compaction__ = SizeTieredCompactionStrategy
# primary-keys-only tables do not create entries in system.schema_columns
# table. Only non-primary keys are added to that table.
# Our code must deal with that eventuality properly (not crash)
# on subsequent runs of sync_table (which runs get_fields internally)
get_fields(PrimaryKeysOnlyModel)
sync_table(PrimaryKeysOnlyModel)
table_settings = management.get_table_settings(PrimaryKeysOnlyModel)
assert SizeTieredCompactionStrategy in table_settings.options['compaction_strategy_class']
def setUpClass(cls):
if PROTOCOL_VERSION < 4:
raise unittest.SkipTest("Date query tests require native protocol 4+, currently using: {0}".format(PROTOCOL_VERSION))
super(TestQuerying, cls).setUpClass()
drop_table(TestQueryModel)
sync_table(TestQueryModel)
def test_can_insert_udts_with_all_datatypes(self):
"""
Test for inserting all column types into a UserType
test_can_insert_udts_with_all_datatypes tests that each cqlengine column type can be inserted into a UserType.
It first creates a UserType that has each cqlengine column type, and a corresponding table/Model. It then creates
a UserType instance where all the fields have corresponding data, and inserts the UserType as an instance of the Model.
Finally, it verifies that each column read from the UserType from Cassandra is the same as the input parameters.
@since 2.5.0
@jira_ticket PYTHON-251
@expected_result The UserType is inserted with each column type, and the resulting read yields proper data for each column.
@test_category data_types:udt
"""
sync_table(AllDatatypesModel)
self.addCleanup(drop_table, AllDatatypesModel)
input = AllDatatypes(a='ascii', b=2 ** 63 - 1, c=bytearray(b'hello world'), d=True,
e=datetime.utcfromtimestamp(872835240), f=Decimal('12.3E+7'), g=2.39,
h=3.4028234663852886e+38, i='123.123.123.123', j=2147483647, k='text',
l=UUID('FE2B4360-28C6-11E2-81C1-0800200C9A66'),
m=UUID('067e6162-3b6f-4ae2-a171-2470b63dff00'), n=int(str(2147483647) + '000'))
AllDatatypesModel.create(id=0, data=input)
self.assertEqual(1, AllDatatypesModel.objects.count())
output = AllDatatypesModel.objects.first().data
for i in range(ord('a'), ord('a') + 14):
self.assertEqual(input[chr(i)], output[chr(i)])
def main():
connection.default()
# Management functions would normally be used in development, and possibly for deployments.
# They are typically not part of a core application.
log.info("### creating keyspace...")
management.create_keyspace_simple(KEYSPACE, 1)
log.info("### syncing model...")
management.sync_table(FamilyMembers)
# default uuid is assigned
simmons = FamilyMembers.create(surname='Simmons', name='Gene', birth_year=1949, sex='m')
# add members to his family later
FamilyMembers.create(id=simmons.id, surname='Simmons', name='Nick', birth_year=1989, sex='m')
sophie = FamilyMembers.create(id=simmons.id, surname='Simmons', name='Sophie', sex='f')
nick = FamilyMembers.objects(id=simmons.id, surname='Simmons', name='Nick')
try:
nick.iff(birth_year=1988).update(birth_year=1989)
except LWTException:
print "precondition not met"
# showing validation
try:
FamilyMembers.create(id=simmons.id, surname='Tweed', name='Shannon', birth_year=1957, sex='f')
except ValidationError:
log.exception('INTENTIONAL VALIDATION EXCEPTION; Failed creating instance:')
FamilyMembers.create(id=simmons.id, surname='Tweed', name='Shannon', sex='f')
log.info("### add multiple as part of a batch")
# If creating many at one time, can use a batch to minimize round-trips
hogan_id = uuid4()
with BatchQuery() as b:
FamilyMembers.batch(b).create(id=hogan_id, surname='Hogan', name='Hulk', sex='m')
FamilyMembers.batch(b).create(id=hogan_id, surname='Hogan', name='Linda', sex='f')
FamilyMembers.batch(b).create(id=hogan_id, surname='Hogan', name='Nick', sex='m')
FamilyMembers.batch(b).create(id=hogan_id, surname='Hogan', name='Brooke', sex='f')
log.info("### All members")
for m in FamilyMembers.all():
print m, m.birth_year, m.sex
log.info("### Select by partition key")
for m in FamilyMembers.objects(id=simmons.id):
print m, m.birth_year, m.sex
log.info("### Constrain on clustering key")
for m in FamilyMembers.objects(id=simmons.id, surname=simmons.surname):
print m, m.birth_year, m.sex
log.info("### Constrain on clustering key")
kids = FamilyMembers.objects(id=simmons.id, surname=simmons.surname, name__in=['Nick', 'Sophie'])
log.info("### Delete a record")
FamilyMembers(id=hogan_id, surname='Hogan', name='Linda').delete()
for m in FamilyMembers.objects(id=hogan_id):
print m, m.birth_year, m.sex
management.drop_keyspace(KEYSPACE)
def test_collection_with_default(self):
"""
Test the updates work as expected when an object is deleted
@since 3.9
@jira_ticket PYTHON-657
@expected_result the non updated column is None and the
updated column has the set value
@test_category object_mapper
"""
sync_table(ModelWithDefaultCollection)
item = ModelWithDefaultCollection.create(id=1, mf={1: 1}, dummy=1).save()
self.assertEqual(ModelWithDefaultCollection.objects().all().get()._as_dict(),
{'id': 1, 'dummy': 1, 'mf': {1: 1}})
item.update(mf={2: 2})
self.assertEqual(ModelWithDefaultCollection.objects().all().get()._as_dict(),
{'id': 1, 'dummy': 1, 'mf': {2: 2}})
item.update(mf=None)
self.assertEqual(ModelWithDefaultCollection.objects().all().get()._as_dict(),
{'id': 1, 'dummy': 1, 'mf': {}})
item = ModelWithDefaultCollection.create(id=2, dummy=2).save()
self.assertEqual(ModelWithDefaultCollection.objects().all().get(id=2)._as_dict(),
{'id': 2, 'dummy': 2, 'mf': {2: 2}})
item.update(mf={1: 1, 4: 4})
self.assertEqual(ModelWithDefaultCollection.objects().all().get(id=2)._as_dict(),
{'id': 2, 'dummy': 2, 'mf': {1: 1, 4: 4}})
drop_table(ModelWithDefaultCollection)
def setUpClass(cls):
super(TestIndexedInheritanceQuery, cls).setUpClass()
management.sync_table(IndexedInherit1)
management.sync_table(IndexedInherit2)
cls.p1 = IndexedInherit1.create(data1='pickle')
cls.p2 = IndexedInherit2.create(partition=cls.p1.partition, data2='bacon')
def test_all_size_tiered_options(self):
class AllSizeTieredOptionsModel(Model):
__compaction__ = SizeTieredCompactionStrategy
__compaction_bucket_low__ = .3
__compaction_bucket_high__ = 2
__compaction_min_threshold__ = 2
__compaction_max_threshold__ = 64
__compaction_tombstone_compaction_interval__ = 86400
cid = columns.UUID(primary_key=True)
name = columns.Text()
drop_table(AllSizeTieredOptionsModel)
sync_table(AllSizeTieredOptionsModel)
options = get_table_settings(AllSizeTieredOptionsModel).options['compaction_strategy_options']
options = json.loads(options)
expected = {u'min_threshold': u'2',
u'bucket_low': u'0.3',
u'tombstone_compaction_interval': u'86400',
u'bucket_high': u'2',
u'max_threshold': u'64'}
self.assertDictEqual(options, expected)
def test_bogus_option_update(self):
sync_table(ModelWithTableProperties)
option = 'no way will this ever be an option'
try:
ModelWithTableProperties.__options__[option] = 'what was I thinking?'
self.assertRaisesRegexp(KeyError, "Invalid table option.*%s.*" % option, sync_table, ModelWithTableProperties)
finally:
ModelWithTableProperties.__options__.pop(option, None)
def test_static_columns(self):
if PROTOCOL_VERSION < 2:
raise unittest.SkipTest("Native protocol 2+ required, currently using: {0}".format(PROTOCOL_VERSION))
class StaticModel(Model):
id = columns.Integer(primary_key=True)
c = columns.Integer(primary_key=True)
name = columns.Text(static=True)
drop_table(StaticModel)
session = get_session()
with mock.patch.object(session, "execute", wraps=session.execute) as m:
sync_table(StaticModel)
self.assertGreater(m.call_count, 0)
statement = m.call_args[0][0].query_string
self.assertIn('"name" text static', statement)
# if we sync again, we should not apply an alter w/ a static
sync_table(StaticModel)
with mock.patch.object(session, "execute", wraps=session.execute) as m2:
sync_table(StaticModel)
self.assertEqual(len(m2.call_args_list), 0)
def build_user_nodes(self):
try:
Utils.get_Connection_SNA4Slack()
sync_table(SlackArchive)
instances = SlackArchive.objects.filter(teamName=self.team_name)
for row in instances:
if row[CHANNEL_NAME] in channel_record.keys():
for user in channel_record[row[CHANNEL_NAME]]:
if self.graph.has_edge(row[SENDER_COLUMN], user):
self.graph[row[SENDER_COLUMN]][user][
EDGE_WEIGHT_LABEL] += 1
else:
self.graph.add_edge(row[SENDER_COLUMN],
user,
weight=1)
tsvalues = SlackArchive.objects.filter(timeStamp=self.timeStamp)
for row in instances:
self.graph.add_node(row[SENDER_COLUMN])
for val in tsvalues:
self.graph.nodes.add_value(val)
except Exception as e:
print "exception found during node building: " + str(e)
def test_nested_udts_inserts(self):
"""
Test for inserting collections of user types using cql engine.
test_nested_udts_inserts Constructs a model that contains a list of usertypes. It will then attempt to insert
them. The expectation is that no exception is thrown during insert. For sanity sake we also validate that our
input and output values match. This combination of model, and UT produces a syntax error in 2.5.1 due to
improper quoting around the names collection.
@since 2.6.0
@jira_ticket PYTHON-311
@expected_result No syntax exception thrown
@test_category data_types:udt
"""
class Name(UserType):
type_name__ = "header"
name = columns.Text()
value = columns.Text()
class Container(Model):
id = columns.UUID(primary_key=True, default=uuid4)
names = columns.List(columns.UserDefinedType(Name))
# Construct the objects and insert them
names = []
for i in range(0, 10):
names.append(Name(name="name{0}".format(i), value="value{0}".format(i)))
# Create table, insert data
sync_table(Container)
Container.create(id=UUID('FE2B4360-28C6-11E2-81C1-0800200C9A66'), names=names)
# Validate input and output matches
self.assertEqual(1, Container.objects.count())
names_output = Container.objects().first().names
self.assertEqual(names_output, names)
def sync_db(db):
keyspaces = [
'book_keyspace',
]
conns = ['node_0', 'node_1']
# registers your connections
# ...
# create all keyspaces on all connections
for ks in keyspaces:
management.create_keyspace_simple(name=ks,
replication_factor=2,
durable_writes=True,
connections=conns)
# define your Automobile model
# ...
# sync your models
management.sync_table(db, keyspaces=keyspaces, connections=conns)
def test_sync_warnings(self):
"""
Test to insure when inconsistent changes are made to a table, or type as part of a sync call that the proper logging messages are surfaced
@since 3.2
@jira_ticket PYTHON-260
@expected_result warnings are logged
@test_category object_mapper
"""
mock_handler = MockLoggingHandler()
logger = logging.getLogger(management.__name__)
logger.addHandler(mock_handler)
sync_table(BaseInconsistent)
sync_table(ChangedInconsistent)
self.assertTrue('differing from the model type' in mock_handler.messages.get('warning')[0])
if CASSANDRA_VERSION >= Version('2.1'):
sync_type(DEFAULT_KEYSPACE, BaseInconsistentType)
mock_handler.reset()
sync_type(DEFAULT_KEYSPACE, ChangedInconsistentType)
self.assertTrue('differing from the model user type' in mock_handler.messages.get('warning')[0])
logger.removeHandler(mock_handler)
def test_all_size_tiered_options(self):
class AllSizeTieredOptionsModel(Model):
__options__ = {
'compaction': {
'class':
'org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy',
'bucket_low': '.3',
'bucket_high': '2',
'min_threshold': '2',
'max_threshold': '64',
'tombstone_compaction_interval': '86400'
}
}
cid = columns.UUID(primary_key=True)
name = columns.Text()
drop_table(AllSizeTieredOptionsModel)
sync_table(AllSizeTieredOptionsModel)
table_meta = _get_table_metadata(AllSizeTieredOptionsModel)
self._verify_options(table_meta, AllSizeTieredOptionsModel.__options__)
def setUpClass(cls):
super(BaseColumnIOTest, cls).setUpClass()
# if the test column hasn't been defined, bail out
if not cls.column:
return
# create a table with the given column
class IOTestModel(Model):
table_name = cls.column.db_type + "_io_test_model_{0}".format(
uuid4().hex[:8])
pkey = cls.column(primary_key=True)
data = cls.column()
cls._generated_model = IOTestModel
sync_table(cls._generated_model)
# tupleify the tested values
if not isinstance(cls.pkey_val, tuple):
cls.pkey_val = cls.pkey_val,
if not isinstance(cls.data_val, tuple):
cls.data_val = cls.data_val,
def test_is_not_null_execution(self):
"""
Verify that CQL statements have correct syntax when executed
If we wanted them to return something meaningful and not a InvalidRequest
we'd have to create an index in search for the column we are using
IsNotNull
@since 2.5
@jira_ticket PYTHON-968
@expected_result InvalidRequest is arisen
@test_category cqlengine
"""
sync_table(TestQueryUpdateModel)
self.addCleanup(drop_table, TestQueryUpdateModel)
# Raises InvalidRequest instead of dse.protocol.SyntaxException
with self.assertRaises(InvalidRequest):
list(TestQueryUpdateModel.filter(IsNotNull("text")))
with self.assertRaises(InvalidRequest):
list(TestQueryUpdateModel.filter(IsNotNull("text"), partition=uuid4()))
def build_nodes(self):
Utils.get_Connection_SNA4Slack()
sync_table(SlackArchive)
# instances = csv.DictReader(csvfile)
instances = SlackArchive.objects.filter(teamName=self.team_name)
memo = set()
node_id = 0
for row in instances:
self.graph.add_node(row[CHANNEL_NAME])
self.graph.add_edge(self.team_name, row[CHANNEL_NAME],
weight=1)
if (row[CHANNEL_NAME], row[SENDER_COLUMN]) not in memo:
self.graph.add_node(node_id)
self.graph.node[node_id]["name"] = row[SENDER_COLUMN]
memo.add((row[CHANNEL_NAME], row[SENDER_COLUMN]))
self.graph.add_edge(row[CHANNEL_NAME], node_id,
weight=1)
self.graph.node[node_id][USER_PROFILE_PIC] = row[
USER_PROFILE_PIC]
node_id += 1
def test_can_insert_nested_udts(self):
class Depth_0(UserType):
age = columns.Integer()
name = columns.Text()
class Depth_1(UserType):
value = columns.UserDefinedType(Depth_0)
class Depth_2(UserType):
value = columns.UserDefinedType(Depth_1)
class Depth_3(UserType):
value = columns.UserDefinedType(Depth_2)
class DepthModel(Model):
id = columns.Integer(primary_key=True)
v_0 = columns.UserDefinedType(Depth_0)
v_1 = columns.UserDefinedType(Depth_1)
v_2 = columns.UserDefinedType(Depth_2)
v_3 = columns.UserDefinedType(Depth_3)
sync_table(DepthModel)
udts = [Depth_0(age=42, name="John")]
udts.append(Depth_1(value=udts[0]))
udts.append(Depth_2(value=udts[1]))
udts.append(Depth_3(value=udts[2]))
DepthModel.create(id=0,
v_0=udts[0],
v_1=udts[1],
v_2=udts[2],
v_3=udts[3])
output = DepthModel.objects().first()
self.assertEqual(udts[0], output.v_0)
self.assertEqual(udts[1], output.v_1)
self.assertEqual(udts[2], output.v_2)
self.assertEqual(udts[3], output.v_3)
def load_csv():
# create the table using the FlightModel
sync_table(FlightModel)
# load the data from a csv
df = pd.read_csv("data/flights_from_pg.csv", header=None)
df.columns = [
'id', 'year', 'day_of_month', 'fl_date', 'airline_id', 'carrier',
'fl_num', 'origin_airport_id', 'origin', 'origin_city_name',
'origin_state_abr', 'dest', 'dest_city_name', 'dest_state_abr',
'dep_time', 'arr_time', 'actual_elapsed_time', 'air_time', 'distance'
]
# MUNGE: dep_time and arr_time have 2400 values - change those to 0000
df.dep_time[df.dep_time == 2400] = 0
df.arr_time[df.arr_time == 2400] = 0
# add the date parts to the departure and arrival times
padtime = lambda x: "%04d" % x
df.dep_time = df.fl_date + " " + df.dep_time.apply(padtime)
df.arr_time = df.fl_date + " " + df.arr_time.apply(padtime)
print(df.ix[1])
#(1048576, 1)
# convert all the timestamp types to pandas datetimes
df.fl_date = pd.to_datetime(df.fl_date)
df.dep_time = pd.to_datetime(df.dep_time)
df.arr_time = pd.to_datetime(df.arr_time)
print(df.shape)
df.apply(lambda r: FlightModel.create(**r.to_dict()), axis=1)
# cqlsh> select count(*) from flughafen.flights;
# count
# ---------
# 1048576
a_airports = len(
list(filter(lambda x: x[0] == 'A', list(df.origin.unique()))))
print("Number of airport codes starting with A: %d" % a_airports)
def __init__(self):
sync_table(User) # create users_info table if not exits
sync_table(
PositionLatent) # create users_position_latent table if not exits
sync_table(PositionRaw) # create users_position_raw table if not exits
typer.secho("âž² [Database initialized successfully]",
fg=typer.colors.WHITE,
bold=True)
def sync_tables(keyspace):
from cassandra.cqlengine import management
management.sync_table(model=Entities, keyspaces=keyspace)
management.sync_table(model=Environments, keyspaces=keyspace)
management.sync_table(model=Devices, keyspaces=keyspace)
def setup_storage(settings=None):
"""Create cassandra models."""
from caliopen_storage.core import core_registry
# Make discovery happen
from caliopen_main.user.core import User
from caliopen_main.contact.objects.contact import Contact
from caliopen_main.message.objects.message import Message
from caliopen_main.common.objects.tag import ResourceTag
from caliopen_main.device.core import Device
from caliopen_main.notification.core import Notification, NotificationTtl
from cassandra.cqlengine.management import sync_table, \
create_keyspace_simple
keyspace = Configuration('global').get('cassandra.keyspace')
if not keyspace:
raise Exception('Configuration missing for cassandra keyspace')
# XXX tofix : define strategy and replication_factor in configuration
create_keyspace_simple(keyspace, 1)
for name, kls in core_registry.items():
log.info('Creating cassandra model %s' % name)
if hasattr(kls._model_class, 'pk'):
# XXX find a better way to detect model from udt
sync_table(kls._model_class)
def test_alter_options(self):
class AlterTable(Model):
__options__ = {
'compaction': {
'class':
'org.apache.cassandra.db.compaction.LeveledCompactionStrategy',
'sstable_size_in_mb': '64'
}
}
user_id = columns.UUID(primary_key=True)
name = columns.Text()
drop_table(AlterTable)
sync_table(AlterTable)
table_meta = _get_table_metadata(AlterTable)
self.assertRegexpMatches(table_meta.export_as_string(),
".*'sstable_size_in_mb': '64'.*")
AlterTable.__options__['compaction']['sstable_size_in_mb'] = '128'
sync_table(AlterTable)
table_meta = _get_table_metadata(AlterTable)
self.assertRegexpMatches(table_meta.export_as_string(),
".*'sstable_size_in_mb': '128'.*")
def test_compaction_not_altered_without_changes_sizetiered(self):
class SizeTieredCompactionChangesDetectionTest(Model):
__options__ = {
'compaction': {
'class':
'org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy',
'bucket_high': '20',
'bucket_low': '10',
'max_threshold': '200',
'min_threshold': '100',
'min_sstable_size': '1000',
'tombstone_threshold': '0.125',
'tombstone_compaction_interval': '3600'
}
}
pk = columns.Integer(primary_key=True)
drop_table(SizeTieredCompactionChangesDetectionTest)
sync_table(SizeTieredCompactionChangesDetectionTest)
self.assertFalse(
_update_options(SizeTieredCompactionChangesDetectionTest))
def test_can_update_udts_with_nones(self):
class User(UserType):
age = columns.Integer()
name = columns.Text()
class UserModel(Model):
id = columns.Integer(primary_key=True)
info = columns.UserDefinedType(User)
sync_table(UserModel)
user = User(age=42, name="John")
created_user = UserModel.create(id=0, info=user)
john_info = UserModel.objects().first().info
self.assertEqual(42, john_info.age)
self.assertEqual("John", john_info.name)
created_user.info = None
created_user.update()
john_info = UserModel.objects().first().info
self.assertIsNone(john_info)
def create_event_data_table():
try:
setup(['127.0.0.1'], 'sparkifydb', retry_connect=True)
sync_table(model.table.EventDataModel)
sync_table(model.table.UserAndSessionModel)
sync_table(model.table.UserAndSongModel)
except Exception as e:
print(e)
exit(1)
def setUpClass(cls):
super(TestUnindexedInheritanceQuery, cls).setUpClass()
management.sync_table(UnindexedInherit1)
management.sync_table(UnindexedInherit2)
management.sync_table(UnindexedInherit3)
cls.p1 = UnindexedInherit1.create(data1='pickle')
cls.p2 = UnindexedInherit2.create(partition=cls.p1.partition, data2='bacon')
cls.p3 = UnindexedInherit3.create(partition=cls.p1.partition, data3='turkey')
def test_can_insert_model_with_all_protocol_v4_column_types(self):
"""
Test for inserting all protocol v4 column types into a Model
test_can_insert_model_with_all_protocol_v4_column_types tests that each cqlengine protocol v4 column type can be
inserted into a Model. It first creates a Model that has each cqlengine protocol v4 column type. It then creates
a Model instance where all the fields have corresponding data, which performs the insert into the Cassandra table.
Finally, it verifies that each column read from the Model from Cassandra is the same as the input parameters.
@since 2.6.0
@jira_ticket PYTHON-245
@expected_result The Model is inserted with each protocol v4 column type, and the resulting read yields proper data for each column.
@test_category data_types:primitive
"""
if PROTOCOL_VERSION < 4:
raise unittest.SkipTest("Protocol v4 datatypes require native protocol 4+, currently using: {0}".format(PROTOCOL_VERSION))
class v4DatatypesModel(Model):
id = columns.Integer(primary_key=True)
a = columns.Date()
b = columns.SmallInt()
c = columns.Time()
d = columns.TinyInt()
sync_table(v4DatatypesModel)
input = [Date(date(1970, 1, 1)), 32523, Time(time(16, 47, 25, 7)), 123]
v4DatatypesModel.create(id=0, a=date(1970, 1, 1), b=32523, c=time(16, 47, 25, 7), d=123)
self.assertEqual(1, v4DatatypesModel.objects.count())
output = v4DatatypesModel.objects().first()
for i, i_char in enumerate(range(ord('a'), ord('a') + 3)):
self.assertEqual(input[i], output[chr(i_char)])
def test_connection_session_switch(self):
"""
Test to ensure that when the default keyspace is changed in a session and that session,
is set in the connection class, that the new defaul keyspace is honored.
@since 3.1
@jira_ticket PYTHON-486
@expected_result CQLENGINE adopts whatever keyspace is passed in vai the set_session method as default
@test_category object_mapper
"""
connection.set_session(self.session1)
sync_table(TestConnectModel)
TCM1 = TestConnectModel.create(id=1, keyspace=self.keyspace1)
connection.set_session(self.session2)
sync_table(TestConnectModel)
TCM2 = TestConnectModel.create(id=1, keyspace=self.keyspace2)
connection.set_session(self.session1)
self.assertEqual(1, TestConnectModel.objects.count())
self.assertEqual(TestConnectModel.objects.first(), TCM1)
connection.set_session(self.session2)
self.assertEqual(1, TestConnectModel.objects.count())
self.assertEqual(TestConnectModel.objects.first(), TCM2)
def test_lots_of_queries(self):
import resource
import objgraph
class LoadTest(Model):
k = columns.Integer(primary_key=True)
v = columns.Integer()
sync_table(LoadTest)
gc.collect()
objgraph.show_most_common_types()
print("Starting...")
for i in range(1000000):
if i % 25000 == 0:
# print memory statistic
print("Memory usage: %s" % (resource.getrusage(resource.RUSAGE_SELF).ru_maxrss))
LoadTest.create(k=i, v=i)
objgraph.show_most_common_types()
raise Exception("you shouldn't be here")
def setUp(self):
class MyModel(Model):
id = columns.Text(max_length=63,
primary_key=True,
default=lambda: str(uuid.uuid4()))
value = columns.Text(max_length=63)
self.model = MyModel
connection.setup(['127.0.0.1'], self.keyspace)
create_keyspace(self.keyspace,
strategy_class='SimpleStrategy',
replication_factor=1)
sync_table(MyModel)
class MyManager(CQLManager):
model = self.model
fields = (
'id',
'value',
)
create_fields = ('value', )
update_fields = ('value', )
self.manager_class = MyManager
def init_database(cluster, keyspace, db_user, db_pass):
try:
auth_provider = PlainTextAuthProvider(username=db_user,
password=db_pass)
connection.setup(cluster, keyspace, auth_provider=auth_provider)
sync_table(User)
sync_table(Tweet)
sync_table(Relation)
except Exception:
raise
def connect_cassandra(role):
auth_provider = PlainTextAuthProvider(username=str(role),
password=str(role))
connection.setup(['127.0.0.1'],
"sensors",
auth_provider=auth_provider,
protocol_version=3)
sync_table(Users)
sync_table(TemperatureSensor)
sync_table(PressureSensor)
return True
def setUpClass(cls):
super(TestUnindexedPolymorphicQuery, cls).setUpClass()
management.sync_table(UnindexedPoly1)
management.sync_table(UnindexedPoly2)
management.sync_table(UnindexedPoly3)
cls.p1 = UnindexedPoly1.create(data1='pickle')
cls.p2 = UnindexedPoly2.create(partition=cls.p1.partition,
data2='bacon')
cls.p3 = UnindexedPoly3.create(partition=cls.p1.partition,
data3='turkey')
def test_add_column(self):
sync_table(FirstModel)
fields = get_fields(FirstModel)
# this should contain the second key
self.assertEqual(len(fields), 2)
# get schema
sync_table(SecondModel)
fields = get_fields(FirstModel)
self.assertEqual(len(fields), 3)
sync_table(ThirdModel)
fields = get_fields(FirstModel)
self.assertEqual(len(fields), 4)
sync_table(FourthModel)
fields = get_fields(FirstModel)
self.assertEqual(len(fields), 4)
def test_add_column(self):
sync_table(FirstModel)
fields = _get_non_pk_field_names(_get_table_metadata(FirstModel))
# this should contain the second key
self.assertEqual(len(fields), 2)
# get schema
sync_table(SecondModel)
fields = _get_non_pk_field_names(_get_table_metadata(FirstModel))
self.assertEqual(len(fields), 3)
sync_table(ThirdModel)
fields = _get_non_pk_field_names(_get_table_metadata(FirstModel))
self.assertEqual(len(fields), 4)
sync_table(FourthModel)
fields = _get_non_pk_field_names(_get_table_metadata(FirstModel))
self.assertEqual(len(fields), 4)
def main(argv):
game = None
if argv[0] == 'csgo':
game = Game.CSGO
elif argv[0] == 'dota2':
game = Game.DOTA2
item_names = steam_market.get_item_names(game)
print('Number of items is %d' % len(item_names))
set_cassandra_hosts(['127.0.0.1'])
set_cassandra_keyspace('scraper')
connection.setup(cassandra_hosts, cassandra_keyspace_name, protocol_version=3)
sync_table(Item)
sync_table(ItemCsgo)
sync_table(ItemDota2)
timer = time.time()
segment = int(len(item_names) / 8)
print('Segment size is %d' % segment)
workers = []
for i in range(0, 8):
if i < 7:
workers.append(Worker('worker ' + str(i + 1), Game.CSGO, item_names[i * segment:(i+1) * segment]))
else:
workers.append(Worker('worker 8', Game.CSGO, item_names[i * segment:len(item_names)]))
for i in range(0, 8):
workers[i].start()
for i in range(0, 8):
workers[i].join()
print('Number of items %d' % ItemCsgo.objects.count())
timer = time.time() - timer
print(timer)
for i in range(0, 60):
print('Sleeping %d seconds' % i)
time.sleep(1)
workers = []
market.config.CHECKED_ITEMS = 0
timer_history = time.time()
for i in range(0, 8):
if i < 7:
workers.append(HistoryWorker('worker ' + str(i + 1), Game.CSGO, item_names[i * segment:(i + 1) * segment]))
else:
workers.append(HistoryWorker('worker 8', Game.CSGO, item_names[i * segment:len(item_names)]))
for i in range(0, 8):
workers[i].start()
for i in range(0, 8):
workers[i].join()
timer_history = time.time() - timer_history
print(timer_history)
async def setup_cassandra(app, loop):
logger.warning('Initializing Database')
cluster = Cluster(['cassandra_seed_node'])
session = cluster.connect()
session.execute(
"""CREATE KEYSPACE IF NOT EXISTS soda WITH replication = {'class': 'SimpleStrategy', 'replication_factor': '1'} AND durable_writes = true;"""
)
session.execute('USE soda')
register_connection(str(session), session=session)
set_default_connection(str(session))
sync_table(User, keyspaces=['soda'])
sync_table(Record, keyspaces=['soda'])
sync_table(Project, keyspaces=['soda'])
aiosession(session)
app.db = session
return app
