def __new__(cls, name, bases, attrs):
field_dict = OrderedDict()
field_defs = [(k, v) for k, v in attrs.items()
if isinstance(v, columns.Column)]
field_defs = sorted(field_defs, key=lambda x: x[1].position)
def _transform_column(field_name, field_obj):
field_dict[field_name] = field_obj
field_obj.set_column_name(field_name)
attrs[field_name] = models.ColumnDescriptor(field_obj)
# transform field definitions
for k, v in field_defs:
# don't allow a field with the same name as a built-in attribute or method
if k in BaseUserType.__dict__:
raise UserTypeDefinitionException(
"field '{0}' conflicts with built-in attribute/method".
format(k))
_transform_column(k, v)
# create db_name -> model name map for loading
db_map = {}
for field_name, field in field_dict.items():
db_map[field.db_field_name] = field_name
attrs['_fields'] = field_dict
attrs['_db_map'] = db_map
klass = super(UserTypeMetaClass, cls).__new__(cls, name, bases, attrs)
return klass
def __init__(self,
keyspace_metadata,
name,
partition_key=None,
clustering_key=None,
columns=None,
triggers=None,
options=None):
self.keyspace = keyspace_metadata
self.name = name
self.partition_key = [] if partition_key is None else partition_key
self.clustering_key = [] if clustering_key is None else clustering_key
self.columns = OrderedDict() if columns is None else columns
self.options = options
self.comparator = None
self.triggers = OrderedDict() if triggers is None else triggers
def test_map_collection(self):
vals = OrderedDict()
vals['a'] = 'a'
vals['b'] = 'b'
vals['c'] = 'c'
result = bind_params("%s", (vals, ), Encoder())
self.assertEqual(result, "{'a': 'a', 'b': 'b', 'c': 'c'}")
def ordered_dict_factory(colnames, rows):
"""
Like :meth:`~cassandra.query.dict_factory`, but returns each row as an OrderedDict,
so the order of the columns is preserved.
.. versionchanged:: 2.0.0
moved from ``cassandra.decoder`` to ``cassandra.query``
"""
return [OrderedDict(zip(colnames, row)) for row in rows]
def _get_partition_keys(self):
try:
table_meta = get_cluster(self._get_connection()).metadata.keyspaces[self.keyspace].tables[self.name]
self.__partition_keys = OrderedDict(
(pk.name, Column(primary_key=True, partition_key=True, db_field=pk.name)) for pk in
table_meta.partition_key)
except Exception as e:
raise CQLEngineException("Failed inspecting partition keys for {0}."
"Ensure cqlengine is connected before attempting this with NamedTable.".format(
self.column_family_name()))
def deserialize_safe(cls, byts):
subkeytype, subvaltype = cls.subtypes
numelements = uint16_unpack(byts[:2])
p = 2
themap = OrderedDict()
for n in xrange(numelements):
key_len = uint16_unpack(byts[p:p + 2])
p += 2
keybytes = byts[p:p + key_len]
p += key_len
val_len = uint16_unpack(byts[p:p + 2])
p += 2
valbytes = byts[p:p + val_len]
p += val_len
key = subkeytype.from_binary(keybytes)
val = subvaltype.from_binary(valbytes)
themap[key] = val
return themap
def sync_table(cls, model, keyspaces=[], connections=[]):
for connection in connections:
assert connection in cls.connections
for keyspace in keyspaces:
keyspace_metadata = MockCQLEngineContext.get_cluster(
connection).metadata.keyspaces.get(keyspace)
assert keyspace_metadata is not None
primary_keys = []
column_keys = []
for key, value in model._columns.items():
if value.partition_key or value.primary_key:
primary_keys.append(key)
else:
column_keys.append(key)
columns = OrderedDict()
partition_keys = []
clustering_keys = []
for key in primary_keys + column_keys:
value = model._columns[key]
meta_data = ColumnMetadata(
None,
key,
value.db_type,
is_reversed=(value.clustering_order == 'DESC'))
columns[key] = meta_data
if value.partition_key:
partition_keys.append(meta_data)
if value.primary_key:
clustering_keys.append(meta_data)
MockCluster.database[keyspace][
model._raw_column_family_name()] = defaultdict(lambda: {})
keyspace_metadata.tables[
model._raw_column_family_name()] = TableMetadataV3(
keyspace,
model._raw_column_family_name(),
columns=columns,
partition_key=partition_keys,
clustering_key=clustering_keys,
)
def deserialize_safe(cls, byts, protocol_version):
subkeytype, subvaltype = cls.subtypes
if protocol_version >= 3:
unpack = int32_unpack
length = 4
else:
unpack = uint16_unpack
length = 2
numelements = unpack(byts[:length])
p = length
themap = OrderedDict()
for _ in range(numelements):
key_len = unpack(byts[p:p + length])
p += length
keybytes = byts[p:p + key_len]
p += key_len
val_len = unpack(byts[p:p + length])
p += length
valbytes = byts[p:p + val_len]
p += val_len
key = subkeytype.from_binary(keybytes, protocol_version)
val = subvaltype.from_binary(valbytes, protocol_version)
themap[key] = val
return themap
from cassandra.protocol import (ReadyMessage, AuthenticateMessage, OptionsMessage,
StartupMessage, ErrorMessage, CredentialsMessage,
QueryMessage, ResultMessage, ProtocolHandler,
InvalidRequestException, SupportedMessage,
AuthResponseMessage, AuthChallengeMessage,
AuthSuccessMessage, ProtocolException,
MAX_SUPPORTED_VERSION, RegisterMessage)
from cassandra.util import OrderedDict
log = logging.getLogger(__name__)
# We use an ordered dictionary and specifically add lz4 before
# snappy so that lz4 will be preferred. Changing the order of this
# will change the compression preferences for the driver.
locally_supported_compressions = OrderedDict()
try:
import lz4
except ImportError:
pass
else:
# Cassandra writes the uncompressed message length in big endian order,
# but the lz4 lib requires little endian order, so we wrap these
# functions to handle that
def lz4_compress(byts):
# write length in big-endian instead of little-endian
return int32_pack(len(byts)) + lz4.compress(byts)[4:]
def __new__(cls, name, bases, attrs):
# move column definitions into columns dict
# and set default column names
column_dict = OrderedDict()
primary_keys = OrderedDict()
pk_name = None
# get inherited properties
inherited_columns = OrderedDict()
for base in bases:
for k, v in getattr(base, '_defined_columns', {}).items():
inherited_columns.setdefault(k, v)
# short circuit __abstract__ inheritance
is_abstract = attrs['__abstract__'] = attrs.get('__abstract__', False)
# short circuit __discriminator_value__ inheritance
attrs['__discriminator_value__'] = attrs.get('__discriminator_value__')
# TODO __default__ttl__ should be removed in the next major release
options = attrs.get('__options__') or {}
attrs['__default_ttl__'] = options.get('default_time_to_live')
column_definitions = [(k, v) for k, v in attrs.items() if isinstance(v, columns.Column)]
column_definitions = sorted(column_definitions, key=lambda x: x[1].position)
is_polymorphic_base = any([c[1].discriminator_column for c in column_definitions])
column_definitions = [x for x in inherited_columns.items()] + column_definitions
discriminator_columns = [c for c in column_definitions if c[1].discriminator_column]
is_polymorphic = len(discriminator_columns) > 0
if len(discriminator_columns) > 1:
raise ModelDefinitionException('only one discriminator_column can be defined in a model, {0} found'.format(len(discriminator_columns)))
if attrs['__discriminator_value__'] and not is_polymorphic:
raise ModelDefinitionException('__discriminator_value__ specified, but no base columns defined with discriminator_column=True')
discriminator_column_name, discriminator_column = discriminator_columns[0] if discriminator_columns else (None, None)
if isinstance(discriminator_column, (columns.BaseContainerColumn, columns.Counter)):
raise ModelDefinitionException('counter and container columns cannot be used as discriminator columns')
# find polymorphic base class
polymorphic_base = None
if is_polymorphic and not is_polymorphic_base:
def _get_polymorphic_base(bases):
for base in bases:
if getattr(base, '_is_polymorphic_base', False):
return base
klass = _get_polymorphic_base(base.__bases__)
if klass:
return klass
polymorphic_base = _get_polymorphic_base(bases)
defined_columns = OrderedDict(column_definitions)
# check for primary key
if not is_abstract and not any([v.primary_key for k, v in column_definitions]):
raise ModelDefinitionException("At least 1 primary key is required.")
counter_columns = [c for c in defined_columns.values() if isinstance(c, columns.Counter)]
data_columns = [c for c in defined_columns.values() if not c.primary_key and not isinstance(c, columns.Counter)]
if counter_columns and data_columns:
raise ModelDefinitionException('counter models may not have data columns')
has_partition_keys = any(v.partition_key for (k, v) in column_definitions)
def _transform_column(col_name, col_obj):
column_dict[col_name] = col_obj
if col_obj.primary_key:
primary_keys[col_name] = col_obj
col_obj.set_column_name(col_name)
# set properties
attrs[col_name] = ColumnDescriptor(col_obj)
partition_key_index = 0
# transform column definitions
for k, v in column_definitions:
# don't allow a column with the same name as a built-in attribute or method
if k in BaseModel.__dict__:
raise ModelDefinitionException("column '{0}' conflicts with built-in attribute/method".format(k))
# counter column primary keys are not allowed
if (v.primary_key or v.partition_key) and isinstance(v, columns.Counter):
raise ModelDefinitionException('counter columns cannot be used as primary keys')
# this will mark the first primary key column as a partition
# key, if one hasn't been set already
if not has_partition_keys and v.primary_key:
v.partition_key = True
has_partition_keys = True
if v.partition_key:
v._partition_key_index = partition_key_index
partition_key_index += 1
overriding = column_dict.get(k)
if overriding:
v.position = overriding.position
v.partition_key = overriding.partition_key
v._partition_key_index = overriding._partition_key_index
_transform_column(k, v)
partition_keys = OrderedDict(k for k in primary_keys.items() if k[1].partition_key)
clustering_keys = OrderedDict(k for k in primary_keys.items() if not k[1].partition_key)
if attrs.get('__compute_routing_key__', True):
key_cols = [c for c in partition_keys.values()]
partition_key_index = dict((col.db_field_name, col._partition_key_index) for col in key_cols)
key_cql_types = [c.cql_type for c in key_cols]
key_serializer = staticmethod(lambda parts, proto_version: [t.to_binary(p, proto_version) for t, p in zip(key_cql_types, parts)])
else:
partition_key_index = {}
key_serializer = staticmethod(lambda parts, proto_version: None)
# setup partition key shortcut
if len(partition_keys) == 0:
if not is_abstract:
raise ModelException("at least one partition key must be defined")
if len(partition_keys) == 1:
pk_name = [x for x in partition_keys.keys()][0]
attrs['pk'] = attrs[pk_name]
else:
# composite partition key case, get/set a tuple of values
_get = lambda self: tuple(self._values[c].getval() for c in partition_keys.keys())
_set = lambda self, val: tuple(self._values[c].setval(v) for (c, v) in zip(partition_keys.keys(), val))
attrs['pk'] = property(_get, _set)
# some validation
col_names = set()
for v in column_dict.values():
# check for duplicate column names
if v.db_field_name in col_names:
raise ModelException("{0} defines the column '{1}' more than once".format(name, v.db_field_name))
if v.clustering_order and not (v.primary_key and not v.partition_key):
raise ModelException("clustering_order may be specified only for clustering primary keys")
if v.clustering_order and v.clustering_order.lower() not in ('asc', 'desc'):
raise ModelException("invalid clustering order '{0}' for column '{1}'".format(repr(v.clustering_order), v.db_field_name))
col_names.add(v.db_field_name)
# create db_name -> model name map for loading
db_map = {}
for col_name, field in column_dict.items():
db_field = field.db_field_name
if db_field != col_name:
db_map[db_field] = col_name
# add management members to the class
attrs['_columns'] = column_dict
attrs['_primary_keys'] = primary_keys
attrs['_defined_columns'] = defined_columns
# maps the database field to the models key
attrs['_db_map'] = db_map
attrs['_pk_name'] = pk_name
attrs['_dynamic_columns'] = {}
attrs['_partition_keys'] = partition_keys
attrs['_partition_key_index'] = partition_key_index
attrs['_key_serializer'] = key_serializer
attrs['_clustering_keys'] = clustering_keys
attrs['_has_counter'] = len(counter_columns) > 0
# add polymorphic management attributes
attrs['_is_polymorphic_base'] = is_polymorphic_base
attrs['_is_polymorphic'] = is_polymorphic
attrs['_polymorphic_base'] = polymorphic_base
attrs['_discriminator_column'] = discriminator_column
attrs['_discriminator_column_name'] = discriminator_column_name
attrs['_discriminator_map'] = {} if is_polymorphic_base else None
# setup class exceptions
DoesNotExistBase = None
for base in bases:
DoesNotExistBase = getattr(base, 'DoesNotExist', None)
if DoesNotExistBase is not None:
break
DoesNotExistBase = DoesNotExistBase or attrs.pop('DoesNotExist', BaseModel.DoesNotExist)
attrs['DoesNotExist'] = type('DoesNotExist', (DoesNotExistBase,), {})
MultipleObjectsReturnedBase = None
for base in bases:
MultipleObjectsReturnedBase = getattr(base, 'MultipleObjectsReturned', None)
if MultipleObjectsReturnedBase is not None:
break
MultipleObjectsReturnedBase = MultipleObjectsReturnedBase or attrs.pop('MultipleObjectsReturned', BaseModel.MultipleObjectsReturned)
attrs['MultipleObjectsReturned'] = type('MultipleObjectsReturned', (MultipleObjectsReturnedBase,), {})
# create the class and add a QuerySet to it
klass = super(ModelMetaClass, cls).__new__(cls, name, bases, attrs)
udts = []
for col in column_dict.values():
columns.resolve_udts(col, udts)
for user_type in set(udts):
user_type.register_for_keyspace(klass._get_keyspace())
return klass
def test_empty_strings_and_nones(self):
c = Cluster(protocol_version=PROTOCOL_VERSION)
s = c.connect()
s.execute("""
CREATE KEYSPACE test_empty_strings_and_nones
WITH replication = { 'class' : 'SimpleStrategy', 'replication_factor': '1'}
""")
s.set_keyspace("test_empty_strings_and_nones")
s.execute(self.create_type_table)
s.execute("INSERT INTO mytable (a, b) VALUES ('a', 'b')")
s.row_factory = dict_factory
results = s.execute("""
SELECT c, d, f, g, h, i, j, k, l, m, n, o, p, q, r, s, t FROM mytable
""")
self.assertTrue(all(x is None for x in results[0].values()))
prepared = s.prepare("""
SELECT c, d, f, g, h, i, j, k, l, m, n, o, p, q, r, s, t FROM mytable
""")
results = s.execute(prepared.bind(()))
self.assertTrue(all(x is None for x in results[0].values()))
# insert empty strings for string-like fields and fetch them
s.execute(
"INSERT INTO mytable (a, b, c, o, s, l, n) VALUES ('a', 'b', %s, %s, %s, %s, %s)",
('', '', '', [''], {
'': 3
}))
self.assertEqual(
{
'c': '',
'o': '',
's': '',
'l': ('', ),
'n': OrderedDict({'': 3})
},
s.execute(
"SELECT c, o, s, l, n FROM mytable WHERE a='a' AND b='b'")[0])
self.assertEqual(
{
'c': '',
'o': '',
's': '',
'l': ('', ),
'n': OrderedDict({'': 3})
},
s.execute(
s.prepare(
"SELECT c, o, s, l, n FROM mytable WHERE a='a' AND b='b'"),
[])[0])
# non-string types shouldn't accept empty strings
for col in ('d', 'f', 'g', 'h', 'i', 'k', 'l', 'm', 'n', 'q', 'r',
't'):
query = "INSERT INTO mytable (a, b, %s) VALUES ('a', 'b', %%s)" % (
col, )
try:
s.execute(query, [''])
except InvalidRequest:
pass
else:
self.fail("Expected an InvalidRequest error when inserting an "
"emptry string for column %s" % (col, ))
prepared = s.prepare(
"INSERT INTO mytable (a, b, %s) VALUES ('a', 'b', ?)" %
(col, ))
try:
s.execute(prepared, [''])
except TypeError:
pass
else:
self.fail(
"Expected an InvalidRequest error when inserting an "
"emptry string for column %s with a prepared statement" %
(col, ))
# insert values for all columns
values = [
'a', 'b', 'a', 1, True,
Decimal('1.0'), 0.1, 0.1, "1.2.3.4", 1, ['a'],
set([1]), {
'a': 1
}, 'a',
datetime.now(),
uuid4(),
uuid1(), 'a', 1
]
s.execute(
"""
INSERT INTO mytable (a, b, c, d, f, g, h, i, j, k, l, m, n, o, p, q, r, s, t)
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
""", values)
# then insert None, which should null them out
null_values = values[:2] + ([None] * (len(values) - 2))
s.execute(
"""
INSERT INTO mytable (a, b, c, d, f, g, h, i, j, k, l, m, n, o, p, q, r, s, t)
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
""", null_values)
results = s.execute("""
SELECT c, d, f, g, h, i, j, k, l, m, n, o, p, q, r, s, t FROM mytable
""")
self.assertEqual(
[], [(name, val)
for (name, val) in results[0].items() if val is not None])
prepared = s.prepare("""
SELECT c, d, f, g, h, i, j, k, l, m, n, o, p, q, r, s, t FROM mytable
""")
results = s.execute(prepared.bind(()))
self.assertEqual(
[], [(name, val)
for (name, val) in results[0].items() if val is not None])
# do the same thing again, but use a prepared statement to insert the nulls
s.execute(
"""
INSERT INTO mytable (a, b, c, d, f, g, h, i, j, k, l, m, n, o, p, q, r, s, t)
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
""", values)
prepared = s.prepare("""
INSERT INTO mytable (a, b, c, d, f, g, h, i, j, k, l, m, n, o, p, q, r, s, t)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
""")
s.execute(prepared, null_values)
results = s.execute("""
SELECT c, d, f, g, h, i, j, k, l, m, n, o, p, q, r, s, t FROM mytable
""")
self.assertEqual(
[], [(name, val)
for (name, val) in results[0].items() if val is not None])
prepared = s.prepare("""
SELECT c, d, f, g, h, i, j, k, l, m, n, o, p, q, r, s, t FROM mytable
""")
results = s.execute(prepared.bind(()))
self.assertEqual(
[], [(name, val)
for (name, val) in results[0].items() if val is not None])
(b'A46\xa9', 'InetAddressType', '65.52.54.169'),
(b'*\x00\x13(\xe1\x02\xcc\xc0\x00\x00\x00\x00\x00\x00\x01"',
'InetAddressType', '2a00:1328:e102:ccc0::122'),
(b'\xe3\x81\xbe\xe3\x81\x97\xe3\x81\xa6', 'UTF8Type',
u'\u307e\u3057\u3066'),
(b'\xe3\x81\xbe\xe3\x81\x97\xe3\x81\xa6' * 1000, 'UTF8Type',
u'\u307e\u3057\u3066' * 1000),
(b'', 'UTF8Type', u''),
(b'\xff' * 16, 'UUIDType', UUID('ffffffff-ffff-ffff-ffff-ffffffffffff')),
(b'I\x15~\xfc\xef<\x9d\xe3\x16\x98\xaf\x80\x1f\xb4\x0b*', 'UUIDType',
UUID('49157efc-ef3c-9de3-1698-af801fb40b2a')),
(b'', 'UUIDType', None),
(b'', 'MapType(AsciiType, BooleanType)', None),
(b'', 'ListType(FloatType)', None),
(b'', 'SetType(LongType)', None),
(b'\x00\x00', 'MapType(DecimalType, BooleanType)', OrderedDict()),
(b'\x00\x00', 'ListType(FloatType)', []),
(b'\x00\x00', 'SetType(IntegerType)', sortedset()),
(b'\x00\x01\x00\x10\xafYC\xa3\xea<\x11\xe1\xabc\xc4,\x03"y\xf0',
'ListType(TimeUUIDType)',
[UUID(bytes=b'\xafYC\xa3\xea<\x11\xe1\xabc\xc4,\x03"y\xf0')]),
)
ordered_dict_value = OrderedDict()
ordered_dict_value[u'\u307fbob'] = 199
ordered_dict_value[u''] = -1
ordered_dict_value[u'\\'] = 0
# these following entries work for me right now, but they're dependent on
# vagaries of internal python ordering for unordered types
marshalled_value_pairs_unsafe = (
def select_from_table(self, table_name, limit=10000, **kwargs):
result = []
schema = self.schema_for_table(table_name)
partition_names = []
for column in schema.partition_key:
partition_names.append(column.name)
# Sort kwargs so they are in the same order as the columns.
ordered_kwargs = OrderedDict()
for name in self._models[table_name]._columns.keys():
for key, value in kwargs.items():
if key.split('__')[0] == name and value is not None:
ordered_kwargs[key] = value
# Convert arguments to filters
partitions = [[]]
filters = []
for key, value in ordered_kwargs.items():
key_value = key.split('__')[0]
operator = None if len(
key.split('__')) == 1 else key.split('__')[1]
if key_value in partition_names:
if operator == 'in':
original_partitions = list(partitions)
partitions = []
for _ in range(len(value)):
for element in original_partitions:
partitions.append(list(element))
for multiplier in range(len(value)):
for index in range(len(original_partitions)):
partitions[multiplier * len(original_partitions) +
index].append(value[multiplier])
else:
for partition in partitions:
partition.append(value)
filters.append(self.filter_for_argument(key, value))
if len(partitions[0]) == 0:
partitions = self.cluster.database[
self.keyspace][table_name].keys()
candidate_elements = []
for partition in partitions:
for key, row in self.cluster.database[self.keyspace][table_name][
tuple(partition)].items():
does_match = True
for filter in filters:
if not filter(row[1]):
does_match = False
break
if not does_match:
continue
candidate_elements.append((partition, key))
# To reverse, they all need to agree. This might not work with compound clustering keys, but we really
# shouldn't use those anyways
is_reversed = True
for column in schema.primary_key:
if column in schema.partition_key:
continue
if not column.is_reversed:
is_reversed = False
candidate_elements.sort(reverse=is_reversed)
for candidate in candidate_elements:
element = self.cluster.database[self.keyspace][table_name][tuple(
candidate[0])][candidate[1]]
deadline = element[0]
if deadline is None or deadline > time.time():
result.append(element[1])
limit -= 1
if limit <= 0:
return result
else:
del self.cluster.database[self.keyspace][table_name][tuple(
candidate[0])][candidate[1]]
return result
def ordered_dict_factory(colnames, rows):
return [OrderedDict(zip(colnames, row)) for row in rows]
