def __new__(cls, name, bases, attrs): field_dict = OrderedDict() field_defs = [(k.decode('utf-8'), v) for k, v in attrs.items() if isinstance(v, columns.Column)] field_defs = sorted(field_defs, key=lambda x: x[1].position) def _transform_column(field_name, field_obj): field_dict[field_name] = field_obj field_obj.set_column_name(field_name) attrs[field_name] = models.ColumnDescriptor(field_obj) # transform field definitions for k, v in field_defs: # don't allow a field with the same name as a built-in attribute or method if k in BaseUserType.__dict__: raise UserTypeDefinitionException("field '{0}' conflicts with built-in attribute/method".format(k)) _transform_column(k, v) # create db_name -> model name map for loading db_map = {} for field_name, field in field_dict.items(): db_map[field.db_field_name] = field_name attrs['_fields'] = field_dict attrs['_db_map'] = db_map klass = super(UserTypeMetaClass, cls).__new__(cls, name, bases, attrs) return klass
def __new__(cls, name, bases, attrs): field_dict = OrderedDict() field_defs = [(k, v) for k, v in attrs.items() if isinstance(v, columns.Column)] field_defs = sorted(field_defs, key=lambda x: x[1].position) def _transform_column(field_name, field_obj): field_dict[field_name] = field_obj field_obj.set_column_name(field_name) attrs[field_name] = models.ColumnDescriptor(field_obj) # transform field definitions for k, v in field_defs: # don't allow a field with the same name as a built-in attribute or method if k in BaseUserType.__dict__: raise UserTypeDefinitionException( "field '{0}' conflicts with built-in attribute/method". format(k)) _transform_column(k, v) # create db_name -> model name map for loading db_map = {} for field_name, field in field_dict.items(): db_map[field.db_field_name] = field_name attrs['_fields'] = field_dict attrs['_db_map'] = db_map klass = super(UserTypeMetaClass, cls).__new__(cls, name, bases, attrs) return klass
def __init__(self, keyspace_metadata, name, partition_key=None, clustering_key=None, columns=None, triggers=None, options=None): self.keyspace = keyspace_metadata self.name = name self.partition_key = [] if partition_key is None else partition_key self.clustering_key = [] if clustering_key is None else clustering_key self.columns = OrderedDict() if columns is None else columns self.options = options self.comparator = None self.triggers = OrderedDict() if triggers is None else triggers
def __init__(self, keyspace_metadata, name, partition_key=None, clustering_key=None, columns=None, options=None): self.keyspace = keyspace_metadata self.name = name self.partition_key = [] if partition_key is None else partition_key self.clustering_key = [] if clustering_key is None else clustering_key self.columns = OrderedDict() if columns is None else columns self.options = options self.comparator = None
def test_map_collection(self): vals = OrderedDict() vals['a'] = 'a' vals['b'] = 'b' vals['c'] = 'c' result = bind_params("%s", (vals, ), Encoder()) self.assertEqual(result, "{'a': 'a', 'b': 'b', 'c': 'c'}")
def ordered_dict_factory(colnames, rows): """ Like :meth:`~cassandra.query.dict_factory`, but returns each row as an OrderedDict, so the order of the columns is preserved. .. versionchanged:: 2.0.0 moved from ``cassandra.decoder`` to ``cassandra.query`` """ return [OrderedDict(zip(colnames, row)) for row in rows]
def _get_partition_keys(self): try: table_meta = get_cluster(self._get_connection()).metadata.keyspaces[self.keyspace].tables[self.name] self.__partition_keys = OrderedDict( (pk.name, Column(primary_key=True, partition_key=True, db_field=pk.name)) for pk in table_meta.partition_key) except Exception as e: raise CQLEngineException("Failed inspecting partition keys for {0}." "Ensure cqlengine is connected before attempting this with NamedTable.".format( self.column_family_name()))
def deserialize_safe(cls, byts): subkeytype, subvaltype = cls.subtypes numelements = uint16_unpack(byts[:2]) p = 2 themap = OrderedDict() for n in xrange(numelements): key_len = uint16_unpack(byts[p:p + 2]) p += 2 keybytes = byts[p:p + key_len] p += key_len val_len = uint16_unpack(byts[p:p + 2]) p += 2 valbytes = byts[p:p + val_len] p += val_len key = subkeytype.from_binary(keybytes) val = subvaltype.from_binary(valbytes) themap[key] = val return themap
def sync_table(cls, model, keyspaces=[], connections=[]): for connection in connections: assert connection in cls.connections for keyspace in keyspaces: keyspace_metadata = MockCQLEngineContext.get_cluster( connection).metadata.keyspaces.get(keyspace) assert keyspace_metadata is not None primary_keys = [] column_keys = [] for key, value in model._columns.items(): if value.partition_key or value.primary_key: primary_keys.append(key) else: column_keys.append(key) columns = OrderedDict() partition_keys = [] clustering_keys = [] for key in primary_keys + column_keys: value = model._columns[key] meta_data = ColumnMetadata( None, key, value.db_type, is_reversed=(value.clustering_order == 'DESC')) columns[key] = meta_data if value.partition_key: partition_keys.append(meta_data) if value.primary_key: clustering_keys.append(meta_data) MockCluster.database[keyspace][ model._raw_column_family_name()] = defaultdict(lambda: {}) keyspace_metadata.tables[ model._raw_column_family_name()] = TableMetadataV3( keyspace, model._raw_column_family_name(), columns=columns, partition_key=partition_keys, clustering_key=clustering_keys, )
def deserialize_safe(cls, byts, protocol_version): subkeytype, subvaltype = cls.subtypes if protocol_version >= 3: unpack = int32_unpack length = 4 else: unpack = uint16_unpack length = 2 numelements = unpack(byts[:length]) p = length themap = OrderedDict() for _ in range(numelements): key_len = unpack(byts[p:p + length]) p += length keybytes = byts[p:p + key_len] p += key_len val_len = unpack(byts[p:p + length]) p += length valbytes = byts[p:p + val_len] p += val_len key = subkeytype.from_binary(keybytes, protocol_version) val = subvaltype.from_binary(valbytes, protocol_version) themap[key] = val return themap
def __new__(cls, name, bases, attrs): # move column definitions into columns dict # and set default column names column_dict = OrderedDict() primary_keys = OrderedDict() pk_name = None # get inherited properties inherited_columns = OrderedDict() for base in bases: for k, v in getattr(base, '_defined_columns', {}).items(): inherited_columns.setdefault(k, v) # short circuit __abstract__ inheritance is_abstract = attrs['__abstract__'] = attrs.get('__abstract__', False) # short circuit __discriminator_value__ inheritance attrs['__discriminator_value__'] = attrs.get('__discriminator_value__') # TODO __default__ttl__ should be removed in the next major release options = attrs.get('__options__') or {} attrs['__default_ttl__'] = options.get('default_time_to_live') column_definitions = [(k, v) for k, v in attrs.items() if isinstance(v, columns.Column)] column_definitions = sorted(column_definitions, key=lambda x: x[1].position) is_polymorphic_base = any([c[1].discriminator_column for c in column_definitions]) column_definitions = [x for x in inherited_columns.items()] + column_definitions discriminator_columns = [c for c in column_definitions if c[1].discriminator_column] is_polymorphic = len(discriminator_columns) > 0 if len(discriminator_columns) > 1: raise ModelDefinitionException('only one discriminator_column can be defined in a model, {0} found'.format(len(discriminator_columns))) if attrs['__discriminator_value__'] and not is_polymorphic: raise ModelDefinitionException('__discriminator_value__ specified, but no base columns defined with discriminator_column=True') discriminator_column_name, discriminator_column = discriminator_columns[0] if discriminator_columns else (None, None) if isinstance(discriminator_column, (columns.BaseContainerColumn, columns.Counter)): raise ModelDefinitionException('counter and container columns cannot be used as discriminator columns') # find polymorphic base class polymorphic_base = None if is_polymorphic and not is_polymorphic_base: def _get_polymorphic_base(bases): for base in bases: if getattr(base, '_is_polymorphic_base', False): return base klass = _get_polymorphic_base(base.__bases__) if klass: return klass polymorphic_base = _get_polymorphic_base(bases) defined_columns = OrderedDict(column_definitions) # check for primary key if not is_abstract and not any([v.primary_key for k, v in column_definitions]): raise ModelDefinitionException("At least 1 primary key is required.") counter_columns = [c for c in defined_columns.values() if isinstance(c, columns.Counter)] data_columns = [c for c in defined_columns.values() if not c.primary_key and not isinstance(c, columns.Counter)] if counter_columns and data_columns: raise ModelDefinitionException('counter models may not have data columns') has_partition_keys = any(v.partition_key for (k, v) in column_definitions) def _transform_column(col_name, col_obj): column_dict[col_name] = col_obj if col_obj.primary_key: primary_keys[col_name] = col_obj col_obj.set_column_name(col_name) # set properties attrs[col_name] = ColumnDescriptor(col_obj) partition_key_index = 0 # transform column definitions for k, v in column_definitions: # don't allow a column with the same name as a built-in attribute or method if k in BaseModel.__dict__: raise ModelDefinitionException("column '{0}' conflicts with built-in attribute/method".format(k)) # counter column primary keys are not allowed if (v.primary_key or v.partition_key) and isinstance(v, columns.Counter): raise ModelDefinitionException('counter columns cannot be used as primary keys') # this will mark the first primary key column as a partition # key, if one hasn't been set already if not has_partition_keys and v.primary_key: v.partition_key = True has_partition_keys = True if v.partition_key: v._partition_key_index = partition_key_index partition_key_index += 1 overriding = column_dict.get(k) if overriding: v.position = overriding.position v.partition_key = overriding.partition_key v._partition_key_index = overriding._partition_key_index _transform_column(k, v) partition_keys = OrderedDict(k for k in primary_keys.items() if k[1].partition_key) clustering_keys = OrderedDict(k for k in primary_keys.items() if not k[1].partition_key) if attrs.get('__compute_routing_key__', True): key_cols = [c for c in partition_keys.values()] partition_key_index = dict((col.db_field_name, col._partition_key_index) for col in key_cols) key_cql_types = [c.cql_type for c in key_cols] key_serializer = staticmethod(lambda parts, proto_version: [t.to_binary(p, proto_version) for t, p in zip(key_cql_types, parts)]) else: partition_key_index = {} key_serializer = staticmethod(lambda parts, proto_version: None) # setup partition key shortcut if len(partition_keys) == 0: if not is_abstract: raise ModelException("at least one partition key must be defined") if len(partition_keys) == 1: pk_name = [x for x in partition_keys.keys()][0] attrs['pk'] = attrs[pk_name] else: # composite partition key case, get/set a tuple of values _get = lambda self: tuple(self._values[c].getval() for c in partition_keys.keys()) _set = lambda self, val: tuple(self._values[c].setval(v) for (c, v) in zip(partition_keys.keys(), val)) attrs['pk'] = property(_get, _set) # some validation col_names = set() for v in column_dict.values(): # check for duplicate column names if v.db_field_name in col_names: raise ModelException("{0} defines the column '{1}' more than once".format(name, v.db_field_name)) if v.clustering_order and not (v.primary_key and not v.partition_key): raise ModelException("clustering_order may be specified only for clustering primary keys") if v.clustering_order and v.clustering_order.lower() not in ('asc', 'desc'): raise ModelException("invalid clustering order '{0}' for column '{1}'".format(repr(v.clustering_order), v.db_field_name)) col_names.add(v.db_field_name) # create db_name -> model name map for loading db_map = {} for col_name, field in column_dict.items(): db_field = field.db_field_name if db_field != col_name: db_map[db_field] = col_name # add management members to the class attrs['_columns'] = column_dict attrs['_primary_keys'] = primary_keys attrs['_defined_columns'] = defined_columns # maps the database field to the models key attrs['_db_map'] = db_map attrs['_pk_name'] = pk_name attrs['_dynamic_columns'] = {} attrs['_partition_keys'] = partition_keys attrs['_partition_key_index'] = partition_key_index attrs['_key_serializer'] = key_serializer attrs['_clustering_keys'] = clustering_keys attrs['_has_counter'] = len(counter_columns) > 0 # add polymorphic management attributes attrs['_is_polymorphic_base'] = is_polymorphic_base attrs['_is_polymorphic'] = is_polymorphic attrs['_polymorphic_base'] = polymorphic_base attrs['_discriminator_column'] = discriminator_column attrs['_discriminator_column_name'] = discriminator_column_name attrs['_discriminator_map'] = {} if is_polymorphic_base else None # setup class exceptions DoesNotExistBase = None for base in bases: DoesNotExistBase = getattr(base, 'DoesNotExist', None) if DoesNotExistBase is not None: break DoesNotExistBase = DoesNotExistBase or attrs.pop('DoesNotExist', BaseModel.DoesNotExist) attrs['DoesNotExist'] = type('DoesNotExist', (DoesNotExistBase,), {}) MultipleObjectsReturnedBase = None for base in bases: MultipleObjectsReturnedBase = getattr(base, 'MultipleObjectsReturned', None) if MultipleObjectsReturnedBase is not None: break MultipleObjectsReturnedBase = MultipleObjectsReturnedBase or attrs.pop('MultipleObjectsReturned', BaseModel.MultipleObjectsReturned) attrs['MultipleObjectsReturned'] = type('MultipleObjectsReturned', (MultipleObjectsReturnedBase,), {}) # create the class and add a QuerySet to it klass = super(ModelMetaClass, cls).__new__(cls, name, bases, attrs) udts = [] for col in column_dict.values(): columns.resolve_udts(col, udts) for user_type in set(udts): user_type.register_for_keyspace(klass._get_keyspace()) return klass
class TableMetadata(object): """ A representation of the schema for a single table. """ keyspace = None """ An instance of :class:`~.KeyspaceMetadata`. """ name = None """ The string name of the table. """ partition_key = None """ A list of :class:`.ColumnMetadata` instances representing the columns in the partition key for this table. This will always hold at least one column. """ clustering_key = None """ A list of :class:`.ColumnMetadata` instances representing the columns in the clustering key for this table. These are all of the :attr:`.primary_key` columns that are not in the :attr:`.partition_key`. Note that a table may have no clustering keys, in which case this will be an empty list. """ @property def primary_key(self): """ A list of :class:`.ColumnMetadata` representing the components of the primary key for this table. """ return self.partition_key + self.clustering_key columns = None """ A dict mapping column names to :class:`.ColumnMetadata` instances. """ is_compact_storage = False options = None """ A dict mapping table option names to their specific settings for this table. """ recognized_options = ( "comment", "read_repair_chance", "dclocal_read_repair_chance", # kept to be safe, but see _build_table_options() "local_read_repair_chance", "replicate_on_write", "gc_grace_seconds", "bloom_filter_fp_chance", "caching", "compaction_strategy_class", "compaction_strategy_options", "min_compaction_threshold", "max_compaction_threshold", "compression_parameters", "min_index_interval", "max_index_interval", "index_interval", "speculative_retry", "rows_per_partition_to_cache", "memtable_flush_period_in_ms", "populate_io_cache_on_flush", "compaction", "compression", "default_time_to_live") compaction_options = { "min_compaction_threshold": "min_threshold", "max_compaction_threshold": "max_threshold", "compaction_strategy_class": "class"} triggers = None """ A dict mapping trigger names to :class:`.TriggerMetadata` instances. """ @property def is_cql_compatible(self): """ A boolean indicating if this table can be represented as CQL in export """ # no such thing as DCT in CQL incompatible = issubclass(self.comparator, types.DynamicCompositeType) # no compact storage with more than one column beyond PK incompatible |= self.is_compact_storage and len(self.columns) > len(self.primary_key) + 1 return not incompatible def __init__(self, keyspace_metadata, name, partition_key=None, clustering_key=None, columns=None, triggers=None, options=None): self.keyspace = keyspace_metadata self.name = name self.partition_key = [] if partition_key is None else partition_key self.clustering_key = [] if clustering_key is None else clustering_key self.columns = OrderedDict() if columns is None else columns self.options = options self.comparator = None self.triggers = OrderedDict() if triggers is None else triggers def export_as_string(self): """ Returns a string of CQL queries that can be used to recreate this table along with all indexes on it. The returned string is formatted to be human readable. """ if self.is_cql_compatible: ret = self.all_as_cql() else: # If we can't produce this table with CQL, comment inline ret = "/*\nWarning: Table %s.%s omitted because it has constructs not compatible with CQL (was created via legacy API).\n" % \ (self.keyspace.name, self.name) ret += "\nApproximate structure, for reference:\n(this should not be used to reproduce this schema)\n\n%s" % self.all_as_cql() ret += "\n*/" return ret def all_as_cql(self): ret = self.as_cql_query(formatted=True) ret += ";" for col_meta in self.columns.values(): if col_meta.index: ret += "\n%s;" % (col_meta.index.as_cql_query(),) for trigger_meta in self.triggers.values(): ret += "\n%s;" % (trigger_meta.as_cql_query(),) return ret def as_cql_query(self, formatted=False): """ Returns a CQL query that can be used to recreate this table (index creations are not included). If `formatted` is set to :const:`True`, extra whitespace will be added to make the query human readable. """ ret = "CREATE TABLE %s.%s (%s" % ( protect_name(self.keyspace.name), protect_name(self.name), "\n" if formatted else "") if formatted: column_join = ",\n" padding = " " else: column_join = ", " padding = "" columns = [] for col in self.columns.values(): columns.append("%s %s%s" % (protect_name(col.name), col.typestring, ' static' if col.is_static else '')) if len(self.partition_key) == 1 and not self.clustering_key: columns[0] += " PRIMARY KEY" ret += column_join.join("%s%s" % (padding, col) for col in columns) # primary key if len(self.partition_key) > 1 or self.clustering_key: ret += "%s%sPRIMARY KEY (" % (column_join, padding) if len(self.partition_key) > 1: ret += "(%s)" % ", ".join(protect_name(col.name) for col in self.partition_key) else: ret += self.partition_key[0].name if self.clustering_key: ret += ", %s" % ", ".join(protect_name(col.name) for col in self.clustering_key) ret += ")" # options ret += "%s) WITH " % ("\n" if formatted else "") option_strings = [] if self.is_compact_storage: option_strings.append("COMPACT STORAGE") if self.clustering_key: cluster_str = "CLUSTERING ORDER BY " clustering_names = protect_names([c.name for c in self.clustering_key]) if self.is_compact_storage and \ not issubclass(self.comparator, types.CompositeType): subtypes = [self.comparator] else: subtypes = self.comparator.subtypes inner = [] for colname, coltype in zip(clustering_names, subtypes): ordering = "DESC" if issubclass(coltype, types.ReversedType) else "ASC" inner.append("%s %s" % (colname, ordering)) cluster_str += "(%s)" % ", ".join(inner) option_strings.append(cluster_str) option_strings.extend(self._make_option_strings()) join_str = "\n AND " if formatted else " AND " ret += join_str.join(option_strings) return ret def _make_option_strings(self): ret = [] options_copy = dict(self.options.items()) if not options_copy.get('compaction'): options_copy.pop('compaction', None) actual_options = json.loads(options_copy.pop('compaction_strategy_options', '{}')) for system_table_name, compact_option_name in self.compaction_options.items(): value = options_copy.pop(system_table_name, None) if value: actual_options.setdefault(compact_option_name, value) compaction_option_strings = ["'%s': '%s'" % (k, v) for k, v in actual_options.items()] ret.append('compaction = {%s}' % ', '.join(compaction_option_strings)) for system_table_name in self.compaction_options.keys(): options_copy.pop(system_table_name, None) # delete if present options_copy.pop('compaction_strategy_option', None) if not options_copy.get('compression'): params = json.loads(options_copy.pop('compression_parameters', '{}')) param_strings = ["'%s': '%s'" % (k, v) for k, v in params.items()] ret.append('compression = {%s}' % ', '.join(param_strings)) for name, value in options_copy.items(): if value is not None: if name == "comment": value = value or "" ret.append("%s = %s" % (name, protect_value(value))) return list(sorted(ret))
def select_from_table(self, table_name, limit=10000, **kwargs): result = [] schema = self.schema_for_table(table_name) partition_names = [] for column in schema.partition_key: partition_names.append(column.name) # Sort kwargs so they are in the same order as the columns. ordered_kwargs = OrderedDict() for name in self._models[table_name]._columns.keys(): for key, value in kwargs.items(): if key.split('__')[0] == name and value is not None: ordered_kwargs[key] = value # Convert arguments to filters partitions = [[]] filters = [] for key, value in ordered_kwargs.items(): key_value = key.split('__')[0] operator = None if len( key.split('__')) == 1 else key.split('__')[1] if key_value in partition_names: if operator == 'in': original_partitions = list(partitions) partitions = [] for _ in range(len(value)): for element in original_partitions: partitions.append(list(element)) for multiplier in range(len(value)): for index in range(len(original_partitions)): partitions[multiplier * len(original_partitions) + index].append(value[multiplier]) else: for partition in partitions: partition.append(value) filters.append(self.filter_for_argument(key, value)) if len(partitions[0]) == 0: partitions = self.cluster.database[ self.keyspace][table_name].keys() candidate_elements = [] for partition in partitions: for key, row in self.cluster.database[self.keyspace][table_name][ tuple(partition)].items(): does_match = True for filter in filters: if not filter(row[1]): does_match = False break if not does_match: continue candidate_elements.append((partition, key)) # To reverse, they all need to agree. This might not work with compound clustering keys, but we really # shouldn't use those anyways is_reversed = True for column in schema.primary_key: if column in schema.partition_key: continue if not column.is_reversed: is_reversed = False candidate_elements.sort(reverse=is_reversed) for candidate in candidate_elements: element = self.cluster.database[self.keyspace][table_name][tuple( candidate[0])][candidate[1]] deadline = element[0] if deadline is None or deadline > time.time(): result.append(element[1]) limit -= 1 if limit <= 0: return result else: del self.cluster.database[self.keyspace][table_name][tuple( candidate[0])][candidate[1]] return result
(b'A46\xa9', 'InetAddressType', '65.52.54.169'), (b'*\x00\x13(\xe1\x02\xcc\xc0\x00\x00\x00\x00\x00\x00\x01"', 'InetAddressType', '2a00:1328:e102:ccc0::122'), (b'\xe3\x81\xbe\xe3\x81\x97\xe3\x81\xa6', 'UTF8Type', u'\u307e\u3057\u3066'), (b'\xe3\x81\xbe\xe3\x81\x97\xe3\x81\xa6' * 1000, 'UTF8Type', u'\u307e\u3057\u3066' * 1000), (b'', 'UTF8Type', u''), (b'\xff' * 16, 'UUIDType', UUID('ffffffff-ffff-ffff-ffff-ffffffffffff')), (b'I\x15~\xfc\xef<\x9d\xe3\x16\x98\xaf\x80\x1f\xb4\x0b*', 'UUIDType', UUID('49157efc-ef3c-9de3-1698-af801fb40b2a')), (b'', 'UUIDType', None), (b'', 'MapType(AsciiType, BooleanType)', None), (b'', 'ListType(FloatType)', None), (b'', 'SetType(LongType)', None), (b'\x00\x00', 'MapType(DecimalType, BooleanType)', OrderedDict()), (b'\x00\x00', 'ListType(FloatType)', []), (b'\x00\x00', 'SetType(IntegerType)', sortedset()), (b'\x00\x01\x00\x10\xafYC\xa3\xea<\x11\xe1\xabc\xc4,\x03"y\xf0', 'ListType(TimeUUIDType)', [UUID(bytes=b'\xafYC\xa3\xea<\x11\xe1\xabc\xc4,\x03"y\xf0')]), ) ordered_dict_value = OrderedDict() ordered_dict_value[u'\u307fbob'] = 199 ordered_dict_value[u''] = -1 ordered_dict_value[u'\\'] = 0 # these following entries work for me right now, but they're dependent on # vagaries of internal python ordering for unordered types marshalled_value_pairs_unsafe = (
def ordered_dict_factory(colnames, rows): return [OrderedDict(zip(colnames, row)) for row in rows]
def test_empty_strings_and_nones(self): c = Cluster(protocol_version=PROTOCOL_VERSION) s = c.connect() s.execute(""" CREATE KEYSPACE test_empty_strings_and_nones WITH replication = { 'class' : 'SimpleStrategy', 'replication_factor': '1'} """) s.set_keyspace("test_empty_strings_and_nones") s.execute(self.create_type_table) s.execute("INSERT INTO mytable (a, b) VALUES ('a', 'b')") s.row_factory = dict_factory results = s.execute(""" SELECT c, d, f, g, h, i, j, k, l, m, n, o, p, q, r, s, t FROM mytable """) self.assertTrue(all(x is None for x in results[0].values())) prepared = s.prepare(""" SELECT c, d, f, g, h, i, j, k, l, m, n, o, p, q, r, s, t FROM mytable """) results = s.execute(prepared.bind(())) self.assertTrue(all(x is None for x in results[0].values())) # insert empty strings for string-like fields and fetch them s.execute( "INSERT INTO mytable (a, b, c, o, s, l, n) VALUES ('a', 'b', %s, %s, %s, %s, %s)", ('', '', '', [''], { '': 3 })) self.assertEqual( { 'c': '', 'o': '', 's': '', 'l': ('', ), 'n': OrderedDict({'': 3}) }, s.execute( "SELECT c, o, s, l, n FROM mytable WHERE a='a' AND b='b'")[0]) self.assertEqual( { 'c': '', 'o': '', 's': '', 'l': ('', ), 'n': OrderedDict({'': 3}) }, s.execute( s.prepare( "SELECT c, o, s, l, n FROM mytable WHERE a='a' AND b='b'"), [])[0]) # non-string types shouldn't accept empty strings for col in ('d', 'f', 'g', 'h', 'i', 'k', 'l', 'm', 'n', 'q', 'r', 't'): query = "INSERT INTO mytable (a, b, %s) VALUES ('a', 'b', %%s)" % ( col, ) try: s.execute(query, ['']) except InvalidRequest: pass else: self.fail("Expected an InvalidRequest error when inserting an " "emptry string for column %s" % (col, )) prepared = s.prepare( "INSERT INTO mytable (a, b, %s) VALUES ('a', 'b', ?)" % (col, )) try: s.execute(prepared, ['']) except TypeError: pass else: self.fail( "Expected an InvalidRequest error when inserting an " "emptry string for column %s with a prepared statement" % (col, )) # insert values for all columns values = [ 'a', 'b', 'a', 1, True, Decimal('1.0'), 0.1, 0.1, "1.2.3.4", 1, ['a'], set([1]), { 'a': 1 }, 'a', datetime.now(), uuid4(), uuid1(), 'a', 1 ] s.execute( """ INSERT INTO mytable (a, b, c, d, f, g, h, i, j, k, l, m, n, o, p, q, r, s, t) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s) """, values) # then insert None, which should null them out null_values = values[:2] + ([None] * (len(values) - 2)) s.execute( """ INSERT INTO mytable (a, b, c, d, f, g, h, i, j, k, l, m, n, o, p, q, r, s, t) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s) """, null_values) results = s.execute(""" SELECT c, d, f, g, h, i, j, k, l, m, n, o, p, q, r, s, t FROM mytable """) self.assertEqual( [], [(name, val) for (name, val) in results[0].items() if val is not None]) prepared = s.prepare(""" SELECT c, d, f, g, h, i, j, k, l, m, n, o, p, q, r, s, t FROM mytable """) results = s.execute(prepared.bind(())) self.assertEqual( [], [(name, val) for (name, val) in results[0].items() if val is not None]) # do the same thing again, but use a prepared statement to insert the nulls s.execute( """ INSERT INTO mytable (a, b, c, d, f, g, h, i, j, k, l, m, n, o, p, q, r, s, t) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s) """, values) prepared = s.prepare(""" INSERT INTO mytable (a, b, c, d, f, g, h, i, j, k, l, m, n, o, p, q, r, s, t) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) """) s.execute(prepared, null_values) results = s.execute(""" SELECT c, d, f, g, h, i, j, k, l, m, n, o, p, q, r, s, t FROM mytable """) self.assertEqual( [], [(name, val) for (name, val) in results[0].items() if val is not None]) prepared = s.prepare(""" SELECT c, d, f, g, h, i, j, k, l, m, n, o, p, q, r, s, t FROM mytable """) results = s.execute(prepared.bind(())) self.assertEqual( [], [(name, val) for (name, val) in results[0].items() if val is not None])
from cassandra.protocol import (ReadyMessage, AuthenticateMessage, OptionsMessage, StartupMessage, ErrorMessage, CredentialsMessage, QueryMessage, ResultMessage, ProtocolHandler, InvalidRequestException, SupportedMessage, AuthResponseMessage, AuthChallengeMessage, AuthSuccessMessage, ProtocolException, MAX_SUPPORTED_VERSION, RegisterMessage) from cassandra.util import OrderedDict log = logging.getLogger(__name__) # We use an ordered dictionary and specifically add lz4 before # snappy so that lz4 will be preferred. Changing the order of this # will change the compression preferences for the driver. locally_supported_compressions = OrderedDict() try: import lz4 except ImportError: pass else: # Cassandra writes the uncompressed message length in big endian order, # but the lz4 lib requires little endian order, so we wrap these # functions to handle that def lz4_compress(byts): # write length in big-endian instead of little-endian return int32_pack(len(byts)) + lz4.compress(byts)[4:]
class TableMetadata(object): """ A representation of the schema for a single table. """ keyspace = None """ An instance of :class:`~.KeyspaceMetadata`. """ name = None """ The string name of the table. """ partition_key = None """ A list of :class:`.ColumnMetadata` instances representing the columns in the partition key for this table. This will always hold at least one column. """ clustering_key = None """ A list of :class:`.ColumnMetadata` instances representing the columns in the clustering key for this table. These are all of the :attr:`.primary_key` columns that are not in the :attr:`.partition_key`. Note that a table may have no clustering keys, in which case this will be an empty list. """ @property def primary_key(self): """ A list of :class:`.ColumnMetadata` representing the components of the primary key for this table. """ return self.partition_key + self.clustering_key columns = None """ A dict mapping column names to :class:`.ColumnMetadata` instances. """ is_compact_storage = False options = None """ A dict mapping table option names to their specific settings for this table. """ recognized_options = ( "comment", "read_repair_chance", "dclocal_read_repair_chance", # kept to be safe, but see _build_table_options() "local_read_repair_chance", "replicate_on_write", "gc_grace_seconds", "bloom_filter_fp_chance", "caching", "compaction_strategy_class", "compaction_strategy_options", "min_compaction_threshold", "max_compaction_threshold", "compression_parameters", "min_index_interval", "max_index_interval", "index_interval", "speculative_retry", "rows_per_partition_to_cache", "memtable_flush_period_in_ms", "populate_io_cache_on_flush", "compaction", "compression", "default_time_to_live") compaction_options = { "min_compaction_threshold": "min_threshold", "max_compaction_threshold": "max_threshold", "compaction_strategy_class": "class" } def __init__(self, keyspace_metadata, name, partition_key=None, clustering_key=None, columns=None, options=None): self.keyspace = keyspace_metadata self.name = name self.partition_key = [] if partition_key is None else partition_key self.clustering_key = [] if clustering_key is None else clustering_key self.columns = OrderedDict() if columns is None else columns self.options = options self.comparator = None def export_as_string(self): """ Returns a string of CQL queries that can be used to recreate this table along with all indexes on it. The returned string is formatted to be human readable. """ ret = self.as_cql_query(formatted=True) ret += ";" for col_meta in self.columns.values(): if col_meta.index: ret += "\n%s;" % (col_meta.index.as_cql_query(), ) return ret def as_cql_query(self, formatted=False): """ Returns a CQL query that can be used to recreate this table (index creations are not included). If `formatted` is set to :const:`True`, extra whitespace will be added to make the query human readable. """ ret = "CREATE TABLE %s.%s (%s" % (protect_name( self.keyspace.name), protect_name( self.name), "\n" if formatted else "") if formatted: column_join = ",\n" padding = " " else: column_join = ", " padding = "" columns = [] for col in self.columns.values(): columns.append("%s %s%s" % (protect_name( col.name), col.typestring, ' static' if col.is_static else '')) if len(self.partition_key) == 1 and not self.clustering_key: columns[0] += " PRIMARY KEY" ret += column_join.join("%s%s" % (padding, col) for col in columns) # primary key if len(self.partition_key) > 1 or self.clustering_key: ret += "%s%sPRIMARY KEY (" % (column_join, padding) if len(self.partition_key) > 1: ret += "(%s)" % ", ".join( protect_name(col.name) for col in self.partition_key) else: ret += self.partition_key[0].name if self.clustering_key: ret += ", %s" % ", ".join( protect_name(col.name) for col in self.clustering_key) ret += ")" # options ret += "%s) WITH " % ("\n" if formatted else "") option_strings = [] if self.is_compact_storage: option_strings.append("COMPACT STORAGE") if self.clustering_key: cluster_str = "CLUSTERING ORDER BY " clustering_names = protect_names( [c.name for c in self.clustering_key]) if self.is_compact_storage and \ not issubclass(self.comparator, types.CompositeType): subtypes = [self.comparator] else: subtypes = self.comparator.subtypes inner = [] for colname, coltype in zip(clustering_names, subtypes): ordering = "DESC" if issubclass(coltype, types.ReversedType) else "ASC" inner.append("%s %s" % (colname, ordering)) cluster_str += "(%s)" % ", ".join(inner) option_strings.append(cluster_str) option_strings.extend(self._make_option_strings()) join_str = "\n AND " if formatted else " AND " ret += join_str.join(option_strings) return ret def _make_option_strings(self): ret = [] options_copy = dict(self.options.items()) if not options_copy.get('compaction'): options_copy.pop('compaction', None) actual_options = json.loads( options_copy.pop('compaction_strategy_options', '{}')) for system_table_name, compact_option_name in self.compaction_options.items( ): value = options_copy.pop(system_table_name, None) if value: actual_options.setdefault(compact_option_name, value) compaction_option_strings = [ "'%s': '%s'" % (k, v) for k, v in actual_options.items() ] ret.append('compaction = {%s}' % ', '.join(compaction_option_strings)) for system_table_name in self.compaction_options.keys(): options_copy.pop(system_table_name, None) # delete if present options_copy.pop('compaction_strategy_option', None) if not options_copy.get('compression'): params = json.loads( options_copy.pop('compression_parameters', '{}')) if params: param_strings = [ "'%s': '%s'" % (k, v) for k, v in params.items() ] ret.append('compression = {%s}' % ', '.join(param_strings)) for name, value in options_copy.items(): if value is not None: if name == "comment": value = value or "" ret.append("%s = %s" % (name, protect_value(value))) return list(sorted(ret))