class SystemManager(object):
"""
Lets you examine and modify schema definitions as well as get basic
information about the cluster.
This class is mainly designed to be used manually in a python shell,
not as part of a program, although it can be used that way.
All operations which modify a keyspace or column family definition
will block until the cluster reports that all nodes have accepted
the modification.
Example Usage:
.. code-block:: python
>>> from pycassa.system_manager import *
>>> sys = SystemManager('192.168.10.2:9160')
>>> sys.create_keyspace('TestKeyspace', SIMPLE_STRATEGY, {'replication_factor': '1'})
>>> sys.create_column_family('TestKeyspace', 'TestCF', super=False,
... comparator_type=LONG_TYPE)
>>> sys.alter_column_family('TestKeyspace', 'TestCF', key_cache_size=42, gc_grace_seconds=1000)
>>> sys.drop_keyspace('TestKeyspace')
>>> sys.close()
"""
def __init__(self, server='localhost:9160', credentials=None, framed_transport=True,
timeout=_DEFAULT_TIMEOUT):
self._conn = Connection(None, server, framed_transport, timeout, credentials)
def close(self):
""" Closes the underlying connection """
self._conn.close()
def get_keyspace_column_families(self, keyspace, use_dict_for_col_metadata=False):
"""
Returns a raw description of the keyspace, which is more useful for use
in programs than :meth:`describe_keyspace()`.
If `use_dict_for_col_metadata` is ``True``, the CfDef's column_metadata will
be stored as a dictionary where the keys are column names instead of a list.
Returns a dictionary of the form ``{column_family_name: CfDef}``
"""
if keyspace is None:
keyspace = self._keyspace
ks_def = self._conn.describe_keyspace(keyspace)
cf_defs = dict()
for cf_def in ks_def.cf_defs:
cf_defs[cf_def.name] = cf_def
if use_dict_for_col_metadata:
old_metadata = cf_def.column_metadata
new_metadata = dict()
for datum in old_metadata:
new_metadata[datum.name] = datum
cf_def.column_metadata = new_metadata
return cf_defs
def get_keyspace_properties(self, keyspace):
"""
Gets a keyspace's properties.
Returns a :class:`dict` with 'strategy_class' and
'strategy_options' as keys.
"""
if keyspace is None:
keyspace = self._keyspace
ks_def = self._conn.describe_keyspace(keyspace)
return {'replication_strategy': ks_def.strategy_class,
'strategy_options': ks_def.strategy_options}
def list_keyspaces(self):
""" Returns a list of all keyspace names. """
return [ks.name for ks in self._conn.describe_keyspaces()]
def describe_ring(self, keyspace):
""" Describes the Cassandra cluster """
return self._conn.describe_ring(keyspace)
def describe_cluster_name(self):
""" Gives the cluster name """
return self._conn.describe_cluster_name()
def describe_version(self):
""" Gives the server's API version """
return self._conn.describe_version()
def describe_schema_versions(self):
""" Lists what schema version each node has """
return self._conn.describe_schema_versions()
def describe_partitioner(self):
""" Gives the partitioner that the cluster is using """
part = self._conn.describe_partitioner()
return part[part.rfind('.') + 1: ]
def describe_snitch(self):
""" Gives the snitch that the cluster is using """
snitch = self._conn.describe_snitch()
return snitch[snitch.rfind('.') + 1: ]
def _system_add_keyspace(self, ksdef):
return self._schema_update(self._conn.system_add_keyspace, ksdef)
def _system_update_keyspace(self, ksdef):
return self._schema_update(self._conn.system_update_keyspace, ksdef)
def create_keyspace(self, name,
replication_strategy=SIMPLE_STRATEGY,
strategy_options=None, durable_writes=True):
"""
Creates a new keyspace. Column families may be added to this keyspace
after it is created using :meth:`create_column_family()`.
`replication_strategy` determines how replicas are chosen for this keyspace.
The strategies that Cassandra provides by default
are available as :const:`SIMPLE_STRATEGY`, :const:`NETWORK_TOPOLOGY_STRATEGY`,
and :const:`OLD_NETWORK_TOPOLOGY_STRATEGY`.
`strategy_options` is a dictionary of strategy options. For
NetworkTopologyStrategy, the dictionary should look like
``{'Datacenter1': '2', 'Datacenter2': '1'}``. This maps each
datacenter (as defined in a Cassandra property file) to a replica count.
For SimpleStrategy, you can specify the replication factor as follows:
``{'replication_factor': '1'}``.
Example Usage:
.. code-block:: python
>>> from pycassa.system_manager import *
>>> sys = SystemManager('192.168.10.2:9160')
>>> # Create a SimpleStrategy keyspace
>>> sys.create_keyspace('SimpleKS', SIMPLE_STRATEGY, {'replication_factor': '1'})
>>> # Create a NetworkTopologyStrategy keyspace
>>> sys.create_keyspace('NTS_KS', NETWORK_TOPOLOGY_STRATEGY, {'DC1': '2', 'DC2': '1'})
>>> sys.close()
"""
if replication_strategy.find('.') == -1:
strategy_class = 'org.apache.cassandra.locator.%s' % replication_strategy
else:
strategy_class = replication_strategy
ksdef = KsDef(name, strategy_class=strategy_class,
strategy_options=strategy_options,
cf_defs=[],
durable_writes=durable_writes)
self._system_add_keyspace(ksdef)
def alter_keyspace(self, keyspace, replication_strategy=None,
strategy_options=None, durable_writes=None):
"""
Alters an existing keyspace.
.. warning:: Don't use this unless you know what you are doing.
Parameters are the same as for :meth:`create_keyspace()`.
"""
old_ksdef = self._conn.describe_keyspace(keyspace)
old_durable = getattr(old_ksdef, 'durable_writes', True)
ksdef = KsDef(name=old_ksdef.name,
strategy_class=old_ksdef.strategy_class,
strategy_options=old_ksdef.strategy_options,
cf_defs=[],
durable_writes=old_durable)
if replication_strategy is not None:
if replication_strategy.find('.') == -1:
ksdef.strategy_class = 'org.apache.cassandra.locator.%s' % replication_strategy
else:
ksdef.strategy_class = replication_strategy
if strategy_options is not None:
ksdef.strategy_options = strategy_options
if durable_writes is not None:
ksdef.durable_writes = durable_writes
self._system_update_keyspace(ksdef)
def drop_keyspace(self, keyspace):
"""
Drops a keyspace from the cluster.
"""
self._schema_update(self._conn.system_drop_keyspace, keyspace)
def _system_add_column_family(self, cfdef):
self._conn.set_keyspace(cfdef.keyspace)
return self._schema_update(self._conn.system_add_column_family, cfdef)
def _qualify_type_class(self, classname):
if classname:
if isinstance(classname, types.CassandraType):
s = str(classname)
elif isinstance(classname, basestring):
s = classname
else:
raise TypeError(
"Column family validators and comparators " \
"must be specified as instances of " \
"pycassa.types.CassandraType subclasses or strings.")
if s.find('.') == -1:
return 'org.apache.cassandra.db.marshal.%s' % s
else:
return s
else:
return None
def create_column_family(self, keyspace, name, super=False,
comparator_type=None,
subcomparator_type=None,
key_cache_size=None,
row_cache_size=None,
gc_grace_seconds=None,
read_repair_chance=None,
default_validation_class=None,
key_validation_class=None,
min_compaction_threshold=None,
max_compaction_threshold=None,
key_cache_save_period_in_seconds=None,
row_cache_save_period_in_seconds=None,
replicate_on_write=None,
merge_shards_chance=None,
row_cache_provider=None,
key_alias=None,
compaction_strategy=None,
compaction_strategy_options=None,
row_cache_keys_to_save=None,
compression_options=None,
comment=None):
"""
Creates a new column family in a given keyspace. If a value is not
supplied for any of optional parameters, Cassandra will use a reasonable
default value.
:param str keyspace: what keyspace the column family will be created in
:param str name: the name of the column family
:param bool super: Whether or not this column family is a super column family
:param str comparator_type: What type the column names will be, which affects
their sort order. By default, :const:`LONG_TYPE`, :const:`INT_TYPE`,
:const:`ASCII_TYPE`, :const:`UTF8_TYPE`, :const:`TIME_UUID_TYPE`,
:const:`LEXICAL_UUID_TYPE` and :const:`BYTES_TYPE` are provided. Custom
types may be used as well by providing the class name; if the custom
comparator class is not in ``org.apache.cassandra.db.marshal``, the fully
qualified class name must be given.
:param str subcomparator_type: Like `comparator_type`, but if the column family
is a super column family, this applies to the type of the subcolumn names
:param key_cache_size: The size of the key cache, either in a percentage of
total keys (0.15, for example) or in an absolute number of
keys (20000, for example).
:param float row_cache_size: Same as `key_cache_size`, but for the row cache
:param int gc_grace_seconds: Number of seconds before tombstones are removed
:param float read_repair_chance: probability of a read repair occuring
:param str default_validation_class: the data type for all column values in the CF.
The choices for this are the same as for `comparator_type`.
:param str key_validation_class: the data type for row keys in the CF. The choices
for this are the same as for `comparator_type`.
:param int min_compaction_threshold: Number of similarly sized SSTables that must
be present before a minor compaction is scheduled. Setting to 0 disables minor
compactions.
:param int max_compaction_threshold: Number of similarly sized SSTables that must
be present before a minor compaction is performed immediately. Setting to 0
disables minor compactions.
:param int key_cache_save_period_in_seconds: How often the key cache should
be saved; this helps to avoid a cold cache on restart
:param int row_cache_save_period_in_seconds: How often the row cache should
be saved; this helps to avoid a cold cache on restart
:param bool replicate_on_write: Whether counter operations are replicated at write-time.
This should almost always be True.
:param float merge_shards_chance: The probability that counter shards will be merged.
:param str row_cache_provider: The class that will be used to store the row cache.
Defaults to ``org.apache.cassandra.cache.ConcurrentLinkedHashCacheProvider``.
:param key_alias: A "column name" to be used for the row key. This currently only
matters for CQL. The default is "KEY".
:param compaction_strategy: The name of the compaction strategy. Choices include
TieredCompactionStrategy and LeveledCompactionStrategy.
:param compaction_strategy_options: A ``dict`` of options for the compaction
strategy.
:param row_cache_keys_to_save: A list of keys to be saved in the row cache. The
default of ``None`` allows any row to be cached.
:param compression_options: A ``dict`` of options for compression. Available
keys include "sstable_compression", which may be ``None`` for no compression,
"SnappyCompressor", "DeflateCompressor", or a custom compressor, and
"chunk_length_kb", which must be a power of 2.
:param str comment: A human readable description
"""
self._conn.set_keyspace(keyspace)
cfdef = CfDef()
cfdef.keyspace = keyspace
cfdef.name = name
if super:
cfdef.column_type = 'Super'
cfdef.comparator_type = self._qualify_type_class(comparator_type)
cfdef.subcomparator_type = self._qualify_type_class(subcomparator_type)
cfdef.default_validation_class = self._qualify_type_class(default_validation_class)
cfdef.key_validation_class = self._qualify_type_class(key_validation_class)
cfdef.replicate_on_write = replicate_on_write
cfdef.comment = comment
cfdef.key_alias = key_alias
if row_cache_provider:
cfdef.row_cache_provider = row_cache_provider
self._cfdef_assign(key_cache_size, cfdef, 'key_cache_size')
self._cfdef_assign(row_cache_size, cfdef, 'row_cache_size')
self._cfdef_assign(gc_grace_seconds, cfdef, 'gc_grace_seconds')
self._cfdef_assign(read_repair_chance, cfdef, 'read_repair_chance')
self._cfdef_assign(min_compaction_threshold, cfdef, 'min_compaction_threshold')
self._cfdef_assign(max_compaction_threshold, cfdef, 'max_compaction_threshold')
self._cfdef_assign(key_cache_save_period_in_seconds, cfdef, 'key_cache_save_period_in_seconds')
self._cfdef_assign(row_cache_save_period_in_seconds, cfdef, 'row_cache_save_period_in_seconds')
self._cfdef_assign(merge_shards_chance, cfdef, 'merge_shards_chance')
self._cfdef_assign(compaction_strategy, cfdef, 'compaction_strategy')
self._cfdef_assign(compaction_strategy_options, cfdef, 'compaction_strategy_options')
self._cfdef_assign(row_cache_keys_to_save, cfdef, 'row_cache_keys_to_save')
self._cfdef_assign(compression_options, cfdef, 'compression_options')
self._system_add_column_family(cfdef)
def _cfdef_assign(self, attr, cfdef, attr_name):
if attr is not None:
if attr < 0:
self._raise_ire('%s must be non-negative' % attr_name)
else:
setattr(cfdef, attr_name, attr)
def _raise_ire(self, why):
ire = InvalidRequestException()
ire.why = why
raise ire
def _system_update_column_family(self, cfdef):
return self._schema_update(self._conn.system_update_column_family, cfdef)
def alter_column_family(self, keyspace, column_family,
key_cache_size=None,
row_cache_size=None,
gc_grace_seconds=None,
read_repair_chance=None,
default_validation_class=None,
column_validation_classes={},
min_compaction_threshold=None,
max_compaction_threshold=None,
key_cache_save_period_in_seconds=None,
row_cache_save_period_in_seconds=None,
replicate_on_write=None,
merge_shards_chance=None,
row_cache_provider=None,
key_alias=None,
compaction_strategy=None,
compaction_strategy_options=None,
row_cache_keys_to_save=None,
compression_options=None,
comment=None):
"""
Alters an existing column family.
Parameter meanings are the same as for :meth:`create_column_family`,
but column family attributes which may not be modified are not
included here. There is an additional optional parameter:
:param dict column_validation_classes: keys are column names, values are
comparator_type. The effect is like calling alter_column for each key as column,
with each value as value_type
"""
self._conn.set_keyspace(keyspace)
cfdef = self.get_keyspace_column_families(keyspace)[column_family]
self._cfdef_assign(key_cache_size, cfdef, 'key_cache_size')
self._cfdef_assign(row_cache_size, cfdef, 'row_cache_size')
self._cfdef_assign(gc_grace_seconds, cfdef, 'gc_grace_seconds')
self._cfdef_assign(read_repair_chance, cfdef, 'read_repair_chance')
self._cfdef_assign(min_compaction_threshold, cfdef, 'min_compaction_threshold')
self._cfdef_assign(max_compaction_threshold, cfdef, 'max_compaction_threshold')
self._cfdef_assign(key_cache_save_period_in_seconds, cfdef, 'key_cache_save_period_in_seconds')
self._cfdef_assign(row_cache_save_period_in_seconds, cfdef, 'row_cache_save_period_in_seconds')
self._cfdef_assign(compaction_strategy, cfdef, 'compaction_strategy')
self._cfdef_assign(compaction_strategy_options, cfdef, 'compaction_strategy_options')
self._cfdef_assign(row_cache_keys_to_save, cfdef, 'row_cache_keys_to_save')
self._cfdef_assign(compression_options, cfdef, 'compression_options')
self._cfdef_assign(merge_shards_chance, cfdef, 'merge_shards_chance')
cfdef.replicate_on_write = replicate_on_write
cfdef.comment = comment
cfdef.key_alias = key_alias
if row_cache_provider:
cfdef.row_cache_provider = row_cache_provider
for (columnName, value_type) in column_validation_classes.items():
cfdef = self._alter_column_cfdef(cfdef, columnName, value_type)
self._system_update_column_family(cfdef)
def drop_column_family(self, keyspace, column_family):
"""
Drops a column family from the keyspace.
"""
self._conn.set_keyspace(keyspace)
self._schema_update(self._conn.system_drop_column_family, column_family)
def _alter_column_cfdef(self, cfdef, column, value_type):
if cfdef.column_type == 'Super':
packer = marshal.packer_for(cfdef.subcomparator_type)
else:
packer = marshal.packer_for(cfdef.comparator_type)
packed_column = packer(column)
value_type = self._qualify_type_class(value_type)
matched = False
for c in cfdef.column_metadata:
if c.name == packed_column:
c.validation_class = value_type
matched = True
break
if not matched:
cfdef.column_metadata.append(ColumnDef(packed_column, value_type, None, None))
return cfdef
def alter_column(self, keyspace, column_family, column, value_type):
"""
Sets a data type for the value of a specific column.
`value_type` is a string that determines what type the column value will be.
By default, :const:`LONG_TYPE`, :const:`INT_TYPE`,
:const:`ASCII_TYPE`, :const:`UTF8_TYPE`, :const:`TIME_UUID_TYPE`,
:const:`LEXICAL_UUID_TYPE` and :const:`BYTES_TYPE` are provided. Custom
types may be used as well by providing the class name; if the custom
comparator class is not in ``org.apache.cassandra.db.marshal``, the fully
qualified class name must be given.
For super column families, this sets the subcolumn value type for
any subcolumn named `column`, regardless of the super column name.
"""
self._conn.set_keyspace(keyspace)
cfdef = self.get_keyspace_column_families(keyspace)[column_family]
self._system_update_column_family(self._alter_column_cfdef(cfdef, column, value_type))
def create_index(self, keyspace, column_family, column, value_type,
index_type=KEYS_INDEX, index_name=None):
"""
Creates an index on a column.
This allows efficient for index usage via
:meth:`~pycassa.columnfamily.ColumnFamily.get_indexed_slices()`
`column` specifies what column to index, and `value_type` is a string
that describes that column's value's data type; see
:meth:`alter_column()` for a full description of `value_type`.
`index_type` determines how the index will be stored internally. Currently,
:const:`KEYS_INDEX` is the only option. `index_name` is an optional name
for the index.
Example Usage:
.. code-block:: python
>>> from pycassa.system_manager import *
>>> sys = SystemManager('192.168.2.10:9160')
>>> sys.create_index('Keyspace1', 'Standard1', 'birthdate', LONG_TYPE, index_name='bday_index')
>>> sys.close
"""
self._conn.set_keyspace(keyspace)
cfdef = self.get_keyspace_column_families(keyspace)[column_family]
packer = marshal.packer_for(cfdef.comparator_type)
packed_column = packer(column)
value_type = self._qualify_type_class(value_type)
coldef = ColumnDef(packed_column, value_type, index_type, index_name)
for c in cfdef.column_metadata:
if c.name == packed_column:
cfdef.column_metadata.remove(c)
break
cfdef.column_metadata.append(coldef)
self._system_update_column_family(cfdef)
def drop_index(self, keyspace, column_family, column):
"""
Drops an index on a column.
"""
self._conn.set_keyspace(keyspace)
cfdef = self.get_keyspace_column_families(keyspace)[column_family]
matched = False
for c in cfdef.column_metadata:
if c.name == column:
c.index_type = None
c.index_name = None
matched = True
break
if matched:
self._system_update_column_family(cfdef)
def _wait_for_agreement(self):
while True:
versions = self._conn.describe_schema_versions()
# ignore unreachable nodes
live_versions = [key for key in versions.keys() if key != 'UNREACHABLE']
if len(live_versions) == 1:
break
else:
time.sleep(_SAMPLE_PERIOD)
def _schema_update(self, schema_func, *args):
"""
Call schema updates functions and properly
waits for agreement if needed.
"""
while True:
try:
schema_version = schema_func(*args)
except SchemaDisagreementException:
self._wait_for_agreement()
else:
break
return schema_version
class SystemManager(object):
"""
Lets you examine and modify schema definitions as well as get basic
information about the cluster.
This class is mainly designed to be used manually in a python shell,
not as part of a program, although it can be used that way.
All operations which modify a keyspace or column family definition
will block until the cluster reports that all nodes have accepted
the modification.
Example Usage:
.. code-block:: python
>>> from pycassa.system_manager import *
>>> sys = SystemManager('192.168.10.2:9160')
>>> sys.create_keyspace('TestKeyspace', SIMPLE_STRATEGY, {'replication_factor': '1'})
>>> sys.create_column_family('TestKeyspace', 'TestCF', super=False,
... comparator_type=LONG_TYPE)
>>> sys.alter_column_family('TestKeyspace', 'TestCF', key_cache_size=42, gc_grace_seconds=1000)
>>> sys.drop_keyspace('TestKeyspace')
>>> sys.close()
"""
def __init__(self, server='localhost:9160', credentials=None, framed_transport=True,
timeout=_DEFAULT_TIMEOUT):
self._conn = Connection(None, server, framed_transport, timeout, credentials)
def close(self):
""" Closes the underlying connection """
self._conn.close()
def get_keyspace_column_families(self, keyspace, use_dict_for_col_metadata=False):
"""
Returns a raw description of the keyspace, which is more useful for use
in programs than :meth:`describe_keyspace()`.
If `use_dict_for_col_metadata` is ``True``, the CfDef's column_metadata will
be stored as a dictionary where the keys are column names instead of a list.
Returns a dictionary of the form ``{column_family_name: CfDef}``
"""
if keyspace is None:
keyspace = self._keyspace
ks_def = self._conn.describe_keyspace(keyspace)
cf_defs = dict()
for cf_def in ks_def.cf_defs:
cf_defs[cf_def.name] = cf_def
if use_dict_for_col_metadata:
old_metadata = cf_def.column_metadata
new_metadata = dict()
for datum in old_metadata:
new_metadata[datum.name] = datum
cf_def.column_metadata = new_metadata
return cf_defs
def get_keyspace_properties(self, keyspace):
"""
Gets a keyspace's properties.
Returns a :class:`dict` with 'strategy_class' and
'strategy_options' as keys.
"""
if keyspace is None:
keyspace = self._keyspace
ks_def = self._conn.describe_keyspace(keyspace)
return {'replication_strategy': ks_def.strategy_class,
'strategy_options': ks_def.strategy_options}
def list_keyspaces(self):
""" Returns a list of all keyspace names. """
return [ks.name for ks in self._conn.describe_keyspaces()]
def describe_ring(self, keyspace):
""" Describes the Cassandra cluster """
return self._conn.describe_ring(keyspace)
def describe_cluster_name(self):
""" Gives the cluster name """
return self._conn.describe_cluster_name()
def describe_version(self):
""" Gives the server's API version """
return self._conn.describe_version()
def describe_schema_versions(self):
""" Lists what schema version each node has """
return self._conn.describe_schema_versions()
def describe_partitioner(self):
""" Gives the partitioner that the cluster is using """
part = self._conn.describe_partitioner()
return part[part.rfind('.') + 1:]
def describe_snitch(self):
""" Gives the snitch that the cluster is using """
snitch = self._conn.describe_snitch()
return snitch[snitch.rfind('.') + 1:]
def _system_add_keyspace(self, ksdef):
return self._schema_update(self._conn.system_add_keyspace, ksdef)
def _system_update_keyspace(self, ksdef):
return self._schema_update(self._conn.system_update_keyspace, ksdef)
def create_keyspace(self, name,
replication_strategy=SIMPLE_STRATEGY,
strategy_options=None, durable_writes=True, **ks_kwargs):
"""
Creates a new keyspace. Column families may be added to this keyspace
after it is created using :meth:`create_column_family()`.
`replication_strategy` determines how replicas are chosen for this keyspace.
The strategies that Cassandra provides by default
are available as :const:`SIMPLE_STRATEGY`, :const:`NETWORK_TOPOLOGY_STRATEGY`,
and :const:`OLD_NETWORK_TOPOLOGY_STRATEGY`.
`strategy_options` is a dictionary of strategy options. For
NetworkTopologyStrategy, the dictionary should look like
``{'Datacenter1': '2', 'Datacenter2': '1'}``. This maps each
datacenter (as defined in a Cassandra property file) to a replica count.
For SimpleStrategy, you can specify the replication factor as follows:
``{'replication_factor': '1'}``.
Example Usage:
.. code-block:: python
>>> from pycassa.system_manager import *
>>> sys = SystemManager('192.168.10.2:9160')
>>> # Create a SimpleStrategy keyspace
>>> sys.create_keyspace('SimpleKS', SIMPLE_STRATEGY, {'replication_factor': '1'})
>>> # Create a NetworkTopologyStrategy keyspace
>>> sys.create_keyspace('NTS_KS', NETWORK_TOPOLOGY_STRATEGY, {'DC1': '2', 'DC2': '1'})
>>> sys.close()
"""
if replication_strategy.find('.') == -1:
strategy_class = 'org.apache.cassandra.locator.%s' % replication_strategy
else:
strategy_class = replication_strategy
ksdef = KsDef(name, strategy_class=strategy_class,
strategy_options=strategy_options,
cf_defs=[],
durable_writes=durable_writes)
for k, v in ks_kwargs.iteritems():
setattr(ksdef, k, v)
self._system_add_keyspace(ksdef)
def alter_keyspace(self, keyspace, replication_strategy=None,
strategy_options=None, durable_writes=None, **ks_kwargs):
"""
Alters an existing keyspace.
.. warning:: Don't use this unless you know what you are doing.
Parameters are the same as for :meth:`create_keyspace()`.
"""
old_ksdef = self._conn.describe_keyspace(keyspace)
old_durable = getattr(old_ksdef, 'durable_writes', True)
ksdef = KsDef(name=old_ksdef.name,
strategy_class=old_ksdef.strategy_class,
strategy_options=old_ksdef.strategy_options,
cf_defs=[],
durable_writes=old_durable)
if replication_strategy is not None:
if replication_strategy.find('.') == -1:
ksdef.strategy_class = 'org.apache.cassandra.locator.%s' % replication_strategy
else:
ksdef.strategy_class = replication_strategy
if strategy_options is not None:
ksdef.strategy_options = strategy_options
if durable_writes is not None:
ksdef.durable_writes = durable_writes
for k, v in ks_kwargs.iteritems():
setattr(ksdef, k, v)
self._system_update_keyspace(ksdef)
def drop_keyspace(self, keyspace):
"""
Drops a keyspace from the cluster.
"""
self._schema_update(self._conn.system_drop_keyspace, keyspace)
def _system_add_column_family(self, cfdef):
self._conn.set_keyspace(cfdef.keyspace)
return self._schema_update(self._conn.system_add_column_family, cfdef)
def create_column_family(self, keyspace, name, column_validation_classes=None, **cf_kwargs):
"""
Creates a new column family in a given keyspace. If a value is not
supplied for any of optional parameters, Cassandra will use a reasonable
default value.
`keyspace` should be the name of the keyspace the column family will
be created in. `name` gives the name of the column family.
"""
self._conn.set_keyspace(keyspace)
cfdef = CfDef()
cfdef.keyspace = keyspace
cfdef.name = name
if cf_kwargs.pop('super', False):
cf_kwargs.setdefault('column_type', 'Super')
for k, v in cf_kwargs.iteritems():
v = self._convert_class_attrs(k, v)
setattr(cfdef, k, v)
if column_validation_classes:
for (colname, value_type) in column_validation_classes.items():
cfdef = self._alter_column_cfdef(cfdef, colname, value_type)
self._system_add_column_family(cfdef)
def _system_update_column_family(self, cfdef):
return self._schema_update(self._conn.system_update_column_family, cfdef)
def alter_column_family(self, keyspace, column_family, column_validation_classes=None, **cf_kwargs):
"""
Alters an existing column family.
Parameter meanings are the same as for :meth:`create_column_family`.
"""
self._conn.set_keyspace(keyspace)
cfdef = self.get_keyspace_column_families(keyspace)[column_family]
for k, v in cf_kwargs.iteritems():
v = self._convert_class_attrs(k, v)
setattr(cfdef, k, v)
if column_validation_classes:
for (colname, value_type) in column_validation_classes.items():
cfdef = self._alter_column_cfdef(cfdef, colname, value_type)
self._system_update_column_family(cfdef)
def drop_column_family(self, keyspace, column_family):
"""
Drops a column family from the keyspace.
"""
self._conn.set_keyspace(keyspace)
self._schema_update(self._conn.system_drop_column_family, column_family)
def _convert_class_attrs(self, attr, value):
if attr in ('comparator_type', 'subcomparator_type',
'key_validation_class', 'default_validation_class'):
return self._qualify_type_class(value)
else:
return value
def _qualify_type_class(self, classname):
if classname:
if isinstance(classname, types.CassandraType):
s = str(classname)
elif isinstance(classname, basestring):
s = classname
else:
raise TypeError(
"Column family validators and comparators " \
"must be specified as instances of " \
"pycassa.types.CassandraType subclasses or strings.")
if s.find('.') == -1:
return 'org.apache.cassandra.db.marshal.%s' % s
else:
return s
else:
return None
def _alter_column_cfdef(self, cfdef, column, value_type):
if cfdef.column_type == 'Super':
packer = marshal.packer_for(cfdef.subcomparator_type)
else:
packer = marshal.packer_for(cfdef.comparator_type)
packed_column = packer(column)
value_type = self._qualify_type_class(value_type)
cfdef.column_metadata = cfdef.column_metadata or []
matched = False
for c in cfdef.column_metadata:
if c.name == packed_column:
c.validation_class = value_type
matched = True
break
if not matched:
cfdef.column_metadata.append(ColumnDef(packed_column, value_type, None, None))
return cfdef
def alter_column(self, keyspace, column_family, column, value_type):
"""
Sets a data type for the value of a specific column.
`value_type` is a string that determines what type the column value will be.
By default, :const:`LONG_TYPE`, :const:`INT_TYPE`,
:const:`ASCII_TYPE`, :const:`UTF8_TYPE`, :const:`TIME_UUID_TYPE`,
:const:`LEXICAL_UUID_TYPE` and :const:`BYTES_TYPE` are provided. Custom
types may be used as well by providing the class name; if the custom
comparator class is not in ``org.apache.cassandra.db.marshal``, the fully
qualified class name must be given.
For super column families, this sets the subcolumn value type for
any subcolumn named `column`, regardless of the super column name.
"""
self._conn.set_keyspace(keyspace)
cfdef = self.get_keyspace_column_families(keyspace)[column_family]
self._system_update_column_family(self._alter_column_cfdef(cfdef, column, value_type))
def create_index(self, keyspace, column_family, column, value_type,
index_type=KEYS_INDEX, index_name=None):
"""
Creates an index on a column.
This allows efficient for index usage via
:meth:`~pycassa.columnfamily.ColumnFamily.get_indexed_slices()`
`column` specifies what column to index, and `value_type` is a string
that describes that column's value's data type; see
:meth:`alter_column()` for a full description of `value_type`.
`index_type` determines how the index will be stored internally. Currently,
:const:`KEYS_INDEX` is the only option. `index_name` is an optional name
for the index.
Example Usage:
.. code-block:: python
>>> from pycassa.system_manager import *
>>> sys = SystemManager('192.168.2.10:9160')
>>> sys.create_index('Keyspace1', 'Standard1', 'birthdate', LONG_TYPE, index_name='bday_index')
>>> sys.close
"""
self._conn.set_keyspace(keyspace)
cfdef = self.get_keyspace_column_families(keyspace)[column_family]
packer = marshal.packer_for(cfdef.comparator_type)
packed_column = packer(column)
value_type = self._qualify_type_class(value_type)
coldef = ColumnDef(packed_column, value_type, index_type, index_name)
for c in cfdef.column_metadata:
if c.name == packed_column:
cfdef.column_metadata.remove(c)
break
cfdef.column_metadata.append(coldef)
self._system_update_column_family(cfdef)
def drop_index(self, keyspace, column_family, column):
"""
Drops an index on a column.
"""
self._conn.set_keyspace(keyspace)
cfdef = self.get_keyspace_column_families(keyspace)[column_family]
matched = False
for c in cfdef.column_metadata:
if c.name == column:
c.index_type = None
c.index_name = None
matched = True
break
if matched:
self._system_update_column_family(cfdef)
def _wait_for_agreement(self):
while True:
versions = self._conn.describe_schema_versions()
# ignore unreachable nodes
live_versions = [key for key in versions.keys() if key != 'UNREACHABLE']
if len(live_versions) == 1:
break
else:
time.sleep(_SAMPLE_PERIOD)
def _schema_update(self, schema_func, *args):
"""
Call schema updates functions and properly
waits for agreement if needed.
"""
while True:
try:
schema_version = schema_func(*args)
except SchemaDisagreementException:
self._wait_for_agreement()
else:
break
return schema_version
class SystemManager(object):
"""
Lets you examine and modify schema definitions as well as get basic
information about the cluster.
This class is mainly designed to be used manually in a python shell,
not as part of a program, although it can be used that way.
All operations which modify a keyspace or column family definition
will block until the cluster reports that all nodes have accepted
the modification.
Example Usage:
.. code-block:: python
>>> from pycassa.system_manager import *
>>> sys = SystemManager('192.168.10.2:9160')
>>> sys.create_keyspace('TestKeyspace', replication_factor=1)
>>> sys.create_column_family('TestKeyspace', 'TestCF', column_type='Standard',
... comparator_type=LONG_TYPE)
>>> sys.alter_column_family('TestKeyspace', 'TestCF', key_cache_size=42, gc_grace_seconds=1000)
>>> sys.drop_keyspace('TestKeyspace')
>>> sys.close()
"""
def __init__(self,
server='localhost:9160',
credentials=None,
framed_transport=True,
timeout=_DEFAULT_TIMEOUT):
self._conn = Connection(None, server, framed_transport, timeout,
credentials)
def close(self):
""" Closes the underlying connection """
self._conn.close()
def get_keyspace_column_families(self,
keyspace,
use_dict_for_col_metadata=False):
"""
Returns a raw description of the keyspace, which is more useful for use
in programs than :meth:`describe_keyspace()`.
If `use_dict_for_col_metadata` is ``True``, the CfDef's column_metadata will
be stored as a dictionary where the keys are column names instead of a list.
Returns a dictionary of the form ``{column_family_name: CfDef}``
"""
if keyspace is None:
keyspace = self._keyspace
ks_def = self._conn.describe_keyspace(keyspace)
cf_defs = dict()
for cf_def in ks_def.cf_defs:
cf_defs[cf_def.name] = cf_def
if use_dict_for_col_metadata:
old_metadata = cf_def.column_metadata
new_metadata = dict()
for datum in old_metadata:
new_metadata[datum.name] = datum
cf_def.column_metadata = new_metadata
return cf_defs
def get_keyspace_properties(self, keyspace):
"""
Gets a keyspace's properties.
Returns a :class:`dict` with 'replication_factor', 'strategy_class',
and 'strategy_options' as keys.
"""
if keyspace is None:
keyspace = self._keyspace
ks_def = self._conn.describe_keyspace(keyspace)
return {
'replication_factor': ks_def.replication_factor,
'replication_strategy': ks_def.strategy_class,
'strategy_options': ks_def.strategy_options
}
def list_keyspaces(self):
""" Returns a list of all keyspace names. """
return [ks.name for ks in self._conn.describe_keyspaces()]
def describe_ring(self, keyspace):
""" Describes the Cassandra cluster """
return self._conn.describe_ring(keyspace)
def describe_cluster_name(self):
""" Gives the cluster name """
return self._conn.describe_cluster_name()
def describe_version(self):
""" Gives the server's API version """
return self._conn.describe_version()
def describe_schema_versions(self):
""" Lists what schema version each node has """
return self._conn.describe_schema_versions()
def describe_partitioner(self):
""" Gives the partitioner that the cluster is using """
part = self._conn.describe_partitioner()
return part[part.rfind('.') + 1:]
def describe_snitch(self):
""" Gives the snitch that the cluster is using """
snitch = self._conn.describe_snitch()
return snitch[snitch.rfind('.') + 1:]
def _system_add_keyspace(self, ksdef):
schema_version = self._conn.system_add_keyspace(ksdef)
self._wait_for_agreement()
return schema_version
def _system_update_keyspace(self, ksdef):
schema_version = self._conn.system_update_keyspace(ksdef)
self._wait_for_agreement()
return schema_version
def create_keyspace(self,
name,
replication_factor=None,
replication_strategy=SIMPLE_STRATEGY,
strategy_options=None):
"""
Creates a new keyspace. Column families may be added to this keyspace
after it is created using :meth:`create_column_family()`.
`replication_factor` determines how many nodes hold a copy of a given
row. Note that for Cassandra 0.8, `replication_factor` should be an entry
in the strategy_options :class:`dict`. For example:
.. code-block:: python
>>> sys.create_keyspace('KS', None, SIMPLE_STRATEGY, {'replication_factor': '1'})
However, pycassa will copy the value from `replication_factor` or `strategy_options`
to the other as needed, depending on the version of Cassandra.
`replication_strategy` determines how replicas are chosen for this keyspace.
The strategies that Cassandra provides by default
are available as :const:`SIMPLE_STRATEGY`, :const:`NETWORK_TOPOLOGY_STRATEGY`,
and :const:`OLD_NETWORK_TOPOLOGY_STRATEGY`. `NETWORK_TOPOLOGY_STRATEGY` requires
`strategy_options` to be present.
`strategy_options` is an optional dictionary of strategy options. By default, these
are only used by NetworkTopologyStrategy; in this case, the dictionary should
look like: ``{'Datacenter1': '2', 'Datacenter2': '1'}``. This maps each
datacenter (as defined in a Cassandra property file) to a replica count.
Example Usage:
.. code-block:: python
>>> from pycassa.system_manager import *
>>> sys = SystemManager('192.168.10.2:9160')
>>> # Create a SimpleStrategy keyspace
>>> sys.create_keyspace('SimpleKS', 1)
>>> # Create a NetworkTopologyStrategy keyspace
>>> sys.create_keyspace('NTS_KS', 3, NETWORK_TOPOLOGY_STRATEGY, {'DC1': '2', 'DC2': '1'})
>>> sys.close()
"""
if replication_strategy.find('.') == -1:
strategy_class = 'org.apache.cassandra.locator.%s' % replication_strategy
else:
strategy_class = replication_strategy
ksdef = KsDef(name, strategy_class, strategy_options,
replication_factor, [])
if self._conn.version == CASSANDRA_08:
ksdef = ksdef.to08()
else:
ksdef = ksdef.to07()
self._system_add_keyspace(ksdef)
def alter_keyspace(self,
keyspace,
replication_factor=None,
replication_strategy=None,
strategy_options=None):
"""
Alters an existing keyspace.
.. warning:: Don't use this unless you know what you are doing.
Parameters are the same as for :meth:`create_keyspace()`.
"""
old_ksdef = self._conn.describe_keyspace(keyspace)
ksdef = KsDef(name=old_ksdef.name,
strategy_class=old_ksdef.strategy_class,
strategy_options=old_ksdef.strategy_options,
replication_factor=old_ksdef.replication_factor,
cf_defs=[])
if replication_strategy is not None:
if replication_strategy.find('.') == -1:
ksdef.strategy_class = 'org.apache.cassandra.locator.%s' % replication_strategy
else:
ksdef.strategy_class = replication_strategy
if strategy_options is not None:
ksdef.strategy_options = strategy_options
if replication_factor is not None:
ksdef.replication_factor = replication_factor
if self._conn.version == CASSANDRA_08:
ksdef = ksdef.to08()
else:
ksdef = ksdef.to07()
self._system_update_keyspace(ksdef)
def drop_keyspace(self, keyspace):
"""
Drops a keyspace from the cluster.
"""
schema_version = self._conn.system_drop_keyspace(keyspace)
self._wait_for_agreement()
def _system_add_column_family(self, cfdef):
self._conn.set_keyspace(cfdef.keyspace)
schema_version = self._conn.system_add_column_family(cfdef)
self._wait_for_agreement()
return schema_version
def create_column_family(self,
keyspace,
name,
super=False,
comparator_type=None,
subcomparator_type=None,
key_cache_size=None,
row_cache_size=None,
gc_grace_seconds=None,
read_repair_chance=None,
default_validation_class=None,
key_validation_class=None,
min_compaction_threshold=None,
max_compaction_threshold=None,
key_cache_save_period_in_seconds=None,
row_cache_save_period_in_seconds=None,
memtable_flush_after_mins=None,
memtable_throughput_in_mb=None,
memtable_operations_in_millions=None,
replicate_on_write=None,
merge_shards_chance=None,
row_cache_provider=None,
key_alias=None,
comment=None):
"""
Creates a new column family in a given keyspace. If a value is not
supplied for any of optional parameters, Cassandra will use a reasonable
default value.
:param str keyspace: what keyspace the column family will be created in
:param str name: the name of the column family
:param bool super: Whether or not this column family is a super column family
:param str comparator_type: What type the column names will be, which affects
their sort order. By default, :const:`LONG_TYPE`, :const:`INT_TYPE`,
:const:`ASCII_TYPE`, :const:`UTF8_TYPE`, :const:`TIME_UUID_TYPE`,
:const:`LEXICAL_UUID_TYPE` and :const:`BYTES_TYPE` are provided. Custom
types may be used as well by providing the class name; if the custom
comparator class is not in ``org.apache.cassandra.db.marshal``, the fully
qualified class name must be given.
:param str subcomparator_type: Like `comparator_type`, but if the column family
is a super column family, this applies to the type of the subcolumn names
:param key_cache_size: The size of the key cache, either in a percentage of
total keys (0.15, for example) or in an absolute number of
keys (20000, for example).
:param float row_cache_size: Same as `key_cache_size`, but for the row cache
:param int gc_grace_seconds: Number of seconds before tombstones are removed
:param float read_repair_chance: probability of a read repair occuring
:param str default_validation_class: the data type for all column values in the CF.
The choices for this are the same as for `comparator_type`.
:param str key_validation_class: the data type for row keys in the CF. The choices
for this are the same as for `comparator_type`.
:param int min_compaction_threshold: Number of similarly sized SSTables that must
be present before a minor compaction is scheduled. Setting to 0 disables minor
compactions.
:param int max_compaction_threshold: Number of similarly sized SSTables that must
be present before a minor compaction is performed immediately. Setting to 0
disables minor compactions.
:param int key_cache_save_period_in_seconds: How often the key cache should
be saved; this helps to avoid a cold cache on restart
:param int row_cache_save_period_in_seconds: How often the row cache should
be saved; this helps to avoid a cold cache on restart
:param int memtable_flush_after_mins: Memtables are flushed when they reach this age
:param int memtable_throughput_in_mb: Memtables are flushed when this many MBs have
been written to them
:param int memtable_operations_in_millions: Memtables are flushed when this many million
operations have been performed on them
:param bool replicate_on_write: Whether counter operations are replicated at write-time.
This should almost always be True.
:param float merge_shards_chance: The probability that counter shards will be merged.
:param str row_cache_provider: The class that will be used to store the row cache.
Defaults to ``org.apache.cassandra.cache.ConcurrentLinkedHashCacheProvider``.
:param key_alias: A "column name" to be used for the row key. This currently only
matters for CQL. The default is "KEY".
:param str comment: A human readable description
"""
self._conn.set_keyspace(keyspace)
cfdef = CfDef()
cfdef.keyspace = keyspace
cfdef.name = name
if super:
cfdef.column_type = 'Super'
def qualify_class(classname):
if classname and classname.find('.') == -1:
classname = 'org.apache.cassandra.db.marshal.%s' % classname
return classname
cfdef.comparator_type = qualify_class(comparator_type)
cfdef.subcomparator_type = qualify_class(subcomparator_type)
cfdef.default_validation_class = qualify_class(
default_validation_class)
cfdef.key_validation_class = qualify_class(key_validation_class)
cfdef.replicate_on_write = replicate_on_write
cfdef.comment = comment
cfdef.key_alias = key_alias
if row_cache_provider:
cfdef.row_cache_provider = row_cache_provider
self._cfdef_assign(key_cache_size, cfdef, 'key_cache_size')
self._cfdef_assign(row_cache_size, cfdef, 'row_cache_size')
self._cfdef_assign(gc_grace_seconds, cfdef, 'gc_grace_seconds')
self._cfdef_assign(read_repair_chance, cfdef, 'read_repair_chance')
self._cfdef_assign(min_compaction_threshold, cfdef,
'min_compaction_threshold')
self._cfdef_assign(max_compaction_threshold, cfdef,
'max_compaction_threshold')
self._cfdef_assign(key_cache_save_period_in_seconds, cfdef,
'key_cache_save_period_in_seconds')
self._cfdef_assign(row_cache_save_period_in_seconds, cfdef,
'row_cache_save_period_in_seconds')
self._cfdef_assign(memtable_flush_after_mins, cfdef,
'memtable_flush_after_mins')
self._cfdef_assign(memtable_throughput_in_mb, cfdef,
'memtable_throughput_in_mb')
self._cfdef_assign(memtable_operations_in_millions, cfdef,
'memtable_operations_in_millions')
self._cfdef_assign(merge_shards_chance, cfdef, 'merge_shards_chance')
self._system_add_column_family(cfdef)
def _cfdef_assign(self, attr, cfdef, attr_name):
if attr is not None:
if attr < 0:
self._raise_ire('%s must be non-negative' % attr_name)
else:
setattr(cfdef, attr_name, attr)
def _raise_ire(self, why):
ire = InvalidRequestException()
ire.why = why
raise ire
def _system_update_column_family(self, cfdef):
schema_version = self._conn.system_update_column_family(cfdef)
self._wait_for_agreement()
return schema_version
def alter_column_family(self,
keyspace,
column_family,
key_cache_size=None,
row_cache_size=None,
gc_grace_seconds=None,
read_repair_chance=None,
default_validation_class=None,
min_compaction_threshold=None,
max_compaction_threshold=None,
key_cache_save_period_in_seconds=None,
row_cache_save_period_in_seconds=None,
memtable_flush_after_mins=None,
memtable_throughput_in_mb=None,
memtable_operations_in_millions=None,
replicate_on_write=None,
merge_shards_chance=None,
row_cache_provider=None,
key_alias=None,
comment=None):
"""
Alters an existing column family.
Parameter meanings are the same as for :meth:`create_column_family`,
but column family attributes which may not be modified are not
included here.
"""
self._conn.set_keyspace(keyspace)
cfdef = self.get_keyspace_column_families(keyspace)[column_family]
self._cfdef_assign(key_cache_size, cfdef, 'key_cache_size')
self._cfdef_assign(row_cache_size, cfdef, 'row_cache_size')
self._cfdef_assign(gc_grace_seconds, cfdef, 'gc_grace_seconds')
self._cfdef_assign(read_repair_chance, cfdef, 'read_repair_chance')
self._cfdef_assign(min_compaction_threshold, cfdef,
'min_compaction_threshold')
self._cfdef_assign(max_compaction_threshold, cfdef,
'max_compaction_threshold')
self._cfdef_assign(key_cache_save_period_in_seconds, cfdef,
'key_cache_save_period_in_seconds')
self._cfdef_assign(row_cache_save_period_in_seconds, cfdef,
'row_cache_save_period_in_seconds')
self._cfdef_assign(memtable_flush_after_mins, cfdef,
'memtable_flush_after_mins')
self._cfdef_assign(memtable_throughput_in_mb, cfdef,
'memtable_throughput_in_mb')
self._cfdef_assign(memtable_operations_in_millions, cfdef,
'memtable_operations_in_millions')
self._cfdef_assign(merge_shards_chance, cfdef, 'merge_shards_chance')
cfdef.replicate_on_write = replicate_on_write
cfdef.comment = comment
cfdef.key_alias = key_alias
if row_cache_provider:
cfdef.row_cache_provider = row_cache_provider
self._system_update_column_family(cfdef)
def drop_column_family(self, keyspace, column_family):
"""
Drops a column family from the keyspace.
"""
self._conn.set_keyspace(keyspace)
schema_version = self._conn.system_drop_column_family(column_family)
self._wait_for_agreement()
def alter_column(self, keyspace, column_family, column, value_type):
"""
Sets a data type for the value of a specific column.
`value_type` is a string that determines what type the column value will be.
By default, :const:`LONG_TYPE`, :const:`INT_TYPE`,
:const:`ASCII_TYPE`, :const:`UTF8_TYPE`, :const:`TIME_UUID_TYPE`,
:const:`LEXICAL_UUID_TYPE` and :const:`BYTES_TYPE` are provided. Custom
types may be used as well by providing the class name; if the custom
comparator class is not in ``org.apache.cassandra.db.marshal``, the fully
qualified class name must be given.
For super column families, this sets the subcolumn value type for
any subcolumn named `column`, regardless of the super column name.
"""
self._conn.set_keyspace(keyspace)
cfdef = self.get_keyspace_column_families(keyspace)[column_family]
if cfdef.column_type == 'Super':
col_name_data_type = util.extract_type_name(
cfdef.subcomparator_type)
else:
col_name_data_type = util.extract_type_name(cfdef.comparator_type)
packed_column = util.pack(column, col_name_data_type)
if value_type.find('.') == -1:
value_type = 'org.apache.cassandra.db.marshal.%s' % value_type
matched = False
for c in cfdef.column_metadata:
if c.name == packed_column:
c.validation_class = value_type
matched = True
break
if not matched:
cfdef.column_metadata.append(
ColumnDef(packed_column, value_type, None, None))
self._system_update_column_family(cfdef)
def create_index(self,
keyspace,
column_family,
column,
value_type,
index_type=KEYS_INDEX,
index_name=None):
"""
Creates an index on a column.
This allows efficient for index usage via
:meth:`~pycassa.columnfamily.ColumnFamily.get_indexed_slices()`
`column` specifies what column to index, and `value_type` is a string
that describes that column's value's data type; see
:meth:`alter_column()` for a full description of `value_type`.
`index_type` determines how the index will be stored internally. Currently,
:const:`KEYS_INDEX` is the only option. `index_name` is an optional name
for the index.
Example Usage:
.. code-block:: python
>>> from pycassa.system_manager import *
>>> sys = SystemManager('192.168.2.10:9160')
>>> sys.create_index('Keyspace1', 'Standard1', 'birthdate', LONG_TYPE, index_name='bday_index')
>>> sys.close
"""
self._conn.set_keyspace(keyspace)
cfdef = self.get_keyspace_column_families(keyspace)[column_family]
col_name_data_type = util.extract_type_name(cfdef.comparator_type)
packed_column = util.pack(column, col_name_data_type)
if value_type.find('.') == -1:
value_type = 'org.apache.cassandra.db.marshal.%s' % value_type
coldef = ColumnDef(packed_column, value_type, index_type, index_name)
for c in cfdef.column_metadata:
if c.name == packed_column:
cfdef.column_metadata.remove(c)
break
cfdef.column_metadata.append(coldef)
self._system_update_column_family(cfdef)
def drop_index(self, keyspace, column_family, column):
"""
Drops an index on a column.
"""
self._conn.set_keyspace(keyspace)
cfdef = self.get_keyspace_column_families(keyspace)[column_family]
matched = False
for c in cfdef.column_metadata:
if c.name == column:
c.index_type = None
c.index_name = None
matched = True
break
if matched:
self._system_update_column_family(cfdef)
def _wait_for_agreement(self):
while True:
versions = self._conn.describe_schema_versions()
if len(versions) == 1:
break
time.sleep(_SAMPLE_PERIOD)
class SystemManager(object):
"""
Lets you examine and modify schema definitions as well as get basic
information about the cluster.
This class is mainly designed to be used manually in a python shell,
not as part of a program, although it can be used that way.
All operations which modify a keyspace or column family definition
will block until the cluster reports that all nodes have accepted
the modification.
Example Usage:
.. code-block:: python
>>> from pycassa.system_manager import *
>>> sys = SystemManager('192.168.10.2:9160')
>>> sys.create_keyspace('TestKeyspace', replication_factor=1)
>>> sys.create_column_family('TestKeyspace', 'TestCF', column_type='Standard',
... comparator_type=LONG_TYPE)
>>> sys.alter_column_family('TestKeyspace', 'TestCF', key_cache_size=42, gc_grace_seconds=1000)
>>> sys.drop_keyspace('TestKeyspace')
>>> sys.close()
"""
def __init__(self, server='localhost:9160', credentials=None, framed_transport=True,
timeout=_DEFAULT_TIMEOUT):
self._conn = Connection(None, server, framed_transport, timeout, credentials)
def close(self):
""" Closes the underlying connection """
self._conn.close()
def get_keyspace_column_families(self, keyspace, use_dict_for_col_metadata=False):
"""
Returns a raw description of the keyspace, which is more useful for use
in programs than :meth:`describe_keyspace()`.
If `use_dict_for_col_metadata` is ``True``, the CfDef's column_metadata will
be stored as a dictionary where the keys are column names instead of a list.
Returns a dictionary of the form ``{column_family_name: CfDef}``
"""
if keyspace is None:
keyspace = self._keyspace
ks_def = self._conn.describe_keyspace(keyspace)
cf_defs = dict()
for cf_def in ks_def.cf_defs:
cf_defs[cf_def.name] = cf_def
if use_dict_for_col_metadata:
old_metadata = cf_def.column_metadata
new_metadata = dict()
for datum in old_metadata:
new_metadata[datum.name] = datum
cf_def.column_metadata = new_metadata
return cf_defs
def get_keyspace_properties(self, keyspace):
"""
Gets a keyspace's properties.
Returns a :class:`dict` with 'replication_factor', 'strategy_class',
and 'strategy_options' as keys.
"""
if keyspace is None:
keyspace = self._keyspace
ks_def = self._conn.describe_keyspace(keyspace)
return {'replication_factor': ks_def.replication_factor,
'replication_strategy': ks_def.strategy_class,
'strategy_options': ks_def.strategy_options}
def list_keyspaces(self):
""" Returns a list of all keyspace names. """
return [ks.name for ks in self._conn.describe_keyspaces()]
def describe_ring(self, keyspace):
""" Describes the Cassandra cluster """
return self._conn.describe_ring(keyspace)
def describe_cluster_name(self):
""" Gives the cluster name """
return self._conn.describe_cluster_name()
def describe_version(self):
""" Gives the server's API version """
return self._conn.describe_version()
def describe_schema_versions(self):
""" Lists what schema version each node has """
return self._conn.describe_schema_versions()
def describe_partitioner(self):
""" Gives the partitioner that the cluster is using """
part = self._conn.describe_partitioner()
return part[part.rfind('.') + 1: ]
def describe_snitch(self):
""" Gives the snitch that the cluster is using """
snitch = self._conn.describe_snitch()
return snitch[snitch.rfind('.') + 1: ]
def _system_add_keyspace(self, ksdef):
schema_version = self._conn.system_add_keyspace(ksdef)
self._wait_for_agreement()
return schema_version
def _system_update_keyspace(self, ksdef):
schema_version = self._conn.system_update_keyspace(ksdef)
self._wait_for_agreement()
return schema_version
def create_keyspace(self, name, replication_factor=None,
replication_strategy=SIMPLE_STRATEGY,
strategy_options=None):
"""
Creates a new keyspace. Column families may be added to this keyspace
after it is created using :meth:`create_column_family()`.
`replication_factor` determines how many nodes hold a copy of a given
row. Note that for Cassandra 0.8, `replication_factor` should be an entry
in the strategy_options :class:`dict`. For example:
.. code-block:: python
>>> sys.create_keyspace('KS', None, SIMPLE_STRATEGY, {'replication_factor': '1'})
However, pycassa will copy the value from `replication_factor` or `strategy_options`
to the other as needed, depending on the version of Cassandra.
`replication_strategy` determines how replicas are chosen for this keyspace.
The strategies that Cassandra provides by default
are available as :const:`SIMPLE_STRATEGY`, :const:`NETWORK_TOPOLOGY_STRATEGY`,
and :const:`OLD_NETWORK_TOPOLOGY_STRATEGY`. `NETWORK_TOPOLOGY_STRATEGY` requires
`strategy_options` to be present.
`strategy_options` is an optional dictionary of strategy options. By default, these
are only used by NetworkTopologyStrategy; in this case, the dictionary should
look like: ``{'Datacenter1': '2', 'Datacenter2': '1'}``. This maps each
datacenter (as defined in a Cassandra property file) to a replica count.
Example Usage:
.. code-block:: python
>>> from pycassa.system_manager import *
>>> sys = SystemManager('192.168.10.2:9160')
>>> # Create a SimpleStrategy keyspace
>>> sys.create_keyspace('SimpleKS', 1)
>>> # Create a NetworkTopologyStrategy keyspace
>>> sys.create_keyspace('NTS_KS', 3, NETWORK_TOPOLOGY_STRATEGY, {'DC1': '2', 'DC2': '1'})
>>> sys.close()
"""
if replication_strategy.find('.') == -1:
strategy_class = 'org.apache.cassandra.locator.%s' % replication_strategy
else:
strategy_class = replication_strategy
ksdef = KsDef(name, strategy_class, strategy_options, replication_factor, [])
if self._conn.version == CASSANDRA_08:
ksdef = ksdef.to08()
else:
ksdef = ksdef.to07()
self._system_add_keyspace(ksdef)
def alter_keyspace(self, keyspace, replication_factor=None,
replication_strategy=None,
strategy_options=None):
"""
Alters an existing keyspace.
.. warning:: Don't use this unless you know what you are doing.
Parameters are the same as for :meth:`create_keyspace()`.
"""
old_ksdef = self._conn.describe_keyspace(keyspace)
ksdef = KsDef(name=old_ksdef.name,
strategy_class=old_ksdef.strategy_class,
strategy_options=old_ksdef.strategy_options,
replication_factor=old_ksdef.replication_factor,
cf_defs=[])
if replication_strategy is not None:
if replication_strategy.find('.') == -1:
ksdef.strategy_class = 'org.apache.cassandra.locator.%s' % replication_strategy
else:
ksdef.strategy_class = replication_strategy
if strategy_options is not None:
ksdef.strategy_options = strategy_options
if replication_factor is not None:
ksdef.replication_factor = replication_factor
if self._conn.version == CASSANDRA_08:
ksdef = ksdef.to08()
else:
ksdef = ksdef.to07()
self._system_update_keyspace(ksdef)
def drop_keyspace(self, keyspace):
"""
Drops a keyspace from the cluster.
"""
schema_version = self._conn.system_drop_keyspace(keyspace)
self._wait_for_agreement()
def _system_add_column_family(self, cfdef):
self._conn.set_keyspace(cfdef.keyspace)
schema_version = self._conn.system_add_column_family(cfdef)
self._wait_for_agreement()
return schema_version
def create_column_family(self, keyspace, name, super=False,
comparator_type=None,
subcomparator_type=None,
key_cache_size=None,
row_cache_size=None,
gc_grace_seconds=None,
read_repair_chance=None,
default_validation_class=None,
min_compaction_threshold=None,
max_compaction_threshold=None,
key_cache_save_period_in_seconds=None,
row_cache_save_period_in_seconds=None,
memtable_flush_after_mins=None,
memtable_throughput_in_mb=None,
memtable_operations_in_millions=None,
comment=None):
"""
Creates a new column family in a given keyspace. If a value is not
supplied for any of optional parameters, Cassandra will use a reasonable
default value.
:param str keyspace: what keyspace the column family will be created in
:param str name: the name of the column family
:param bool super: Whether or not this column family is a super column family
:param str comparator_type: What type the column names will be, which affects
their sort order. By default, :const:`LONG_TYPE`, :const:`INT_TYPE`,
:const:`ASCII_TYPE`, :const:`UTF8_TYPE`, :const:`TIME_UUID_TYPE`,
:const:`LEXICAL_UUID_TYPE` and :const:`BYTES_TYPE` are provided. Custom
types may be used as well by providing the class name; if the custom
comparator class is not in ``org.apache.cassandra.db.marshal``, the fully
qualified class name must be given.
:param str subcomparator_type: Like `comparator_type`, but if the column family
is a super column family, this applies to the type of the subcolumn names
:param key_cache_size: The size of the key cache, either in a percentage of
total keys (0.15, for example) or in an absolute number of
keys (20000, for example).
:param float row_cache_size: Same as `key_cache_size`, but for the row cache
:param int gc_grace_seconds: Number of seconds before tombstones are removed
:param float read_repair_chance: probability of a read repair occuring
:param str default_validation_class: the data type for all column values in the CF.
the choices for this are the same as for `comparator_type`.
:param int min_compaction_threshold: Number of similarly sized SSTables that must
be present before a minor compaction is scheduled. Setting to 0 disables minor
compactions.
:param int max_compaction_threshold: Number of similarly sized SSTables that must
be present before a minor compaction is performed immediately. Setting to 0
disables minor compactions.
:param int key_cache_save_period_in_seconds: How often the key cache should
be saved; this helps to avoid a cold cache on restart
:param int row_cache_save_period_in_seconds: How often the row cache should
be saved; this helps to avoid a cold cache on restart
:param int memtable_flush_after_mins: Memtables are flushed when they reach this age
:param int memtable_throughput_in_mb: Memtables are flushed when this many MBs have
been written to them
:param int memtable_operations_in_millions: Memtables are flushed when this many million
operations have been performed on them
:param str comment: A human readable description
"""
self._conn.set_keyspace(keyspace)
cfdef = CfDef()
cfdef.keyspace = keyspace
cfdef.name = name
if super:
cfdef.column_type = 'Super'
if comparator_type is not None:
if comparator_type.find('.') == -1:
cfdef.comparator_type = 'org.apache.cassandra.db.marshal.%s' % comparator_type
else:
cfdef.comparator_type = comparator_type
if subcomparator_type is not None:
if cfdef.column_type != 'Super':
self._raise_ire('subcomparator_type may only be used for super column families')
if subcomparator_type.find('.') == -1:
cfdef.subcomparator_type = 'org.apache.cassandra.db.marshal.%s' % subcomparator_type
else:
cfdef.subcomparator_type = subcomparator_type
if default_validation_class is not None:
if default_validation_class.find('.') == -1:
cfdef.default_validation_class = 'org.apache.cassandra.db.marshal.%s' % default_validation_class
else:
cfdef.default_validation_class = default_validation_class
if comment is not None:
cfdef.comment = comment
self._cfdef_assign(key_cache_size, cfdef, 'key_cache_size')
self._cfdef_assign(row_cache_size, cfdef, 'row_cache_size')
self._cfdef_assign(gc_grace_seconds, cfdef, 'gc_grace_seconds')
self._cfdef_assign(read_repair_chance, cfdef, 'read_repair_chance')
self._cfdef_assign(min_compaction_threshold, cfdef, 'min_compaction_threshold')
self._cfdef_assign(max_compaction_threshold, cfdef, 'max_compaction_threshold')
self._cfdef_assign(key_cache_save_period_in_seconds, cfdef, 'key_cache_save_period_in_seconds')
self._cfdef_assign(row_cache_save_period_in_seconds, cfdef, 'row_cache_save_period_in_seconds')
self._cfdef_assign(memtable_flush_after_mins, cfdef, 'memtable_flush_after_mins')
self._cfdef_assign(memtable_throughput_in_mb, cfdef, 'memtable_throughput_in_mb')
self._cfdef_assign(memtable_operations_in_millions, cfdef, 'memtable_operations_in_millions')
self._system_add_column_family(cfdef)
def _cfdef_assign(self, attr, cfdef, attr_name):
if attr is not None:
if attr < 0:
self._raise_ire('%s must be non-negative' % attr_name)
else:
setattr(cfdef, attr_name, attr)
def _raise_ire(self, why):
ire = InvalidRequestException()
ire.why = why
raise ire
def _system_update_column_family(self, cfdef):
schema_version = self._conn.system_update_column_family(cfdef)
self._wait_for_agreement()
return schema_version
def alter_column_family(self, keyspace, column_family,
key_cache_size=None,
row_cache_size=None,
gc_grace_seconds=None,
read_repair_chance=None,
default_validation_class=None,
min_compaction_threshold=None,
max_compaction_threshold=None,
key_cache_save_period_in_seconds=None,
row_cache_save_period_in_seconds=None,
memtable_flush_after_mins=None,
memtable_throughput_in_mb=None,
memtable_operations_in_millions=None,
comment=None):
"""
Alters an existing column family.
Parameter meanings are the same as for :meth:`create_column_family`,
but column family attributes which may not be modified are not
included here.
"""
self._conn.set_keyspace(keyspace)
cfdef = self.get_keyspace_column_families(keyspace)[column_family]
self._cfdef_assign(key_cache_size, cfdef, 'key_cache_size')
self._cfdef_assign(row_cache_size, cfdef, 'row_cache_size')
self._cfdef_assign(gc_grace_seconds, cfdef, 'gc_grace_seconds')
self._cfdef_assign(read_repair_chance, cfdef, 'read_repair_chance')
self._cfdef_assign(min_compaction_threshold, cfdef, 'min_compaction_threshold')
self._cfdef_assign(max_compaction_threshold, cfdef, 'max_compaction_threshold')
self._cfdef_assign(key_cache_save_period_in_seconds, cfdef, 'key_cache_save_period_in_seconds')
self._cfdef_assign(row_cache_save_period_in_seconds, cfdef, 'row_cache_save_period_in_seconds')
self._cfdef_assign(memtable_flush_after_mins, cfdef, 'memtable_flush_after_mins')
self._cfdef_assign(memtable_throughput_in_mb, cfdef, 'memtable_throughput_in_mb')
self._cfdef_assign(memtable_operations_in_millions, cfdef, 'memtable_operations_in_millions')
if comment is not None:
cfdef.comment = comment
self._system_update_column_family(cfdef)
def drop_column_family(self, keyspace, column_family):
"""
Drops a column family from the keyspace.
"""
self._conn.set_keyspace(keyspace)
schema_version = self._conn.system_drop_column_family(column_family)
self._wait_for_agreement()
def alter_column(self, keyspace, column_family, column, value_type):
"""
Sets a data type for the value of a specific column.
`value_type` is a string that determines what type the column value will be.
By default, :const:`LONG_TYPE`, :const:`INT_TYPE`,
:const:`ASCII_TYPE`, :const:`UTF8_TYPE`, :const:`TIME_UUID_TYPE`,
:const:`LEXICAL_UUID_TYPE` and :const:`BYTES_TYPE` are provided. Custom
types may be used as well by providing the class name; if the custom
comparator class is not in ``org.apache.cassandra.db.marshal``, the fully
qualified class name must be given.
For super column families, this sets the subcolumn value type for
any subcolumn named `column`, regardless of the super column name.
"""
self._conn.set_keyspace(keyspace)
cfdef = self.get_keyspace_column_families(keyspace)[column_family]
if cfdef.column_type == 'Super':
col_name_data_type = util.extract_type_name(cfdef.subcomparator_type)
else:
col_name_data_type = util.extract_type_name(cfdef.comparator_type)
packed_column = util.pack(column, col_name_data_type)
if value_type.find('.') == -1:
value_type = 'org.apache.cassandra.db.marshal.%s' % value_type
matched = False
for c in cfdef.column_metadata:
if c.name == packed_column:
c.validation_class = value_type
matched = True
break
if not matched:
cfdef.column_metadata.append(ColumnDef(packed_column, value_type, None, None))
self._system_update_column_family(cfdef)
def create_index(self, keyspace, column_family, column, value_type,
index_type=KEYS_INDEX, index_name=None):
"""
Creates an index on a column.
This allows efficient for index usage via
:meth:`~pycassa.columnfamily.ColumnFamily.get_indexed_slices()`
`column` specifies what column to index, and `value_type` is a string
that describes that column's value's data type; see
:meth:`alter_column()` for a full description of `value_type`.
`index_type` determines how the index will be stored internally. Currently,
:const:`KEYS_INDEX` is the only option. `index_name` is an optional name
for the index.
Example Usage:
.. code-block:: python
>>> from pycassa.system_manager import *
>>> sys = SystemManager('192.168.2.10:9160')
>>> sys.create_index('Keyspace1', 'Standard1', 'birthdate', LONG_TYPE, index_name='bday_index')
>>> sys.close
"""
self._conn.set_keyspace(keyspace)
cfdef = self.get_keyspace_column_families(keyspace)[column_family]
col_name_data_type = util.extract_type_name(cfdef.comparator_type)
packed_column = util.pack(column, col_name_data_type)
if value_type.find('.') == -1:
value_type = 'org.apache.cassandra.db.marshal.%s' % value_type
coldef = ColumnDef(packed_column, value_type, index_type, index_name)
for c in cfdef.column_metadata:
if c.name == packed_column:
cfdef.column_metadata.remove(c)
break
cfdef.column_metadata.append(coldef)
self._system_update_column_family(cfdef)
def drop_index(self, keyspace, column_family, column):
"""
Drops an index on a column.
"""
self._conn.set_keyspace(keyspace)
cfdef = self.get_keyspace_column_families(keyspace)[column_family]
matched = False
for c in cfdef.column_metadata:
if c.name == column:
c.index_type = None
c.index_name = None
matched = True
break
if matched:
self._system_update_column_family(cfdef)
def _wait_for_agreement(self):
while True:
versions = self._conn.describe_schema_versions()
if len(versions) == 1:
break
time.sleep(_SAMPLE_PERIOD)