def __init__(self, controller_name, host, port, username, password, keyspace, instance_factory, sensor_item_factory, prefix=''): self.controller_name = str(controller_name) # keep a set of known instances and sensors so we can save on # unnecessary inserts to the controller instance/sensor lists self.seen_instances = set() self.seen_sensors = set() self.instance_factory = instance_factory self.sensor_item_factory = sensor_item_factory authorization_dictionary = {'username': username, 'password': password} self.manager = ManagedCassandraClientFactory( credentials=authorization_dictionary, check_api_version=True, keyspace=keyspace) TCPConnection.__init__(self, host, port, self.manager) self.client = CassandraClient(self.manager) self.instance_cf = prefix + self.INSTANCE_CF_NAME self.instance_id_cf = prefix + self.INSTANCE_ID_CF_NAME self.sensor_cf = prefix + self.SENSOR_CF_NAME self.sensor_id_cf = prefix + self.SENSOR_ID_CF_NAME self.config_cf = prefix + self.CONFIG_CF_NAME
def do_this(): for gzfilename in sys.argv[1:]: n = 0 with gzip.GzipFile(gzfilename) as f: csv_file = csv.DictReader(f) first = True last_key = "" for row in csv_file: if first: factory = ManagedCassandraClientFactory(row["keyspace"]) reactor.connectTCP("localhost", 9160, factory) client = CassandraClient(factory) first = False yield client.insert( column_family=row["cf"], key=uuidify(row["key"].decode("string_escape")), column=row["col"], value=uuidify(row["val"].decode("string_escape")), ) if last_key != row["key"].decode("string_escape"): n += 1 last_key = row["key"].decode("string_escape") print "Successfully restored %d rows from %s" % (n, gzfilename) reactor.stop()
def test_api_match(self): cmanager = ManagedCassandraClientFactory(require_api_version=None) client = CassandraClient(cmanager) d = cmanager.deferred conn = reactor.connectTCP(HOST, PORT, cmanager) yield d yield client.describe_schema_versions() api_ver = cmanager._protos[0].api_version yield cmanager.shutdown() # try with the right version explicitly required cmanager = ManagedCassandraClientFactory(require_api_version=api_ver) client = CassandraClient(cmanager) d = cmanager.deferred conn = reactor.connectTCP(HOST, PORT, cmanager) yield d yield client.describe_schema_versions() yield cmanager.shutdown() # try with a mismatching version explicitly required bad_ver = [v for (_, v) in translate.supported_versions if v != api_ver][0] cmanager = ManagedCassandraClientFactory(require_api_version=bad_ver) client = CassandraClient(cmanager) d = cmanager.deferred conn = reactor.connectTCP(HOST, PORT, cmanager) yield self.assertFailure(d, translate.APIMismatch) yield cmanager.shutdown()
def initialize(self, factory_name, table, column): """ The factory_name, table and column are set as part of the Application router, see api/__init__.py The table corresponds to the cassandra table, while the column specifies the cassandra column to operate on The row to operate on is passed to each function, while the value is in the request body, if relevant """ self.table = table self.column = column self.cass = CassandraClient(self.cass_factories[factory_name])
def test_api_match(self): for version in [translate.CASSANDRA_07_VERSION, translate.CASSANDRA_08_VERSION, None]: cmanager = ManagedCassandraClientFactory(api_version=version) client = CassandraClient(cmanager) d = cmanager.deferred conn = reactor.connectTCP(HOST, PORT, cmanager) yield d # do something innocuous, make sure connection is good yield client.describe_schema_versions() yield cmanager.shutdown()
def test_api_match(self): cmanager = ManagedCassandraClientFactory(require_api_version=None) client = CassandraClient(cmanager) d = cmanager.deferred conn = reactor.connectTCP(HOST, PORT, cmanager) yield d yield client.describe_schema_versions() api_ver = cmanager._protos[0].api_version yield cmanager.shutdown() # try with the right version explicitly required cmanager = ManagedCassandraClientFactory(require_api_version=api_ver) client = CassandraClient(cmanager) d = cmanager.deferred conn = reactor.connectTCP(HOST, PORT, cmanager) yield d yield client.describe_schema_versions() yield cmanager.shutdown() # try with a mismatching version explicitly required bad_ver = [ v for (_, v) in translate.supported_versions if v != api_ver ][0] cmanager = ManagedCassandraClientFactory(require_api_version=bad_ver) client = CassandraClient(cmanager) d = cmanager.deferred conn = reactor.connectTCP(HOST, PORT, cmanager) yield self.assertFailure(d, translate.APIMismatch) yield cmanager.shutdown()
def __init__(self, host, port, username, password, keyspace, prefix=''): self._launch_column_family = prefix + self.LAUNCH_CF_NAME self._node_column_family = prefix + self.NODE_CF_NAME authz = {'username': username, 'password': password} self._manager = ManagedCassandraClientFactory(credentials=authz, check_api_version=True, keyspace=keyspace) self.client = CassandraClient(self._manager) self._host = host self._port = port self._connector = None
def setUp(self): self.cmanager = ManagedCassandraClientFactory(keyspace='system') self.client = CassandraClient(self.cmanager) for i in xrange(CONNS): reactor.connectTCP(HOST, PORT, self.cmanager) yield self.cmanager.deferred remote_ver = yield self.client.describe_version() self.version = thrift_api_ver_to_cassandra_ver(remote_ver) self.my_keyspace = KsDef( name=KEYSPACE, strategy_class='org.apache.cassandra.locator.SimpleStrategy', strategy_options={'replication_factor': '1'}, cf_defs=[ CfDef(keyspace=KEYSPACE, name=CF, column_type='Standard'), CfDef(keyspace=KEYSPACE, name=SCF, column_type='Super'), CfDef( keyspace=KEYSPACE, name=IDX_CF, column_type='Standard', comparator_type='org.apache.cassandra.db.marshal.UTF8Type', column_metadata=[ ColumnDef(name='col1', validation_class= 'org.apache.cassandra.db.marshal.UTF8Type', index_type=IndexType.KEYS, index_name='idxCol1') ], default_validation_class= 'org.apache.cassandra.db.marshal.BytesType'), ]) if self.version == CASSANDRA_08_VERSION: self.my_keyspace.cf_defs.extend([ CfDef(keyspace=KEYSPACE, name=COUNTER_CF, column_type='Standard', default_validation_class= 'org.apache.cassandra.db.marshal.CounterColumnType'), CfDef(keyspace=KEYSPACE, name=SUPERCOUNTER_CF, column_type='Super', default_validation_class= 'org.apache.cassandra.db.marshal.CounterColumnType'), ]) yield self.client.system_add_keyspace(self.my_keyspace) yield self.client.set_keyspace(KEYSPACE)
def __init__(self, keyspace): self._keyspace = keyspace self.factory = ManagedCassandraClientFactory(keyspace) host = settings.CASS_HOST.strip() # Ensure that the host isn't blank if host == '': host = 'localhost' addresses = socket.getaddrinfo(host, settings.CASS_PORT) # If we are using IPv6, we need to provide an IPv6 address directly # to twisted, as it doesn't support IPv6. # addresses is a list of 5 tuple: # (family, socktype, proto, cannonname, sockaddr) shuffle(addresses) if len(addresses) > 1 and addresses[0][0] == socket.AF_INET6: # sockaddr is a 4 tuple: # (address, port, flow, scope) address = addresses[0][4][0] else: address = host _log.debug("Cassandra is connecting to %s - for host %s", address, settings.CASS_HOST) reactor.connectTCP(address, settings.CASS_PORT, self.factory) self.client = CassandraClient(self.factory)
class PassthroughHandler(BaseHandler): """ The passthrough handler simply takes what has been sent in from the router and reads/writes/deletes without any validation. Handlers should subclass this handler in order to do parameter validation. After validation, the handlers should call through the passthrough handler to write to the database """ def initialize(self, table, column): """ The table and column are set as part of the Application router, see api/__init__.py The table corresponds to the cassandra table, while the column specifies the cassandra column to operate on The row to operate on is passed to each function, while the value is in the request body, if relevant """ self.table = table self.column = column self.cass = CassandraClient(self.application.cassandra_factory) @defer.inlineCallbacks def get(self, row): try: result = yield self.cass.get(column_family=self.table, key=row, column=self.column) self.finish(result.column.value) except NotFoundException, e: raise HTTPError(404)
def test_initial_connection_failure(self): cmanager = ManagedCassandraClientFactory() client = CassandraClient(cmanager) d = cmanager.deferred reactor.connectTCP('nonexistent.example.com', PORT, cmanager) yield self.assertFailure(d, error.DNSLookupError) cmanager.shutdown()
def get(self): # Attempt to connect to Cassandra (by asking for a non-existent key). # We need this check as we've seen cases where telephus fails to # connect to Cassandra, and requests sit on the queue forever without # being processed. clients = (CassandraClient(factory) for factory in self.cass_factories) # If Cassandra is up, it will throw an exception (because we're asking # for a nonexistent key). That's fine - it proves Cassandra is up and # we have a connection to it. If Cassandra is down, this call will # never return and Monit will time it out and kill the process for # unresponsiveness. # Catch the Twisted error made (as 'ping' isn't a configured column) - # as with the Exception we don't care about the error, we just want to # test if we can contact Cassandra def ping_error(err): # pragma: no cover pass try: _log.debug("Handling ping request") gets = (client.get(key='ping', column_family='ping').addErrback(ping_error) for client in clients) yield defer.DeferredList(gets) except Exception: # We don't care about the result, just whether it returns # in a timely fashion. Writing a log would be spammy. pass _log.debug("Handled ping request successfully") self.finish("OK")
def connect(self, host=None, port=9160, username=None, password=None): if not host: host, port = get_host_port() if username or password: if not (username and password): raise CassandraConfigurationError( "Specify both username and password or neither") else: username, password = get_credentials() authz = dict(username=username, password=password) self.manager = ManagedCassandraClientFactory(credentials=authz, check_api_version=True) self.connector = reactor.connectTCP(host, port, self.manager) self.client = CassandraClient(self.manager)
def setUp(self): self.cmanager = ManagedCassandraClientFactory(keyspace='system') self.client = CassandraClient(self.cmanager) for i in xrange(CONNS): reactor.connectTCP(HOST, PORT, self.cmanager) yield self.cmanager.deferred remote_ver = yield self.client.describe_version() self.version = getAPIVersion(remote_ver) self.my_keyspace = KsDef( name=KEYSPACE, strategy_class='org.apache.cassandra.locator.SimpleStrategy', strategy_options={'replication_factor': '1'}, cf_defs=[ CfDef( keyspace=KEYSPACE, name=CF, column_type='Standard' ), CfDef( keyspace=KEYSPACE, name=SCF, column_type='Super' ), CfDef( keyspace=KEYSPACE, name=IDX_CF, column_type='Standard', comparator_type='org.apache.cassandra.db.marshal.UTF8Type', column_metadata=[ ColumnDef( name='col1', validation_class='org.apache.cassandra.db.marshal.UTF8Type', index_type=IndexType.KEYS, index_name='idxCol1') ], default_validation_class='org.apache.cassandra.db.marshal.BytesType' ), ] ) if self.version == CASSANDRA_08_VERSION: self.my_keyspace.cf_defs.extend([ CfDef( keyspace=KEYSPACE, name=COUNTER_CF, column_type='Standard', default_validation_class='org.apache.cassandra.db.marshal.CounterColumnType' ), CfDef( keyspace=KEYSPACE, name=SUPERCOUNTER_CF, column_type='Super', default_validation_class='org.apache.cassandra.db.marshal.CounterColumnType' ), ]) yield self.client.system_add_keyspace(self.my_keyspace) yield self.client.set_keyspace(KEYSPACE)
def test_api_check(self): cmanager = ManagedCassandraClientFactory(check_api_version=False) client = CassandraClient(cmanager) conn = reactor.connectTCP(HOST, PORT, cmanager) # we don't necessarily want to force an api match while testing; # get the remote value and pretend ours matches, even if it doesn't ver = yield client.describe_version() cmanager.shutdown() constants.VERSION = ver cmanager = ManagedCassandraClientFactory(check_api_version=True) client = CassandraClient(cmanager) d = cmanager.deferred conn = reactor.connectTCP(HOST, PORT, cmanager) yield d # do something innocuous, make sure connection is good yield client.describe_schema_versions() cmanager.shutdown()
def keyspaceConnection(self, keyspace, consistency=ConsistencyLevel.ONE): """ Return a CassandraClient instance which uses this CassandraClusterPool by way of a CassandraKeyspaceConnection, so that all requests made through it are guaranteed to go to the given keyspace, no matter what other consumers of this pool may do. """ return CassandraClient(CassandraKeyspaceConnection(self, keyspace), consistency=consistency)
def initialize(self, table, column): """ The table and column are set as part of the Application router, see api/__init__.py The table corresponds to the cassandra table, while the column specifies the cassandra column to operate on The row to operate on is passed to each function, while the value is in the request body, if relevant """ self.table = table self.column = column self.cass = CassandraClient(self.application.cassandra_factory)
def connect(keyspace='system', hosts=['localhost:9160'], connections=5, consistency=ConsistencyLevel.ONE): manager = ManagedCassandraClientFactory(keyspace=keyspace) client = CassandraClient(manager, consistency=consistency) for i in xrange(connections): for host, port in [x.split(':') for x in hosts]: reactor.connectTCP(host, int(port), manager) return client
def do_this(): for gzfilename in sys.argv[1:]: n = 0 with gzip.GzipFile(gzfilename) as f: csv_file = csv.DictReader(f) first = True last_key = '' for row in csv_file: if first: factory = ManagedCassandraClientFactory(row["keyspace"]) reactor.connectTCP("localhost", 9160, factory) client = CassandraClient(factory) first = False yield client.insert(column_family=row["cf"], key=uuidify(row["key"].decode("string_escape")), column=row["col"], value=uuidify(row["val"].decode("string_escape"))) if last_key != row["key"].decode("string_escape"): n += 1 last_key = row["key"].decode("string_escape") print "Successfully restored %d rows from %s" % (n, gzfilename) reactor.stop()
def main(): sasl_kwargs = { "host": "thobbs-laptop2", "service": "host", "mechanism": "GSSAPI"} f = ManagedCassandraClientFactory(keyspace='system', sasl_kwargs=sasl_kwargs) reactor.connectTCP(HOST, PORT, f) yield f.deferred c = CassandraClient(f) yield setup_schema(c) yield dostuff(c)
def do_this(): for keyspace in keyspaces: factory = ManagedCassandraClientFactory(keyspace) reactor.connectTCP("localhost", 9160, factory) client = CassandraClient(factory) k = yield client.describe_keyspace(keyspace) for cf in [c.name for c in k.cf_defs]: n = 0 db_rows = yield client.get_range_slices(column_family=cf, count=10000000) with gzip.GzipFile("%s.%s.csv.gz" % (keyspace, cf), "w") as f: out = csv.DictWriter(f, fieldnames=["keyspace", "cf", "key", "col", "val"]) out.writeheader() for row in db_rows: for col in row.columns: out.writerow({"keyspace": keyspace, "cf": cf, "key": row.key.encode("string_escape"), "col": col.column.name, "val": col.column.value.encode("string_escape")}) n += 1 print "Successfully backed up %d rows from %s.%s" % (n, keyspace, cf) reactor.stop()
def __init__(self, host, port, username, password, keyspace, prefix=''): self._launch_column_family = prefix + self.LAUNCH_CF_NAME self._node_column_family = prefix + self.NODE_CF_NAME authz= {'username': username, 'password': password} self._manager = ManagedCassandraClientFactory( credentials=authz, check_api_version=True, keyspace=keyspace) self.client = CassandraClient(self._manager) self._host = host self._port = port self._connector = None
def setUp(self): self.cmanager = ManagedCassandraClientFactory(keyspace='system') self.client = CassandraClient(self.cmanager) for i in xrange(CONNS): reactor.connectTCP(HOST, PORT, self.cmanager) yield self.cmanager.deferred self.my_keyspace = KsDef( name=KEYSPACE, strategy_class='org.apache.cassandra.locator.SimpleStrategy', replication_factor=1, cf_defs=[ CfDef( keyspace=KEYSPACE, name=CF, column_type='Standard' ), CfDef( keyspace=KEYSPACE, name=SCF, column_type='Super' ), CfDef( keyspace=KEYSPACE, name=IDX_CF, column_type='Standard', comparator_type='org.apache.cassandra.db.marshal.UTF8Type', column_metadata=[ ColumnDef( name='col1', validation_class='org.apache.cassandra.db.marshal.UTF8Type', index_type=IndexType.KEYS, index_name='idxCol1') ], default_validation_class='org.apache.cassandra.db.marshal.BytesType') ] ) yield self.client.system_add_keyspace(self.my_keyspace) yield self.client.set_keyspace(KEYSPACE)
def __init__(self, seed_list, keyspace=None, creds=None, thrift_port=None, pool_size=None, conn_timeout=10, bind_address=None, log_cb=log.msg, reactor=None, ssl_ctx_factory=None, sasl_cred_factory=None, auto_node_discovery=True, fill_pool_throttle=0.5): """ Initialize a CassandraClusterPool. @param keyspace: If given and not None, determines the keyspace to which all connections in this pool will be made. @param creds: Credentials to use to authenticate Cassandra connections @type creds: A dict (or other mapping) of strings to strings @param seed_list: An initial set of host addresses which, if connectable, are part of this cluster. @type seed_list: iterable @param thrift_port: The port to use for connections to Cassandra nodes @param pool_size: The target size for the connection pool. Naturally, the actual size may be higher or lower as nodes connect and disconnect, but an effort will be made to adjust toward this size. @type pool_size: int @param conn_timeout: The number of seconds before a pending connection is deemed unsuccessful and aborted. Of course, when a connection error can be detected before this time, the connection will be aborted appropriately. @type conn_timeout: float @param bind_address: The local interface to which to bind when making outbound connections. Default: determined by the system's socket layer. @type bind_address: str @param log_cb: A callable which is expected to work like L{twisted.python.log.msg}. Will be used when certain connection and disconnection events occur. The default is log.msg. @param reactor: The reactor instance to use when starting thrift connections or setting timers. @param ssl_ctx_factory: A L{twisted.internet.ssl.ClientContextFactory} instance. If not None, SSL connections will be opened using the provided context factory; if None, SSL will not be used. @param sasl_cred_factory: A callable which, when called with two parameters, a host and port, returns a dictionary of keyword arguments to be used for the L{puresasl.client.SASLClient} constructor. If supplied, the ThriftSASLClientProtocol will be used. @param auto_node_discovery: Bool that Enables/Disables node auto-discovery, defaults to True @param fill_pool_throttle: Float that indicates keeps fill_pool from being called too fast in a failure situation """ self.describing_ring = False self.seed_list = list(seed_list) if thrift_port is None: thrift_port = self.default_cassandra_thrift_port self.thrift_port = thrift_port if pool_size is None: pool_size = len(self.seed_list) self.target_pool_size = pool_size self.log = log_cb self.conn_timeout = conn_timeout self.bind_address = bind_address self.keyspace = keyspace self.creds = creds self.ssl_ctx_factory = ssl_ctx_factory self.sasl_cred_factory = sasl_cred_factory self.request_queue = defer.DeferredQueue() self.future_fill_pool = None self.removed_nodes = set() self._client_instance = CassandraClient(self) self.auto_node_discovery = auto_node_discovery self.fill_pool_throttle = fill_pool_throttle self.throttle_timer = None self.fill_pool_last_called = None if reactor is None: from twisted.internet import reactor self.reactor = reactor self.retryables.extend( (ttypes.TimedOutException, ttypes.UnavailableException)) # A set of CassandraNode instances representing known nodes. This # includes nodes from the initial seed list, nodes seen in # describe_ring calls to existing nodes, and nodes explicitly added # by the addNode() method. Nodes are only removed from this set if # no connections have been successful in self.forget_node_interval # seconds, or by an explicit call to removeNode(). self.nodes = set() # A set of CassandraPoolReconnectorFactory instances corresponding to # connections which are either live or pending. Failed attempts to # connect will remove a connector from this set. When connections are # lost, an immediate reconnect will be attempted. self.connectors = set() # A collection of objects from self.connectors corresponding to # existing, working (as far as we know) connections. This will be # derivable from self.connectors, but hopefully will be maintained to # present a good snapshot of what is alive, now, and what is not. # This is stored in a deque so that it can be efficiently rotated # to distribute requests. self.good_conns = set() # A set of CassandraPoolReconnectorFactory instances, formerly in # self.connectors, the connections for which are draining. No new # requests should be fed to these instances; they are tracked only so # that they can be terminated more fully in case this service is shut # down before they finish. self.dying_conns = set()
class CassandraSchemaManager(object): """Manages creation and destruction of cassandra schemas. Useful for both testing and production """ def __init__(self, keyspace_def, error_if_existing=False): self.keyspace_def = keyspace_def self.error_if_existing=error_if_existing self.created_keyspace = False self.created_cfs = [] self.client = None self.manager = None self.connector = None def connect(self, host=None, port=9160, username=None, password=None): if not host: host, port = get_host_port() if username or password: if not (username and password): raise CassandraConfigurationError( "Specify both username and password or neither") else: username, password = get_credentials() authz = dict(username=username, password=password) self.manager = ManagedCassandraClientFactory(credentials=authz, check_api_version=True) self.connector = reactor.connectTCP(host, port, self.manager) self.client = CassandraClient(self.manager) def disconnect(self): if self.manager: self.manager.shutdown() if self.connector: self.connector.disconnect() @timeout(DEFAULT_CASSANDRA_TIMEOUT) @defer.inlineCallbacks def create(self, truncate=False): if not self.client: self.connect() keyspace = self.keyspace_def try: existing = yield self.client.describe_keyspace(keyspace.name) except NotFoundException: existing = None # keyspace already exists if existing: yield self.client.set_keyspace(keyspace.name) _compare_ks_properties(existing, keyspace) existing_cfs = dict((cf.name, cf) for cf in existing.cf_defs) for cf in keyspace.cf_defs: if cf.name in existing_cfs: if truncate: # in truncate mode we drop and readd any existing CFs. yield self.client.system_drop_column_family(cf.name) yield self.client.system_add_column_family(cf) else: _compare_cf_properties(existing_cfs[cf.name], cf) else: if cf.keyspace != keyspace.name: raise CassandraSchemaError( "CF %s has wrong keyspace name", cf.name) self.created_cfs.append(cf.name) yield self.client.system_add_column_family(cf) else: self.created_keyspace = True yield self.client.system_add_keyspace(keyspace) yield self.client.set_keyspace(keyspace.name) @timeout(DEFAULT_CASSANDRA_TIMEOUT) @defer.inlineCallbacks def teardown(self): if self.created_keyspace: yield self.client.system_drop_keyspace(self.keyspace_def.name) elif self.created_cfs: for cf in self.created_cfs: yield self.client.system_drop_column_family(cf)
class PassthroughHandler(BaseHandler): """ The passthrough handler simply takes what has been sent in from the router and reads/writes/deletes without any validation. Handlers should subclass this handler in order to do parameter validation. After validation, the handlers should call through the passthrough handler to write to the database """ cass_factories = {} @classmethod def add_cass_factory(cls, factory_name, factory): cls.cass_factories[factory_name] = factory def initialize(self, factory_name, table, column): """ The factory_name, table and column are set as part of the Application router, see api/__init__.py The table corresponds to the cassandra table, while the column specifies the cassandra column to operate on The row to operate on is passed to each function, while the value is in the request body, if relevant """ self.table = table self.column = column self.cass = CassandraClient(self.cass_factories[factory_name]) @defer.inlineCallbacks def get(self, row): try: result = yield self.ha_get(column_family=self.table, key=row, column=self.column) self.finish(result.column.value) except NotFoundException: raise HTTPError(404) # POST is difficult to generalize as it resource-specific - so force subclasses to implement def post(self, *args): raise HTTPError(405) @defer.inlineCallbacks def put(self, row): yield self.cass.insert(column_family=self.table, key=row, column=self.column, value=self.request.body) self.finish({}) @defer.inlineCallbacks def delete(self, row): yield self.cass.remove(column_family=self.table, key=row, column=self.column) self.set_status(httplib.NO_CONTENT) self.finish() # After growing a cluster, Cassandra does not pro-actively populate the # new nodes with their data (the nodes are expected to use `nodetool # repair` if they need to get their data). Combining this with # the fact that we generally use consistency ONE when reading data, the # behaviour on new nodes is to return NotFoundException or empty result # sets to queries, even though the other nodes have a copy of the data. # # To resolve this issue, these two functions can be used as drop-in # replacements for `CassandraClient#get` and `CassandraClient#get_slice` # and will attempt a QUORUM read in the event that a ONE read returns # no data. If the QUORUM read fails due to unreachable nodes, the # original result will be returned (i.e. an empty set or NotFound). @defer.inlineCallbacks def ha_get(self, *args, **kwargs): try: result = yield self.cass.get(*args, **kwargs) defer.returnValue(result) except NotFoundException as e: kwargs['consistency'] = ConsistencyLevel.QUORUM try: result = yield self.cass.get(*args, **kwargs) defer.returnValue(result) except (NotFoundException, UnavailableException): raise e @defer.inlineCallbacks def ha_get_slice(self, *args, **kwargs): result = yield self.cass.get_slice(*args, **kwargs) if len(result) == 0: kwargs['consistency'] = ConsistencyLevel.QUORUM try: qresult = yield self.cass.get_slice(*args, **kwargs) result = qresult except UnavailableException: pass defer.returnValue(result)
# from the data structure res = yield client.batch_insert(key='test', column_family=CF, mapping={colname: 'bar'}) print "batch_insert", res res = yield client.batch_insert(key='test', column_family=SCF, mapping={'foo': {colname: 'bar'}}) print "batch_insert", res # with ttypes, you pass a list as you would for raw thrift # this way you can set custom timestamps cols = [Column(colname, 'bar', 1234), Column('bar', 'baz', 54321)] res = yield client.batch_insert(key='test', column_family=CF, mapping=cols) print "batch_insert", res cols = [SuperColumn(name=colname, columns=cols)] # of course you don't have to use kwargs if the order is correct res = yield client.batch_insert('test', SCF, cols) print "batch_insert", res if __name__ == '__main__': from twisted.internet import reactor from twisted.python import log import sys log.startLogging(sys.stdout) f = ManagedCassandraClientFactory(KEYSPACE) c = CassandraClient(f) dostuff(c) reactor.connectTCP(HOST, PORT, f) reactor.run()
class PassthroughHandler(BaseHandler): """ The passthrough handler simply takes what has been sent in from the router and reads/writes/deletes without any validation. Handlers should subclass this handler in order to do parameter validation. After validation, the handlers should call through the passthrough handler to write to the database """ cass_factories = {} @classmethod def add_cass_factory(cls, factory_name, factory): cls.cass_factories[factory_name] = factory def initialize(self, factory_name, table, column): """ The factory_name, table and column are set as part of the Application router, see api/__init__.py The table corresponds to the cassandra table, while the column specifies the cassandra column to operate on The row to operate on is passed to each function, while the value is in the request body, if relevant """ self.table = table self.column = column self.cass = CassandraClient(self.cass_factories[factory_name]) @defer.inlineCallbacks def get(self, row): try: result = yield self.ha_get(column_family=self.table, key=row, column=self.column) self.finish(result.column.value) except NotFoundException: raise HTTPError(404) # POST is difficult to generalize as it resource-specific - so force subclasses to implement def post(self, *args): raise HTTPError(405) @defer.inlineCallbacks def put(self, row): yield self.cass.insert(column_family=self.table, key=row, column=self.column, value=self.request.body) self.finish({}) @defer.inlineCallbacks def delete(self, row): yield self.cass.remove(column_family=self.table, key=row, column=self.column) self.set_status(httplib.NO_CONTENT) self.finish() # After growing a cluster, Cassandra does not pro-actively populate the # new nodes with their data (the nodes are expected to use `nodetool # repair` if they need to get their data). # @defer.inlineCallbacks def ha_get(self, *args, **kwargs): kwargs['consistency'] = ConsistencyLevel.LOCAL_QUORUM try: result = yield self.cass.get(*args, **kwargs) defer.returnValue(result) except UnavailableException as e: try: kwargs['consistency'] = ConsistencyLevel.ONE result = yield self.cass.get(*args, **kwargs) defer.returnValue(result) except (NotFoundException, UnavailableException) as e: raise e @defer.inlineCallbacks def ha_get_slice(self, *args, **kwargs): kwargs['consistency'] = ConsistencyLevel.LOCAL_QUORUM try: result = yield self.cass.get_slice(*args, **kwargs) defer.returnValue(result) except UnavailableException as e: try: kwargs['consistency'] = ConsistencyLevel.ONE result = yield self.cass.get_slice(*args, **kwargs) defer.returnValue(result) except (NotFoundException, UnavailableException) as e: raise e
class CassandraProvisionerStore(object): """ Provides high level provisioner storage operations for Cassandra """ # default size of paged fetches _PAGE_SIZE = 100 LAUNCH_CF_NAME = "ProvisionerLaunches" NODE_CF_NAME = "ProvisionerNodes" @classmethod def get_column_families(cls, keyspace=None, prefix=''): """Builds a list of column families needed by this store. @param keyspace Name of keyspace. If None, it must be added manually. @param prefix Optional prefix for cf names. Useful for testing. @retval list of CfDef objects """ launch_cf_name = prefix + cls.LAUNCH_CF_NAME node_cf_name = prefix + cls.NODE_CF_NAME return [ CfDef(keyspace, launch_cf_name, comparator_type='org.apache.cassandra.db.marshal.UTF8Type'), CfDef(keyspace, node_cf_name, comparator_type='org.apache.cassandra.db.marshal.UTF8Type') ] def __init__(self, host, port, username, password, keyspace, prefix=''): self._launch_column_family = prefix + self.LAUNCH_CF_NAME self._node_column_family = prefix + self.NODE_CF_NAME authz = {'username': username, 'password': password} self._manager = ManagedCassandraClientFactory(credentials=authz, check_api_version=True, keyspace=keyspace) self.client = CassandraClient(self._manager) self._host = host self._port = port self._connector = None def connect(self): self._connector = reactor.connectTCP(self._host, self._port, self._manager) def disconnect(self): self._manager.shutdown() if self._connector: self._connector.disconnect() self._connector = None @timeout(CASSANDRA_TIMEOUT) def put_launch(self, launch): """ @brief Stores a single launch record @param launch Launch record to store @retval Deferred for success """ launch_id = launch['launch_id'] state = launch['state'] value = json.dumps(launch) return self.client.insert(launch_id, self._launch_column_family, value, column=state) @timeout(CASSANDRA_TIMEOUT) @defer.inlineCallbacks def put_nodes(self, nodes): """ @brief Stores a set of node records @param nodes Iterable of node records @retval Deferred for success """ # could be more efficient with a batch_mutate for node in nodes: yield self.put_node(node) @timeout(CASSANDRA_TIMEOUT) def put_node(self, node): """ @brief Stores a node record @param node Node record @retval Deferred for success """ node_id = node['node_id'] state = node['state'] value = json.dumps(node) return self.client.insert(node_id, self._node_column_family, value, column=state) @timeout(CASSANDRA_TIMEOUT) def get_launch(self, launch_id, count=1): """ @brief Retrieves a launch record by id @param launch_id Id of launch record to retrieve @param count Number of launch state records to retrieve @retval Deferred record(s), or None. A list of records if count > 1 """ return self._get_record(launch_id, self._launch_column_family, count) @timeout(CASSANDRA_TIMEOUT) def get_launches(self, state=None, min_state=None, max_state=None): """ @brief Retrieves the latest record for all launches within a state range @param state Only retrieve nodes in this state. @param min_state Inclusive start bound @param max_state Inclusive end bound @retval Deferred list of launch records """ return self._get_records(self._launch_column_family, state=state, min_state=min_state, max_state=max_state) @timeout(CASSANDRA_TIMEOUT) def get_node(self, node_id, count=1): """ @brief Retrieves a launch record by id @param node_id Id of node record to retrieve @param count Number of node state records to retrieve @retval Deferred record(s), or None. A list of records if count > 1 """ return self._get_record(node_id, self._node_column_family, count) @timeout(CASSANDRA_TIMEOUT) def get_nodes(self, state=None, min_state=None, max_state=None): """ @brief Retrieves all launch record within a state range @param state Only retrieve nodes in this state. @param min_state Inclusive start bound. @param max_state Inclusive end bound @retval Deferred list of launch records """ return self._get_records(self._node_column_family, state=state, min_state=min_state, max_state=max_state) @defer.inlineCallbacks def _get_record(self, key, column_family, count): slice = yield self.client.get_slice(key, column_family, reverse=True, count=count) # we're probably only interested in the last record, in sorted order. # This is the latest state the object has recorded. records = [json.loads(column.column.value) for column in slice] if count == 1: if records: ret = records[0] else: ret = None else: ret = records defer.returnValue(ret) @defer.inlineCallbacks def _get_records(self, column_family, state=None, min_state=None, max_state=None, reverse=True): # overrides range arguments if state: min_state = max_state = state start = '' end = min_state or '' if not reverse: start, end = end, start # this is tricky. We are only concerned with the latest state record # (by sort order not necessarily time). So when we look for records # within a state range, we effectively must pull down the latest state # for each record, and filter them locally. This is slightly improved # when a first_state (or last when reverse=False) is specified as the # server can skip any records not >= that state. records = [] done = False start_key = '' iterations = 0 while not done: slices = yield self.client.get_range_slices(column_family, column_start=start, column_finish=end, reverse=reverse, column_count=1, start=start_key, count=self._PAGE_SIZE) skipped_one = False for slice in slices: if not skipped_one and iterations: # if this not the first batch, skip the first element as it # will be a dupe. skipped_one = True continue if not slice.columns: # rows without matching columns will still be returned continue record = json.loads(slice.columns[0].column.value) if not max_state or record['state'] <= max_state: if not min_state or record['state'] >= min_state: records.append(record) # page through results. by default only 100 are returned at a time if len(slices) == self._PAGE_SIZE: start_key = slices[-1].key else: done = True iterations += 1 defer.returnValue(records) def on_deactivate(self, *args, **kwargs): self._manager.shutdown() log.info('on_deactivate: Lose Connection TCP') def on_terminate(self, *args, **kwargs): self._manager.shutdown() log.info('on_terminate: Lose Connection TCP')
class CassandraControllerStore(TCPConnection): """Cassandra persistence for EPU controller state All EPU controllers within a system share the same column families. The "known" CFs hold a list of known instance IDs and sensor IDs for each EPU controller. The value is irrelevant and inserts are of course idempotent. These are used for figuring out all information about a controller without walking the entire instances and sensors CFs. ControllerKnownInstances = { Controller1 = { instance_id_1, instance_id_2 }, Controller2 = { instance_id_1, instance_id_2 } } ControllerKnownSensors = { Controller1 = { sensor_1, sensor_2 }, Controller2 = { sensor_1, sensor_2 } } All records for an instance are held in a single row. The column keys are TimeUUIDs. Because instance records have more complicated ordering than time, it is necessary to ensure a record is actually new before inserting. This is safe since in the current architecture there is only one writer for controller. One limitation is that out-of-order records cannot be inserted in the store. So if the messaging layer provides the STARTED state record for an instance *after* the RUNNING record, it cannot be stored and must be dropped. The correct state will be preserved, but not all of history will be. ControllerInstances = { #comparator = TimeUUIDType Controller1Instance1 = { TimeUUID1 : 'the actual record', TimeUUID2 : 'the actual record', TimeUUID3 : 'the actual record', }, Controller2Instance1 = { TimeUUID1 : 'the actual record', TimeUUID2 : 'the actual record', TimeUUID3 : 'the actual record', } } Sensor records are stored similarly. Instead of a TimeUUID, they use longs as keys which are likely to be a timestamp. Again, the controller must check the timestamp and not treat the most recently arrived value as the latest. However it can still write older values as they will be correctly inserted into history. ControllerSensors = { #comparator = LongType Controller1Sensor1 = { timestamp1 : 'sensor message', timestamp2 : 'sensor message' } } The controller starts up with a dictionary of config values that are passed to the decision engine. These can be changed by calling the reconfigure operation. Reconfigured values are stored here in Cassandra and on reboot/recovery, they are folded into the original config before it is passed to the decision engine. Each controller gets a row. Each column represents a single key/value pair where the key is a string and the value is a JSON-encoded object. ControllerEngineConfig = { Controller1 = { key1 : 'value1', key2 : 'value2' }, Controller2 = { key1 : 'value1', key2 : 'value2' } } """ CONFIG_CF_NAME = "ControllerEngineConfig" INSTANCE_CF_NAME = "ControllerInstances" INSTANCE_ID_CF_NAME = "ControllerKnownInstances" SENSOR_CF_NAME = "ControllerSensors" SENSOR_ID_CF_NAME = "ControllerKnownSensors" _PAGE_SIZE = 100 @classmethod def get_column_families(cls, keyspace=None, prefix=''): """Builds a list of column families needed by this store. @param keyspace Name of keyspace. If None, it must be added manually. @param prefix Optional prefix for cf names. Useful for testing. @retval list of CfDef objects """ instance_cf=prefix+cls.INSTANCE_CF_NAME instance_id_cf=prefix+cls.INSTANCE_ID_CF_NAME sensor_cf=prefix+cls.SENSOR_CF_NAME sensor_id_cf=prefix+cls.SENSOR_ID_CF_NAME config_cf = prefix+cls.CONFIG_CF_NAME return [CfDef(keyspace, instance_cf, comparator_type='org.apache.cassandra.db.marshal.TimeUUIDType'), CfDef(keyspace, instance_id_cf, comparator_type='org.apache.cassandra.db.marshal.UTF8Type'), CfDef(keyspace, sensor_cf, comparator_type='org.apache.cassandra.db.marshal.LongType'), CfDef(keyspace, sensor_id_cf, comparator_type='org.apache.cassandra.db.marshal.UTF8Type'), CfDef(keyspace, config_cf, comparator_type='org.apache.cassandra.db.marshal.UTF8Type'), ] def __init__(self, controller_name, host, port, username, password, keyspace, instance_factory, sensor_item_factory, prefix=''): self.controller_name = str(controller_name) # keep a set of known instances and sensors so we can save on # unnecessary inserts to the controller instance/sensor lists self.seen_instances = set() self.seen_sensors = set() self.instance_factory = instance_factory self.sensor_item_factory = sensor_item_factory authorization_dictionary = {'username': username, 'password': password} self.manager = ManagedCassandraClientFactory( credentials=authorization_dictionary, check_api_version=True, keyspace=keyspace) TCPConnection.__init__(self, host, port, self.manager) self.client = CassandraClient(self.manager) self.instance_cf = prefix + self.INSTANCE_CF_NAME self.instance_id_cf = prefix + self.INSTANCE_ID_CF_NAME self.sensor_cf = prefix + self.SENSOR_CF_NAME self.sensor_id_cf = prefix + self.SENSOR_ID_CF_NAME self.config_cf = prefix + self.CONFIG_CF_NAME @timeout(CASSANDRA_TIMEOUT) @defer.inlineCallbacks def check_schema(self): ks = yield self.client.describe_keyspace(self.manager.keyspace) cfs = dict((cf.name,cf) for cf in ks.cf_defs) missing = [cf for cf in (self.instance_cf, self.instance_id_cf, self.sensor_cf, self.sensor_id_cf, self.config_cf) if cf in cfs] if missing: error = "EPU Controller is missing Cassandra column families: %s" raise Exception(error % ", ".join(missing)) @timeout(CASSANDRA_TIMEOUT) @defer.inlineCallbacks def add_instance(self, instance): """Adds a new instance object to persistence @param instance Instance to add @retval Deferred """ instance_id = str(instance.instance_id) if instance_id not in self.seen_instances: yield self.client.insert(self.controller_name, self.instance_id_cf, "", column=instance_id) self.seen_instances.add(instance_id) key = self.controller_name + instance_id value = json.dumps(dict(instance.iteritems())) col = uuid.uuid1().bytes yield self.client.insert(key, self.instance_cf, value, column=col) @timeout(CASSANDRA_TIMEOUT) def get_instance_ids(self): """Retrieves a list of known instances @retval Deferred of list of instance IDs """ return self._get_ids(self.instance_id_cf) @timeout(CASSANDRA_TIMEOUT) @defer.inlineCallbacks def get_instance(self, instance_id): """Retrieves the latest instance object for the specified id @param instance_id ID of instance to retrieve @retval Deferred of Instance object or None """ key = self.controller_name + str(instance_id) slice = yield self.client.get_slice(key, self.instance_cf, reverse=True, count=1) if slice: d = json.loads(slice[0].column.value) ret = self.instance_factory(**d) else: ret = None defer.returnValue(ret) @timeout(CASSANDRA_TIMEOUT) @defer.inlineCallbacks def add_sensor(self, sensor): """Adds a new sensor object to persistence @param sensor Sensor to add @retval Deferred """ sensor_id = str(sensor.sensor_id) if sensor_id not in self.seen_sensors: yield self.client.insert(self.controller_name, self.sensor_id_cf, "", column=sensor_id) self.seen_sensors.add(sensor_id) key = self.controller_name + sensor_id value = json.dumps(sensor.value) col = struct.pack('!Q', int(sensor.time)) yield self.client.insert(key, self.sensor_cf, value, column=col) @timeout(CASSANDRA_TIMEOUT) def get_sensor_ids(self): """Retrieves a list of known sensors @retval Deferred of list of sensor IDs """ return self._get_ids(self.sensor_id_cf) @timeout(CASSANDRA_TIMEOUT) @defer.inlineCallbacks def get_sensor(self, sensor_id): """Retrieve the latest sensor item for the specified sensor @param sensor_id ID of the sensor item to retrieve @retval Deferred of SensorItem object or None """ key = self.controller_name + str(sensor_id) slice = yield self.client.get_slice(key, self.sensor_cf, reverse=True, count=1) if slice: col = slice[0].column timestamp = struct.unpack("!Q", col.name)[0] val = json.loads(col.value) ret = self.sensor_item_factory(sensor_id, long(timestamp), val) else: ret = None defer.returnValue(ret) @timeout(CASSANDRA_TIMEOUT) @defer.inlineCallbacks def get_config(self, keys=None): """Retrieve the engine config dictionary. @param keys optional list of keys to retrieve @retval Deferred of config dictionary object """ key = self.controller_name slice = yield self.client.get_slice(key, self.config_cf, names=keys) cfg = {} if slice: for col in slice: key = col.column.name val = col.column.value cfg[key] = json.loads(val) defer.returnValue(cfg) @timeout(CASSANDRA_TIMEOUT) def add_config(self, conf): """Store a dictionary of new engine conf values. These are folded into the existing configuration map. So for example if you first store {'a' : 1, 'b' : 1} and then store {'b' : 2}, the result from get_config() will be {'a' : 1, 'b' : 2}. @param conf dictionary mapping strings to JSON-serializable objects @retval Deferred """ d = dict((k, json.dumps(v)) for k,v in conf.iteritems()) return self.client.batch_insert(self.controller_name, self.config_cf, d) @defer.inlineCallbacks def _get_ids(self, cf): """Retrieves IDs from either instance or sensor column families """ # using a set because it isn't totally clear if the start parameter # to get_slice() is always inclusive or exclusive. Seeing mixed # messages on mailing list, so also not convinced if this isn't # something that has changed, or might. found_ids = set() start = "" done = False while not done: slice = yield self.client.get_slice(self.controller_name, cf, count=self._PAGE_SIZE, start=start) if slice: found_ids.update(col.column.name for col in slice) if len(slice) == self._PAGE_SIZE: start = slice[-1].column.name else: done = True else: done = True defer.returnValue(list(found_ids)) def on_deactivate(self, *args, **kwargs): self.manager.shutdown() def on_terminate(self, *args, **kwargs): self.manager.shutdown()
class CassandraClientTest(unittest.TestCase): @defer.inlineCallbacks def setUp(self): self.cmanager = ManagedCassandraClientFactory(keyspace='system') self.client = CassandraClient(self.cmanager) for i in xrange(CONNS): reactor.connectTCP(HOST, PORT, self.cmanager) yield self.cmanager.deferred self.my_keyspace = KsDef( name=KEYSPACE, strategy_class='org.apache.cassandra.locator.SimpleStrategy', replication_factor=1, cf_defs=[ CfDef( keyspace=KEYSPACE, name=CF, column_type='Standard' ), CfDef( keyspace=KEYSPACE, name=SCF, column_type='Super' ), CfDef( keyspace=KEYSPACE, name=IDX_CF, column_type='Standard', comparator_type='org.apache.cassandra.db.marshal.UTF8Type', column_metadata=[ ColumnDef( name='col1', validation_class='org.apache.cassandra.db.marshal.UTF8Type', index_type=IndexType.KEYS, index_name='idxCol1') ], default_validation_class='org.apache.cassandra.db.marshal.BytesType') ] ) yield self.client.system_add_keyspace(self.my_keyspace) yield self.client.set_keyspace(KEYSPACE) @defer.inlineCallbacks def tearDown(self): yield self.client.system_drop_keyspace(self.my_keyspace.name) self.cmanager.shutdown() for c in reactor.getDelayedCalls(): c.cancel() reactor.removeAll() @defer.inlineCallbacks def test_insert_get(self): yield self.client.insert('test', CF, 'testval', column=COLUMN) yield self.client.insert('test2', CF, 'testval2', column=COLUMN) yield self.client.insert('test', SCF, 'superval', column=COLUMN, super_column=SCOLUMN) yield self.client.insert('test2', SCF, 'superval2', column=COLUMN, super_column=SCOLUMN) res = yield self.client.get('test', CF, column=COLUMN) self.assertEqual(res.column.value, 'testval') res = yield self.client.get('test2', CF, column=COLUMN) self.assertEqual(res.column.value, 'testval2') res = yield self.client.get('test', SCF, column=COLUMN, super_column=SCOLUMN) self.assertEqual(res.column.value, 'superval') res = yield self.client.get('test2', SCF, column=COLUMN, super_column=SCOLUMN) self.assertEqual(res.column.value, 'superval2') @defer.inlineCallbacks def test_batch_insert_get_slice_and_count(self): yield self.client.batch_insert('test', CF, {COLUMN: 'test', COLUMN2: 'test2'}) yield self.client.batch_insert('test', SCF, {SCOLUMN: {COLUMN: 'test', COLUMN2: 'test2'}}) res = yield self.client.get_slice('test', CF, names=(COLUMN, COLUMN2)) self.assertEqual(res[0].column.value, 'test') self.assertEqual(res[1].column.value, 'test2') res = yield self.client.get_slice('test', SCF, names=(COLUMN, COLUMN2), super_column=SCOLUMN) self.assertEqual(res[0].column.value, 'test') self.assertEqual(res[1].column.value, 'test2') res = yield self.client.get_count('test', CF) self.assertEqual(res, 2) @defer.inlineCallbacks def test_batch_mutate_and_remove(self): yield self.client.batch_mutate({'test': {CF: {COLUMN: 'test', COLUMN2: 'test2'}, SCF: { SCOLUMN: { COLUMN: 'test', COLUMN2: 'test2'} } }, 'test2': {CF: {COLUMN: 'test', COLUMN2: 'test2'}, SCF: { SCOLUMN: { COLUMN: 'test', COLUMN2: 'test2'} } } }) res = yield self.client.get_slice('test', CF, names=(COLUMN, COLUMN2)) self.assertEqual(res[0].column.value, 'test') self.assertEqual(res[1].column.value, 'test2') res = yield self.client.get_slice('test2', CF, names=(COLUMN, COLUMN2)) self.assertEqual(res[0].column.value, 'test') self.assertEqual(res[1].column.value, 'test2') res = yield self.client.get_slice('test', SCF, names=(COLUMN, COLUMN2), super_column=SCOLUMN) self.assertEqual(res[0].column.value, 'test') self.assertEqual(res[1].column.value, 'test2') res = yield self.client.get_slice('test2', SCF, names=(COLUMN, COLUMN2), super_column=SCOLUMN) self.assertEqual(res[0].column.value, 'test') self.assertEqual(res[1].column.value, 'test2') yield self.client.batch_remove({CF: ['test', 'test2']}, names=['test', 'test2']) yield self.client.batch_remove({SCF: ['test', 'test2']}, names=['test', 'test2'], supercolumn=SCOLUMN) @defer.inlineCallbacks def test_batch_mutate_with_deletion(self): yield self.client.batch_mutate({'test': {CF: {COLUMN: 'test', COLUMN2: 'test2'}}}) res = yield self.client.get_slice('test', CF, names=(COLUMN, COLUMN2)) self.assertEqual(res[0].column.value, 'test') self.assertEqual(res[1].column.value, 'test2') yield self.client.batch_mutate({'test': {CF: {COLUMN: None, COLUMN2: 'test3'}}}) res = yield self.client.get_slice('test', CF, names=(COLUMN, COLUMN2)) self.assertEqual(len(res), 1) self.assertEqual(res[0].column.value, 'test3') @defer.inlineCallbacks def test_multiget_slice_remove(self): yield self.client.insert('test', CF, 'testval', column=COLUMN) yield self.client.insert('test', CF, 'testval', column=COLUMN2) yield self.client.insert('test2', CF, 'testval2', column=COLUMN) res = yield self.client.multiget(['test', 'test2'], CF, column=COLUMN) self.assertEqual(res['test'][0].column.value, 'testval') self.assertEqual(res['test2'][0].column.value, 'testval2') res = yield self.client.multiget_slice(['test', 'test2'], CF) self.assertEqual(res['test'][0].column.value, 'testval') self.assertEqual(res['test'][1].column.value, 'testval') self.assertEqual(res['test2'][0].column.value, 'testval2') yield self.client.remove('test', CF, column=COLUMN) yield self.client.remove('test2', CF, column=COLUMN) res = yield self.client.multiget(['test', 'test2'], CF, column=COLUMN) self.assertEqual(len(res['test']), 0) self.assertEqual(len(res['test2']), 0) @defer.inlineCallbacks def test_range_slices(self): yield self.client.insert('test', CF, 'testval', column=COLUMN) yield self.client.insert('test', CF, 'testval', column=COLUMN2) yield self.client.insert('test2', CF, 'testval2', column=COLUMN) ks = yield self.client.get_range_slices(CF, start='', finish='') keys = [k.key for k in ks] for key in ['test', 'test2']: self.assertIn(key, keys) @defer.inlineCallbacks def test_indexed_slices(self): yield self.client.insert('test1', IDX_CF, 'one', column='col1') yield self.client.insert('test2', IDX_CF, 'two', column='col1') yield self.client.insert('test3', IDX_CF, 'three', column='col1') expressions = [IndexExpression('col1', IndexOperator.EQ, 'two')] res = yield self.client.get_indexed_slices(IDX_CF, expressions, start_key='') self.assertEquals(res[0].columns[0].column.value,'two') def sleep(self, secs): d = defer.Deferred() reactor.callLater(secs, d.callback, None) return d @defer.inlineCallbacks def test_ttls(self): yield self.client.insert('test_ttls', CF, 'testval', column=COLUMN, ttl=1) res = yield self.client.get('test_ttls', CF, column=COLUMN) self.assertEqual(res.column.value, 'testval') yield self.sleep(2) yield self.assertFailure(self.client.get('test_ttls', CF, column=COLUMN), NotFoundException) yield self.client.batch_insert('test_ttls', CF, {COLUMN:'testval'}, ttl=1) res = yield self.client.get('test_ttls', CF, column=COLUMN) self.assertEqual(res.column.value, 'testval') yield self.sleep(2) yield self.assertFailure(self.client.get('test_ttls', CF, column=COLUMN), NotFoundException) yield self.client.batch_mutate({'test_ttls': {CF: {COLUMN: 'testval'}}}, ttl=1) res = yield self.client.get('test_ttls', CF, column=COLUMN) self.assertEqual(res.column.value, 'testval') yield self.sleep(2) yield self.assertFailure(self.client.get('test_ttls', CF, column=COLUMN), NotFoundException) @defer.inlineCallbacks def test_keyspace_manipulation(self): ksdef = KsDef(name=T_KEYSPACE, strategy_class='org.apache.cassandra.locator.SimpleStrategy', replication_factor=1, cf_defs=[]) yield self.client.system_add_keyspace(ksdef) ks2 = yield self.client.describe_keyspace(T_KEYSPACE) self.assertEqual(ksdef, ks2) if DO_SYSTEM_RENAMING: newname = T_KEYSPACE + '2' yield self.client.system_rename_keyspace(T_KEYSPACE, newname) ks2 = yield self.client.describe_keyspace(newname) ksdef.name = newname self.assertEqual(ksdef, ks2) yield self.client.system_drop_keyspace(ksdef.name) yield self.assertFailure(self.client.describe_keyspace(T_KEYSPACE), NotFoundException) if DO_SYSTEM_RENAMING: yield self.assertFailure(self.client.describe_keyspace(ksdef.name), NotFoundException) @defer.inlineCallbacks def test_column_family_manipulation(self): cfdef = CfDef(KEYSPACE, T_CF, column_type='Standard', comparator_type='org.apache.cassandra.db.marshal.BytesType', comment='foo', row_cache_size=0.0, key_cache_size=200000.0, read_repair_chance=1.0, column_metadata=[], gc_grace_seconds=86400, default_validation_class='org.apache.cassandra.db.marshal.BytesType', min_compaction_threshold=5, max_compaction_threshold=31, row_cache_save_period_in_seconds=0, key_cache_save_period_in_seconds=3600, memtable_flush_after_mins=60, memtable_throughput_in_mb=249, memtable_operations_in_millions=1.1671875, ) yield self.client.system_add_column_family(cfdef) ksdef = yield self.client.describe_keyspace(KEYSPACE) cfdef2 = [c for c in ksdef.cf_defs if c.name == T_CF][0] # we don't know the id ahead of time. copy the new one so the equality # comparison won't fail cfdef.id = cfdef2.id self.assertEqual(cfdef, cfdef2) if DO_SYSTEM_RENAMING: newname = T_CF + '2' yield self.client.system_rename_column_family(T_CF, newname) ksdef = yield self.client.describe_keyspace(KEYSPACE) cfdef2 = [c for c in ksdef.cf_defs if c.name == newname][0] self.assertNotIn(T_CF, [c.name for c in ksdef.cf_defs]) cfdef.name = newname self.assertEqual(cfdef, cfdef2) yield self.client.system_drop_column_family(cfdef.name) ksdef = yield self.client.describe_keyspace(KEYSPACE) self.assertNotIn(cfdef.name, [c.name for c in ksdef.cf_defs]) @defer.inlineCallbacks def test_describes(self): name = yield self.client.describe_cluster_name() self.assertIsInstance(name, str) self.assertNotEqual(name, '') partitioner = yield self.client.describe_partitioner() self.assert_(partitioner.startswith('org.apache.cassandra.'), msg='partitioner is %r' % partitioner) snitch = yield self.client.describe_snitch() self.assert_(snitch.startswith('org.apache.cassandra.'), msg='snitch is %r' % snitch) version = yield self.client.describe_version() self.assertIsInstance(version, str) self.assertIn('.', version) schemavers = yield self.client.describe_schema_versions() self.assertIsInstance(schemavers, dict) self.assertNotEqual(schemavers, {}) ring = yield self.client.describe_ring(KEYSPACE) self.assertIsInstance(ring, list) self.assertNotEqual(ring, []) for r in ring: self.assertIsInstance(r.start_token, str) self.assertIsInstance(r.end_token, str) self.assertIsInstance(r.endpoints, list) self.assertNotEqual(r.endpoints, []) for ep in r.endpoints: self.assertIsInstance(ep, str) @defer.inlineCallbacks def test_errback(self): yield self.client.remove('poiqwe', CF) try: yield self.client.get('poiqwe', CF, column='foo') except Exception, e: pass
def __init__(self, seed_list, keyspace=None, creds=None, thrift_port=None, pool_size=None, conn_timeout=10, bind_address=None, log_cb=log.msg, reactor=None, require_api_version=None): """ Initialize a CassandraClusterPool. @param keyspace: If given and not None, determines the keyspace to which all connections in this pool will be made. @param creds: Credentials to use to authenticate Cassandra connections @type creds: A dict (or other mapping) of strings to strings @param seed_list: An initial set of host addresses which, if connectable, are part of this cluster. @type seed_list: iterable @param thrift_port: The port to use for connections to Cassandra nodes @param pool_size: The target size for the connection pool. Naturally, the actual size may be higher or lower as nodes connect and disconnect, but an effort will be made to adjust toward this size. @type pool_size: int @param conn_timeout: The number of seconds before a pending connection is deemed unsuccessful and aborted. Of course, when a connection error can be detected before this time, the connection will be aborted appropriately. @type conn_timeout: float @param bind_address: The local interface to which to bind when making outbound connections. Default: determined by the system's socket layer. @type bind_address: str @param log_cb: A callable which is expected to work like L{twisted.python.log.msg}. Will be used when certain connection and disconnection events occur. The default is log.msg. @param reactor: The reactor instance to use when starting thrift connections or setting timers. @param require_api_version: If not None, Telephus will require that all connections conform to the API for the given Cassandra version. Possible values are "0.7", "0.8", "1.0", etc. If None, Telephus will consider all supported API versions to be acceptable. If the api version reported by a remote node is not compatible, the connection to that node will be aborted. Default: None """ self.seed_list = list(seed_list) if thrift_port is None: thrift_port = self.default_cassandra_thrift_port self.thrift_port = thrift_port if pool_size is None: pool_size = len(self.seed_list) self.target_pool_size = pool_size self.log = log_cb self.conn_timeout = conn_timeout self.bind_address = bind_address self.keyspace = keyspace self.creds = creds self.request_queue = defer.DeferredQueue() self.require_api_version = require_api_version self.future_fill_pool = None self.removed_nodes = set() self._client_instance = CassandraClient(self) if reactor is None: from twisted.internet import reactor self.reactor = reactor # A set of CassandraNode instances representing known nodes. This # includes nodes from the initial seed list, nodes seen in # describe_ring calls to existing nodes, and nodes explicitly added # by the addNode() method. Nodes are only removed from this set if # no connections have been successful in self.forget_node_interval # seconds, or by an explicit call to removeNode(). self.nodes = set() # A set of CassandraPoolReconnectorFactory instances corresponding to # connections which are either live or pending. Failed attempts to # connect will remove a connector from this set. When connections are # lost, an immediate reconnect will be attempted. self.connectors = set() # A collection of objects from self.connectors corresponding to # existing, working (as far as we know) connections. This will be # derivable from self.connectors, but hopefully will be maintained to # present a good snapshot of what is alive, now, and what is not. # This is stored in a deque so that it can be efficiently rotated # to distribute requests. self.good_conns = set() # A set of CassandraPoolReconnectorFactory instances, formerly in # self.connectors, the connections for which are draining. No new # requests should be fed to these instances; they are tracked only so # that they can be terminated more fully in case this service is shut # down before they finish. self.dying_conns = set()
def setUp(self): self.cmanager = ManagedCassandraClientFactory(keyspace='system') self.client = CassandraClient(self.cmanager) for i in xrange(CONNS): reactor.connectTCP(HOST, PORT, self.cmanager) yield self.cmanager.deferred remote_ver = yield self.client.describe_version() self.version = tuple(map(int, remote_ver.split('.'))) self.my_keyspace = ttypes.KsDef( name=KEYSPACE, strategy_class='org.apache.cassandra.locator.SimpleStrategy', strategy_options={}, cf_defs=[ ttypes.CfDef( keyspace=KEYSPACE, name=CF, column_type='Standard' ), ttypes.CfDef( keyspace=KEYSPACE, name=SCF, column_type='Super' ), ttypes.CfDef( keyspace=KEYSPACE, name=IDX_CF, column_type='Standard', comparator_type='org.apache.cassandra.db.marshal.UTF8Type', column_metadata=[ ttypes.ColumnDef( name='col1', validation_class='org.apache.cassandra.db.marshal.UTF8Type', index_type=ttypes.IndexType.KEYS, index_name='idxCol1') ], default_validation_class='org.apache.cassandra.db.marshal.BytesType' ), ] ) if self.version <= KS_RF_ATTRIBUTE: self.my_keyspace.replication_factor = 1 else: self.my_keyspace.strategy_options['replication_factor'] = '1' if self.version >= COUNTERS_SUPPORTED_API: self.my_keyspace.cf_defs.extend([ ttypes.CfDef( keyspace=KEYSPACE, name=COUNTER_CF, column_type='Standard', default_validation_class='org.apache.cassandra.db.marshal.CounterColumnType' ), ttypes.CfDef( keyspace=KEYSPACE, name=SUPERCOUNTER_CF, column_type='Super', default_validation_class='org.apache.cassandra.db.marshal.CounterColumnType' ), ]) yield self.client.system_add_keyspace(self.my_keyspace) yield self.client.set_keyspace(KEYSPACE)
class CassandraProvisionerStore(object): """ Provides high level provisioner storage operations for Cassandra """ # default size of paged fetches _PAGE_SIZE = 100 LAUNCH_CF_NAME = "ProvisionerLaunches" NODE_CF_NAME = "ProvisionerNodes" @classmethod def get_column_families(cls, keyspace=None, prefix=''): """Builds a list of column families needed by this store. @param keyspace Name of keyspace. If None, it must be added manually. @param prefix Optional prefix for cf names. Useful for testing. @retval list of CfDef objects """ launch_cf_name = prefix + cls.LAUNCH_CF_NAME node_cf_name = prefix + cls.NODE_CF_NAME return [CfDef(keyspace, launch_cf_name, comparator_type='org.apache.cassandra.db.marshal.UTF8Type'), CfDef(keyspace, node_cf_name, comparator_type='org.apache.cassandra.db.marshal.UTF8Type')] def __init__(self, host, port, username, password, keyspace, prefix=''): self._launch_column_family = prefix + self.LAUNCH_CF_NAME self._node_column_family = prefix + self.NODE_CF_NAME authz= {'username': username, 'password': password} self._manager = ManagedCassandraClientFactory( credentials=authz, check_api_version=True, keyspace=keyspace) self.client = CassandraClient(self._manager) self._host = host self._port = port self._connector = None def connect(self): self._connector = reactor.connectTCP(self._host, self._port, self._manager) def disconnect(self): self._manager.shutdown() if self._connector: self._connector.disconnect() self._connector = None @timeout(CASSANDRA_TIMEOUT) def put_launch(self, launch): """ @brief Stores a single launch record @param launch Launch record to store @retval Deferred for success """ launch_id = launch['launch_id'] state = launch['state'] value = json.dumps(launch) return self.client.insert(launch_id, self._launch_column_family, value, column=state) @timeout(CASSANDRA_TIMEOUT) @defer.inlineCallbacks def put_nodes(self, nodes): """ @brief Stores a set of node records @param nodes Iterable of node records @retval Deferred for success """ # could be more efficient with a batch_mutate for node in nodes: yield self.put_node(node) @timeout(CASSANDRA_TIMEOUT) def put_node(self, node): """ @brief Stores a node record @param node Node record @retval Deferred for success """ node_id = node['node_id'] state = node['state'] value = json.dumps(node) return self.client.insert(node_id, self._node_column_family, value, column=state) @timeout(CASSANDRA_TIMEOUT) def get_launch(self, launch_id, count=1): """ @brief Retrieves a launch record by id @param launch_id Id of launch record to retrieve @param count Number of launch state records to retrieve @retval Deferred record(s), or None. A list of records if count > 1 """ return self._get_record(launch_id, self._launch_column_family, count) @timeout(CASSANDRA_TIMEOUT) def get_launches(self, state=None, min_state=None, max_state=None): """ @brief Retrieves the latest record for all launches within a state range @param state Only retrieve nodes in this state. @param min_state Inclusive start bound @param max_state Inclusive end bound @retval Deferred list of launch records """ return self._get_records(self._launch_column_family, state=state, min_state=min_state, max_state=max_state) @timeout(CASSANDRA_TIMEOUT) def get_node(self, node_id, count=1): """ @brief Retrieves a launch record by id @param node_id Id of node record to retrieve @param count Number of node state records to retrieve @retval Deferred record(s), or None. A list of records if count > 1 """ return self._get_record(node_id, self._node_column_family, count) @timeout(CASSANDRA_TIMEOUT) def get_nodes(self, state=None, min_state=None, max_state=None): """ @brief Retrieves all launch record within a state range @param state Only retrieve nodes in this state. @param min_state Inclusive start bound. @param max_state Inclusive end bound @retval Deferred list of launch records """ return self._get_records(self._node_column_family, state=state, min_state=min_state, max_state=max_state) @defer.inlineCallbacks def _get_record(self, key, column_family, count): slice = yield self.client.get_slice(key, column_family, reverse=True, count=count) # we're probably only interested in the last record, in sorted order. # This is the latest state the object has recorded. records = [json.loads(column.column.value) for column in slice] if count == 1: if records: ret = records[0] else: ret = None else: ret = records defer.returnValue(ret) @defer.inlineCallbacks def _get_records(self, column_family, state=None, min_state=None, max_state=None, reverse=True): # overrides range arguments if state: min_state = max_state = state start = '' end = min_state or '' if not reverse: start, end = end, start # this is tricky. We are only concerned with the latest state record # (by sort order not necessarily time). So when we look for records # within a state range, we effectively must pull down the latest state # for each record, and filter them locally. This is slightly improved # when a first_state (or last when reverse=False) is specified as the # server can skip any records not >= that state. records = [] done = False start_key = '' iterations = 0 while not done: slices = yield self.client.get_range_slices(column_family, column_start=start, column_finish=end, reverse=reverse, column_count=1, start=start_key, count=self._PAGE_SIZE) skipped_one = False for slice in slices: if not skipped_one and iterations: # if this not the first batch, skip the first element as it # will be a dupe. skipped_one = True continue if not slice.columns: # rows without matching columns will still be returned continue record = json.loads(slice.columns[0].column.value) if not max_state or record['state'] <= max_state: if not min_state or record['state'] >= min_state: records.append(record) # page through results. by default only 100 are returned at a time if len(slices) == self._PAGE_SIZE: start_key = slices[-1].key else: done = True iterations += 1 defer.returnValue(records) def on_deactivate(self, *args, **kwargs): self._manager.shutdown() log.info('on_deactivate: Lose Connection TCP') def on_terminate(self, *args, **kwargs): self._manager.shutdown() log.info('on_terminate: Lose Connection TCP')
class CassandraClientTest(unittest.TestCase): @defer.inlineCallbacks def setUp(self): self.cmanager = ManagedCassandraClientFactory(keyspace='system') self.client = CassandraClient(self.cmanager) for i in xrange(CONNS): reactor.connectTCP(HOST, PORT, self.cmanager) yield self.cmanager.deferred remote_ver = yield self.client.describe_version() self.version = thrift_api_ver_to_cassandra_ver(remote_ver) self.my_keyspace = KsDef( name=KEYSPACE, strategy_class='org.apache.cassandra.locator.SimpleStrategy', strategy_options={'replication_factor': '1'}, cf_defs=[ CfDef(keyspace=KEYSPACE, name=CF, column_type='Standard'), CfDef(keyspace=KEYSPACE, name=SCF, column_type='Super'), CfDef( keyspace=KEYSPACE, name=IDX_CF, column_type='Standard', comparator_type='org.apache.cassandra.db.marshal.UTF8Type', column_metadata=[ ColumnDef(name='col1', validation_class= 'org.apache.cassandra.db.marshal.UTF8Type', index_type=IndexType.KEYS, index_name='idxCol1') ], default_validation_class= 'org.apache.cassandra.db.marshal.BytesType'), ]) if self.version == CASSANDRA_08_VERSION: self.my_keyspace.cf_defs.extend([ CfDef(keyspace=KEYSPACE, name=COUNTER_CF, column_type='Standard', default_validation_class= 'org.apache.cassandra.db.marshal.CounterColumnType'), CfDef(keyspace=KEYSPACE, name=SUPERCOUNTER_CF, column_type='Super', default_validation_class= 'org.apache.cassandra.db.marshal.CounterColumnType'), ]) yield self.client.system_add_keyspace(self.my_keyspace) yield self.client.set_keyspace(KEYSPACE) @defer.inlineCallbacks def tearDown(self): yield self.client.system_drop_keyspace(self.my_keyspace.name) self.cmanager.shutdown() for c in reactor.getDelayedCalls(): c.cancel() reactor.removeAll() @defer.inlineCallbacks def test_insert_get(self): yield self.client.insert('test', CF, 'testval', column=COLUMN) yield self.client.insert('test2', CF, 'testval2', column=COLUMN) yield self.client.insert('test', SCF, 'superval', column=COLUMN, super_column=SCOLUMN) yield self.client.insert('test2', SCF, 'superval2', column=COLUMN, super_column=SCOLUMN) res = yield self.client.get('test', CF, column=COLUMN) self.assertEqual(res.column.value, 'testval') res = yield self.client.get('test2', CF, column=COLUMN) self.assertEqual(res.column.value, 'testval2') res = yield self.client.get('test', SCF, column=COLUMN, super_column=SCOLUMN) self.assertEqual(res.column.value, 'superval') res = yield self.client.get('test2', SCF, column=COLUMN, super_column=SCOLUMN) self.assertEqual(res.column.value, 'superval2') @defer.inlineCallbacks def test_batch_insert_get_slice_and_count(self): yield self.client.batch_insert('test', CF, { COLUMN: 'test', COLUMN2: 'test2' }) yield self.client.batch_insert( 'test', SCF, {SCOLUMN: { COLUMN: 'test', COLUMN2: 'test2' }}) res = yield self.client.get_slice('test', CF, names=(COLUMN, COLUMN2)) self.assertEqual(res[0].column.value, 'test') self.assertEqual(res[1].column.value, 'test2') res = yield self.client.get_slice('test', SCF, names=(COLUMN, COLUMN2), super_column=SCOLUMN) self.assertEqual(res[0].column.value, 'test') self.assertEqual(res[1].column.value, 'test2') res = yield self.client.get_count('test', CF) self.assertEqual(res, 2) @defer.inlineCallbacks def test_batch_mutate_and_remove(self): yield self.client.batch_mutate({ 'test': { CF: { COLUMN: 'test', COLUMN2: 'test2' }, SCF: { SCOLUMN: { COLUMN: 'test', COLUMN2: 'test2' } } }, 'test2': { CF: { COLUMN: 'test', COLUMN2: 'test2' }, SCF: { SCOLUMN: { COLUMN: 'test', COLUMN2: 'test2' } } } }) res = yield self.client.get_slice('test', CF, names=(COLUMN, COLUMN2)) self.assertEqual(res[0].column.value, 'test') self.assertEqual(res[1].column.value, 'test2') res = yield self.client.get_slice('test2', CF, names=(COLUMN, COLUMN2)) self.assertEqual(res[0].column.value, 'test') self.assertEqual(res[1].column.value, 'test2') res = yield self.client.get_slice('test', SCF, names=(COLUMN, COLUMN2), super_column=SCOLUMN) self.assertEqual(res[0].column.value, 'test') self.assertEqual(res[1].column.value, 'test2') res = yield self.client.get_slice('test2', SCF, names=(COLUMN, COLUMN2), super_column=SCOLUMN) self.assertEqual(res[0].column.value, 'test') self.assertEqual(res[1].column.value, 'test2') yield self.client.batch_remove({CF: ['test', 'test2']}, names=['test', 'test2']) yield self.client.batch_remove({SCF: ['test', 'test2']}, names=['test', 'test2'], supercolumn=SCOLUMN) @defer.inlineCallbacks def test_batch_mutate_with_deletion(self): yield self.client.batch_mutate( {'test': { CF: { COLUMN: 'test', COLUMN2: 'test2' } }}) res = yield self.client.get_slice('test', CF, names=(COLUMN, COLUMN2)) self.assertEqual(res[0].column.value, 'test') self.assertEqual(res[1].column.value, 'test2') yield self.client.batch_mutate( {'test': { CF: { COLUMN: None, COLUMN2: 'test3' } }}) res = yield self.client.get_slice('test', CF, names=(COLUMN, COLUMN2)) self.assertEqual(len(res), 1) self.assertEqual(res[0].column.value, 'test3') @defer.inlineCallbacks def test_multiget_slice_remove(self): yield self.client.insert('test', CF, 'testval', column=COLUMN) yield self.client.insert('test', CF, 'testval', column=COLUMN2) yield self.client.insert('test2', CF, 'testval2', column=COLUMN) res = yield self.client.multiget(['test', 'test2'], CF, column=COLUMN) self.assertEqual(res['test'][0].column.value, 'testval') self.assertEqual(res['test2'][0].column.value, 'testval2') res = yield self.client.multiget_slice(['test', 'test2'], CF) self.assertEqual(res['test'][0].column.value, 'testval') self.assertEqual(res['test'][1].column.value, 'testval') self.assertEqual(res['test2'][0].column.value, 'testval2') yield self.client.remove('test', CF, column=COLUMN) yield self.client.remove('test2', CF, column=COLUMN) res = yield self.client.multiget(['test', 'test2'], CF, column=COLUMN) self.assertEqual(len(res['test']), 0) self.assertEqual(len(res['test2']), 0) @defer.inlineCallbacks def test_range_slices(self): yield self.client.insert('test', CF, 'testval', column=COLUMN) yield self.client.insert('test', CF, 'testval', column=COLUMN2) yield self.client.insert('test2', CF, 'testval2', column=COLUMN) ks = yield self.client.get_range_slices(CF, start='', finish='') keys = [k.key for k in ks] for key in ['test', 'test2']: self.assertIn(key, keys) @defer.inlineCallbacks def test_indexed_slices(self): yield self.client.insert('test1', IDX_CF, 'one', column='col1') yield self.client.insert('test2', IDX_CF, 'two', column='col1') yield self.client.insert('test3', IDX_CF, 'three', column='col1') expressions = [IndexExpression('col1', IndexOperator.EQ, 'two')] res = yield self.client.get_indexed_slices(IDX_CF, expressions, start_key='') self.assertEquals(res[0].columns[0].column.value, 'two') @defer.inlineCallbacks def test_counter_add(self): if self.version != CASSANDRA_08_VERSION: raise unittest.SkipTest('Counters are not supported in 0.7') # test standard column counter yield self.client.add('test', COUNTER_CF, 1, column='col') res = yield self.client.get('test', COUNTER_CF, column='col') self.assertEquals(res.counter_column.value, 1) yield self.client.add('test', COUNTER_CF, 1, column='col') res = yield self.client.get('test', COUNTER_CF, column='col') self.assertEquals(res.counter_column.value, 2) # test super column counters yield self.client.add('test', SUPERCOUNTER_CF, 1, column='col', super_column='scol') res = yield self.client.get('test', SUPERCOUNTER_CF, column='col', super_column='scol') self.assertEquals(res.counter_column.value, 1) yield self.client.add('test', SUPERCOUNTER_CF, 1, column='col', super_column='scol') res = yield self.client.get('test', SUPERCOUNTER_CF, column='col', super_column='scol') self.assertEquals(res.counter_column.value, 2) @defer.inlineCallbacks def test_counter_remove(self): if self.version != CASSANDRA_08_VERSION: raise unittest.SkipTest('Counters are not supported in 0.7') # test standard column counter yield self.client.add('test', COUNTER_CF, 1, column='col') res = yield self.client.get('test', COUNTER_CF, column='col') self.assertEquals(res.counter_column.value, 1) yield self.client.remove_counter('test', COUNTER_CF, column='col') yield self.assertFailure( self.client.get('test', COUNTER_CF, column='col'), NotFoundException) # test super column counters yield self.client.add('test', SUPERCOUNTER_CF, 1, column='col', super_column='scol') res = yield self.client.get('test', SUPERCOUNTER_CF, column='col', super_column='scol') self.assertEquals(res.counter_column.value, 1) yield self.client.remove_counter('test', SUPERCOUNTER_CF, column='col', super_column='scol') yield self.assertFailure( self.client.get('test', SUPERCOUNTER_CF, column='col', super_column='scol'), NotFoundException) def sleep(self, secs): d = defer.Deferred() reactor.callLater(secs, d.callback, None) return d @defer.inlineCallbacks def test_ttls(self): yield self.client.insert('test_ttls', CF, 'testval', column=COLUMN, ttl=1) res = yield self.client.get('test_ttls', CF, column=COLUMN) self.assertEqual(res.column.value, 'testval') yield self.sleep(2) yield self.assertFailure( self.client.get('test_ttls', CF, column=COLUMN), NotFoundException) yield self.client.batch_insert('test_ttls', CF, {COLUMN: 'testval'}, ttl=1) res = yield self.client.get('test_ttls', CF, column=COLUMN) self.assertEqual(res.column.value, 'testval') yield self.sleep(2) yield self.assertFailure( self.client.get('test_ttls', CF, column=COLUMN), NotFoundException) yield self.client.batch_mutate( {'test_ttls': { CF: { COLUMN: 'testval' } }}, ttl=1) res = yield self.client.get('test_ttls', CF, column=COLUMN) self.assertEqual(res.column.value, 'testval') yield self.sleep(2) yield self.assertFailure( self.client.get('test_ttls', CF, column=COLUMN), NotFoundException) def compare_keyspaces(self, ks1, ks2): self.assertEqual(ks1.name, ks2.name) self.assertEqual(ks1.strategy_class, ks2.strategy_class) self.assertEqual(ks1.cf_defs, ks2.cf_defs) def get_rf(ksdef): rf = ksdef.replication_factor if ksdef.strategy_options and \ 'replication_factor' in ksdef.strategy_options: rf = int(ksdef.strategy_options['replication_factor']) return rf def strat_opts_no_rf(ksdef): if not ksdef.strategy_options: return {} opts = ksdef.strategy_options.copy() if 'replication_factor' in ksdef.strategy_options: del opts['replication_factor'] return opts self.assertEqual(get_rf(ks1), get_rf(ks2)) self.assertEqual(strat_opts_no_rf(ks1), strat_opts_no_rf(ks2)) @defer.inlineCallbacks def test_keyspace_manipulation(self): try: yield self.client.system_drop_keyspace(T_KEYSPACE) except InvalidRequestException: pass ksdef = KsDef( name=T_KEYSPACE, strategy_class='org.apache.cassandra.locator.SimpleStrategy', strategy_options={'replication_factor': '1'}, cf_defs=[]) yield self.client.system_add_keyspace(ksdef) ks2 = yield self.client.describe_keyspace(T_KEYSPACE) self.compare_keyspaces(ksdef, ks2) if DO_SYSTEM_RENAMING: newname = T_KEYSPACE + '2' yield self.client.system_rename_keyspace(T_KEYSPACE, newname) ks2 = yield self.client.describe_keyspace(newname) ksdef.name = newname self.compare_keyspaces(ksdef, ks2) yield self.client.system_drop_keyspace(ksdef.name) yield self.assertFailure(self.client.describe_keyspace(T_KEYSPACE), NotFoundException) if DO_SYSTEM_RENAMING: yield self.assertFailure(self.client.describe_keyspace(ksdef.name), NotFoundException) @defer.inlineCallbacks def test_column_family_manipulation(self): cfdef = CfDef( KEYSPACE, T_CF, column_type='Standard', comparator_type='org.apache.cassandra.db.marshal.BytesType', comment='foo', row_cache_size=0.0, key_cache_size=200000.0, read_repair_chance=1.0, column_metadata=[], gc_grace_seconds=86400, default_validation_class= 'org.apache.cassandra.db.marshal.BytesType', key_validation_class='org.apache.cassandra.db.marshal.BytesType', min_compaction_threshold=5, max_compaction_threshold=31, row_cache_save_period_in_seconds=0, key_cache_save_period_in_seconds=3600, memtable_flush_after_mins=60, memtable_throughput_in_mb=249, memtable_operations_in_millions=1.1671875, replicate_on_write=False, merge_shards_chance=0.10000000000000001, row_cache_provider=None, key_alias=None, ) post_07_fields = [ 'replicate_on_write', 'merge_shards_chance', 'key_validation_class', 'row_cache_provider', 'key_alias' ] post_08_fields = [ 'memtable_throughput_in_mb', 'memtable_flush_after_mins', 'memtable_operations_in_millions' ] yield self.client.system_add_column_family(cfdef) ksdef = yield self.client.describe_keyspace(KEYSPACE) cfdef2 = [c for c in ksdef.cf_defs if c.name == T_CF][0] for field in post_07_fields: # Most of these are ignored in 0.7, so we can't reliably compare them setattr(cfdef, field, None) setattr(cfdef2, field, None) for field in post_08_fields: # These fields change from 0.8 to 1.0 setattr(cfdef, field, None) setattr(cfdef2, field, None) # we don't know the id ahead of time. copy the new one so the equality # comparison won't fail cfdef.id = cfdef2.id self.assertEqual(cfdef, cfdef2) if DO_SYSTEM_RENAMING: newname = T_CF + '2' yield self.client.system_rename_column_family(T_CF, newname) ksdef = yield self.client.describe_keyspace(KEYSPACE) cfdef2 = [c for c in ksdef.cf_defs if c.name == newname][0] self.assertNotIn(T_CF, [c.name for c in ksdef.cf_defs]) cfdef.name = newname self.assertEqual(cfdef, cfdef2) yield self.client.system_drop_column_family(cfdef.name) ksdef = yield self.client.describe_keyspace(KEYSPACE) self.assertNotIn(cfdef.name, [c.name for c in ksdef.cf_defs]) @defer.inlineCallbacks def test_describes(self): name = yield self.client.describe_cluster_name() self.assertIsInstance(name, str) self.assertNotEqual(name, '') partitioner = yield self.client.describe_partitioner() self.assert_(partitioner.startswith('org.apache.cassandra.'), msg='partitioner is %r' % partitioner) snitch = yield self.client.describe_snitch() self.assert_(snitch.startswith('org.apache.cassandra.'), msg='snitch is %r' % snitch) version = yield self.client.describe_version() self.assertIsInstance(version, str) self.assertIn('.', version) schemavers = yield self.client.describe_schema_versions() self.assertIsInstance(schemavers, dict) self.assertNotEqual(schemavers, {}) ring = yield self.client.describe_ring(KEYSPACE) self.assertIsInstance(ring, list) self.assertNotEqual(ring, []) for r in ring: self.assertIsInstance(r.start_token, str) self.assertIsInstance(r.end_token, str) self.assertIsInstance(r.endpoints, list) self.assertNotEqual(r.endpoints, []) for ep in r.endpoints: self.assertIsInstance(ep, str) @defer.inlineCallbacks def test_errback(self): yield self.client.remove('poiqwe', CF) try: yield self.client.get('poiqwe', CF, column='foo') except Exception, e: pass
class CassandraClientTest(unittest.TestCase): @defer.inlineCallbacks def setUp(self): self.cmanager = ManagedCassandraClientFactory(keyspace='system') self.client = CassandraClient(self.cmanager) for i in xrange(CONNS): reactor.connectTCP(HOST, PORT, self.cmanager) yield self.cmanager.deferred remote_ver = yield self.client.describe_version() self.version = getAPIVersion(remote_ver) self.my_keyspace = KsDef( name=KEYSPACE, strategy_class='org.apache.cassandra.locator.SimpleStrategy', strategy_options={'replication_factor': '1'}, cf_defs=[ CfDef( keyspace=KEYSPACE, name=CF, column_type='Standard' ), CfDef( keyspace=KEYSPACE, name=SCF, column_type='Super' ), CfDef( keyspace=KEYSPACE, name=IDX_CF, column_type='Standard', comparator_type='org.apache.cassandra.db.marshal.UTF8Type', column_metadata=[ ColumnDef( name='col1', validation_class='org.apache.cassandra.db.marshal.UTF8Type', index_type=IndexType.KEYS, index_name='idxCol1') ], default_validation_class='org.apache.cassandra.db.marshal.BytesType' ), ] ) if self.version == CASSANDRA_08_VERSION: self.my_keyspace.cf_defs.extend([ CfDef( keyspace=KEYSPACE, name=COUNTER_CF, column_type='Standard', default_validation_class='org.apache.cassandra.db.marshal.CounterColumnType' ), CfDef( keyspace=KEYSPACE, name=SUPERCOUNTER_CF, column_type='Super', default_validation_class='org.apache.cassandra.db.marshal.CounterColumnType' ), ]) yield self.client.system_add_keyspace(self.my_keyspace) yield self.client.set_keyspace(KEYSPACE) @defer.inlineCallbacks def tearDown(self): yield self.client.system_drop_keyspace(self.my_keyspace.name) self.cmanager.shutdown() for c in reactor.getDelayedCalls(): c.cancel() reactor.removeAll() @defer.inlineCallbacks def test_insert_get(self): yield self.client.insert('test', CF, 'testval', column=COLUMN) yield self.client.insert('test2', CF, 'testval2', column=COLUMN) yield self.client.insert('test', SCF, 'superval', column=COLUMN, super_column=SCOLUMN) yield self.client.insert('test2', SCF, 'superval2', column=COLUMN, super_column=SCOLUMN) res = yield self.client.get('test', CF, column=COLUMN) self.assertEqual(res.column.value, 'testval') res = yield self.client.get('test2', CF, column=COLUMN) self.assertEqual(res.column.value, 'testval2') res = yield self.client.get('test', SCF, column=COLUMN, super_column=SCOLUMN) self.assertEqual(res.column.value, 'superval') res = yield self.client.get('test2', SCF, column=COLUMN, super_column=SCOLUMN) self.assertEqual(res.column.value, 'superval2') @defer.inlineCallbacks def test_batch_insert_get_slice_and_count(self): yield self.client.batch_insert('test', CF, {COLUMN: 'test', COLUMN2: 'test2'}) yield self.client.batch_insert('test', SCF, {SCOLUMN: {COLUMN: 'test', COLUMN2: 'test2'}}) res = yield self.client.get_slice('test', CF, names=(COLUMN, COLUMN2)) self.assertEqual(res[0].column.value, 'test') self.assertEqual(res[1].column.value, 'test2') res = yield self.client.get_slice('test', SCF, names=(COLUMN, COLUMN2), super_column=SCOLUMN) self.assertEqual(res[0].column.value, 'test') self.assertEqual(res[1].column.value, 'test2') res = yield self.client.get_count('test', CF) self.assertEqual(res, 2) @defer.inlineCallbacks def test_batch_mutate_and_remove(self): yield self.client.batch_mutate({'test': {CF: {COLUMN: 'test', COLUMN2: 'test2'}, SCF: { SCOLUMN: { COLUMN: 'test', COLUMN2: 'test2'} } }, 'test2': {CF: {COLUMN: 'test', COLUMN2: 'test2'}, SCF: { SCOLUMN: { COLUMN: 'test', COLUMN2: 'test2'} } } }) res = yield self.client.get_slice('test', CF, names=(COLUMN, COLUMN2)) self.assertEqual(res[0].column.value, 'test') self.assertEqual(res[1].column.value, 'test2') res = yield self.client.get_slice('test2', CF, names=(COLUMN, COLUMN2)) self.assertEqual(res[0].column.value, 'test') self.assertEqual(res[1].column.value, 'test2') res = yield self.client.get_slice('test', SCF, names=(COLUMN, COLUMN2), super_column=SCOLUMN) self.assertEqual(res[0].column.value, 'test') self.assertEqual(res[1].column.value, 'test2') res = yield self.client.get_slice('test2', SCF, names=(COLUMN, COLUMN2), super_column=SCOLUMN) self.assertEqual(res[0].column.value, 'test') self.assertEqual(res[1].column.value, 'test2') yield self.client.batch_remove({CF: ['test', 'test2']}, names=['test', 'test2']) yield self.client.batch_remove({SCF: ['test', 'test2']}, names=['test', 'test2'], supercolumn=SCOLUMN) @defer.inlineCallbacks def test_batch_mutate_with_deletion(self): yield self.client.batch_mutate({'test': {CF: {COLUMN: 'test', COLUMN2: 'test2'}}}) res = yield self.client.get_slice('test', CF, names=(COLUMN, COLUMN2)) self.assertEqual(res[0].column.value, 'test') self.assertEqual(res[1].column.value, 'test2') yield self.client.batch_mutate({'test': {CF: {COLUMN: None, COLUMN2: 'test3'}}}) res = yield self.client.get_slice('test', CF, names=(COLUMN, COLUMN2)) self.assertEqual(len(res), 1) self.assertEqual(res[0].column.value, 'test3') @defer.inlineCallbacks def test_multiget_slice_remove(self): yield self.client.insert('test', CF, 'testval', column=COLUMN) yield self.client.insert('test', CF, 'testval', column=COLUMN2) yield self.client.insert('test2', CF, 'testval2', column=COLUMN) res = yield self.client.multiget(['test', 'test2'], CF, column=COLUMN) self.assertEqual(res['test'][0].column.value, 'testval') self.assertEqual(res['test2'][0].column.value, 'testval2') res = yield self.client.multiget_slice(['test', 'test2'], CF) self.assertEqual(res['test'][0].column.value, 'testval') self.assertEqual(res['test'][1].column.value, 'testval') self.assertEqual(res['test2'][0].column.value, 'testval2') yield self.client.remove('test', CF, column=COLUMN) yield self.client.remove('test2', CF, column=COLUMN) res = yield self.client.multiget(['test', 'test2'], CF, column=COLUMN) self.assertEqual(len(res['test']), 0) self.assertEqual(len(res['test2']), 0) @defer.inlineCallbacks def test_range_slices(self): yield self.client.insert('test', CF, 'testval', column=COLUMN) yield self.client.insert('test', CF, 'testval', column=COLUMN2) yield self.client.insert('test2', CF, 'testval2', column=COLUMN) ks = yield self.client.get_range_slices(CF, start='', finish='') keys = [k.key for k in ks] for key in ['test', 'test2']: self.assertIn(key, keys) @defer.inlineCallbacks def test_indexed_slices(self): yield self.client.insert('test1', IDX_CF, 'one', column='col1') yield self.client.insert('test2', IDX_CF, 'two', column='col1') yield self.client.insert('test3', IDX_CF, 'three', column='col1') expressions = [IndexExpression('col1', IndexOperator.EQ, 'two')] res = yield self.client.get_indexed_slices(IDX_CF, expressions, start_key='') self.assertEquals(res[0].columns[0].column.value,'two') @defer.inlineCallbacks def test_counter_add(self): if self.version != CASSANDRA_08_VERSION: raise unittest.SkipTest('Counters are not supported in 0.7') # test standard column counter yield self.client.add('test', COUNTER_CF, 1, column='col') res = yield self.client.get('test', COUNTER_CF, column='col') self.assertEquals(res.counter_column.value, 1) yield self.client.add('test', COUNTER_CF, 1, column='col') res = yield self.client.get('test', COUNTER_CF, column='col') self.assertEquals(res.counter_column.value, 2) # test super column counters yield self.client.add('test', SUPERCOUNTER_CF, 1, column='col', super_column='scol') res = yield self.client.get('test', SUPERCOUNTER_CF, column='col', super_column='scol') self.assertEquals(res.counter_column.value, 1) yield self.client.add('test', SUPERCOUNTER_CF, 1, column='col', super_column='scol') res = yield self.client.get('test', SUPERCOUNTER_CF, column='col', super_column='scol') self.assertEquals(res.counter_column.value, 2) @defer.inlineCallbacks def test_counter_remove(self): if self.version != CASSANDRA_08_VERSION: raise unittest.SkipTest('Counters are not supported in 0.7') # test standard column counter yield self.client.add('test', COUNTER_CF, 1, column='col') res = yield self.client.get('test', COUNTER_CF, column='col') self.assertEquals(res.counter_column.value, 1) yield self.client.remove_counter('test', COUNTER_CF, column='col') yield self.assertFailure(self.client.get('test', COUNTER_CF, column='col'), NotFoundException) # test super column counters yield self.client.add('test', SUPERCOUNTER_CF, 1, column='col', super_column='scol') res = yield self.client.get('test', SUPERCOUNTER_CF, column='col', super_column='scol') self.assertEquals(res.counter_column.value, 1) yield self.client.remove_counter('test', SUPERCOUNTER_CF, column='col', super_column='scol') yield self.assertFailure(self.client.get('test', SUPERCOUNTER_CF, column='col', super_column='scol'), NotFoundException) @defer.inlineCallbacks def test_cql(self): if self.version != CASSANDRA_08_VERSION: raise unittest.SkipTest('CQL is not supported in 0.7') yield self.client.insert('test', CF, 'testval', column='col1') res = yield self.client.get('test', CF, column='col1') self.assertEquals(res.column.value, 'testval') query = 'SELECT * from %s where KEY = %s' % (CF, 'test'.encode('hex')) uncompressed_result = yield self.client.execute_cql_query(query, Compression.NONE) self.assertEquals(uncompressed_result.rows[0].columns[0].name, 'col1') self.assertEquals(uncompressed_result.rows[0].columns[0].value, 'testval') compressed_query = zlib.compress(query) compressed_result = yield self.client.execute_cql_query(compressed_query, Compression.GZIP) self.assertEquals(uncompressed_result, compressed_result) def sleep(self, secs): d = defer.Deferred() reactor.callLater(secs, d.callback, None) return d @defer.inlineCallbacks def test_ttls(self): yield self.client.insert('test_ttls', CF, 'testval', column=COLUMN, ttl=1) res = yield self.client.get('test_ttls', CF, column=COLUMN) self.assertEqual(res.column.value, 'testval') yield self.sleep(2) yield self.assertFailure(self.client.get('test_ttls', CF, column=COLUMN), NotFoundException) yield self.client.batch_insert('test_ttls', CF, {COLUMN:'testval'}, ttl=1) res = yield self.client.get('test_ttls', CF, column=COLUMN) self.assertEqual(res.column.value, 'testval') yield self.sleep(2) yield self.assertFailure(self.client.get('test_ttls', CF, column=COLUMN), NotFoundException) yield self.client.batch_mutate({'test_ttls': {CF: {COLUMN: 'testval'}}}, ttl=1) res = yield self.client.get('test_ttls', CF, column=COLUMN) self.assertEqual(res.column.value, 'testval') yield self.sleep(2) yield self.assertFailure(self.client.get('test_ttls', CF, column=COLUMN), NotFoundException) def compare_keyspaces(self, ks1, ks2): self.assertEqual(ks1.name, ks2.name) self.assertEqual(ks1.strategy_class, ks2.strategy_class) self.assertEqual(ks1.cf_defs, ks2.cf_defs) def get_rf(ksdef): rf = ksdef.replication_factor if ksdef.strategy_options and \ 'replication_factor' in ksdef.strategy_options: rf = int(ksdef.strategy_options['replication_factor']) return rf def strat_opts_no_rf(ksdef): if not ksdef.strategy_options: return {} opts = ksdef.strategy_options.copy() if 'replication_factor' in ksdef.strategy_options: del opts['replication_factor'] return opts self.assertEqual(get_rf(ks1), get_rf(ks2)) self.assertEqual(strat_opts_no_rf(ks1), strat_opts_no_rf(ks2)) @defer.inlineCallbacks def test_keyspace_manipulation(self): try: yield self.client.system_drop_keyspace(T_KEYSPACE) except InvalidRequestException: pass ksdef = KsDef(name=T_KEYSPACE, strategy_class='org.apache.cassandra.locator.SimpleStrategy', strategy_options={'replication_factor': '1'}, cf_defs=[]) yield self.client.system_add_keyspace(ksdef) ks2 = yield self.client.describe_keyspace(T_KEYSPACE) self.compare_keyspaces(ksdef, ks2) if DO_SYSTEM_RENAMING: newname = T_KEYSPACE + '2' yield self.client.system_rename_keyspace(T_KEYSPACE, newname) ks2 = yield self.client.describe_keyspace(newname) ksdef.name = newname self.compare_keyspaces(ksdef, ks2) yield self.client.system_drop_keyspace(ksdef.name) yield self.assertFailure(self.client.describe_keyspace(T_KEYSPACE), NotFoundException) if DO_SYSTEM_RENAMING: yield self.assertFailure(self.client.describe_keyspace(ksdef.name), NotFoundException) @defer.inlineCallbacks def test_column_family_manipulation(self): cfdef = CfDef(KEYSPACE, T_CF, column_type='Standard', comparator_type='org.apache.cassandra.db.marshal.BytesType', comment='foo', row_cache_size=0.0, key_cache_size=200000.0, read_repair_chance=1.0, column_metadata=[], gc_grace_seconds=86400, default_validation_class='org.apache.cassandra.db.marshal.BytesType', key_validation_class='org.apache.cassandra.db.marshal.BytesType', min_compaction_threshold=5, max_compaction_threshold=31, row_cache_save_period_in_seconds=0, key_cache_save_period_in_seconds=3600, memtable_flush_after_mins=60, memtable_throughput_in_mb=249, memtable_operations_in_millions=1.1671875, replicate_on_write=False, merge_shards_chance=0.10000000000000001, row_cache_provider=None, key_alias=None, ) post_07_fields = ['replicate_on_write', 'merge_shards_chance', 'key_validation_class', 'row_cache_provider', 'key_alias'] yield self.client.system_add_column_family(cfdef) ksdef = yield self.client.describe_keyspace(KEYSPACE) cfdef2 = [c for c in ksdef.cf_defs if c.name == T_CF][0] for field in post_07_fields: # Most of these are ignored in 0.7, so we can't reliably compare them setattr(cfdef, field, None) setattr(cfdef2, field, None) # we don't know the id ahead of time. copy the new one so the equality # comparison won't fail cfdef.id = cfdef2.id self.assertEqual(cfdef, cfdef2) if DO_SYSTEM_RENAMING: newname = T_CF + '2' yield self.client.system_rename_column_family(T_CF, newname) ksdef = yield self.client.describe_keyspace(KEYSPACE) cfdef2 = [c for c in ksdef.cf_defs if c.name == newname][0] self.assertNotIn(T_CF, [c.name for c in ksdef.cf_defs]) cfdef.name = newname self.assertEqual(cfdef, cfdef2) yield self.client.system_drop_column_family(cfdef.name) ksdef = yield self.client.describe_keyspace(KEYSPACE) self.assertNotIn(cfdef.name, [c.name for c in ksdef.cf_defs]) @defer.inlineCallbacks def test_describes(self): name = yield self.client.describe_cluster_name() self.assertIsInstance(name, str) self.assertNotEqual(name, '') partitioner = yield self.client.describe_partitioner() self.assert_(partitioner.startswith('org.apache.cassandra.'), msg='partitioner is %r' % partitioner) snitch = yield self.client.describe_snitch() self.assert_(snitch.startswith('org.apache.cassandra.'), msg='snitch is %r' % snitch) version = yield self.client.describe_version() self.assertIsInstance(version, str) self.assertIn('.', version) schemavers = yield self.client.describe_schema_versions() self.assertIsInstance(schemavers, dict) self.assertNotEqual(schemavers, {}) ring = yield self.client.describe_ring(KEYSPACE) self.assertIsInstance(ring, list) self.assertNotEqual(ring, []) for r in ring: self.assertIsInstance(r.start_token, str) self.assertIsInstance(r.end_token, str) self.assertIsInstance(r.endpoints, list) self.assertNotEqual(r.endpoints, []) for ep in r.endpoints: self.assertIsInstance(ep, str) @defer.inlineCallbacks def test_errback(self): yield self.client.remove('poiqwe', CF) try: yield self.client.get('poiqwe', CF, column='foo') except Exception, e: pass
class CassandraClientTest(unittest.TestCase): @defer.inlineCallbacks def setUp(self): self.cmanager = ManagedCassandraClientFactory(keyspace='system') self.client = CassandraClient(self.cmanager) for i in xrange(CONNS): reactor.connectTCP(HOST, PORT, self.cmanager) yield self.cmanager.deferred remote_ver = yield self.client.describe_version() self.version = tuple(map(int, remote_ver.split('.'))) self.my_keyspace = ttypes.KsDef( name=KEYSPACE, strategy_class='org.apache.cassandra.locator.SimpleStrategy', strategy_options={}, cf_defs=[ ttypes.CfDef( keyspace=KEYSPACE, name=CF, column_type='Standard' ), ttypes.CfDef( keyspace=KEYSPACE, name=SCF, column_type='Super' ), ttypes.CfDef( keyspace=KEYSPACE, name=IDX_CF, column_type='Standard', comparator_type='org.apache.cassandra.db.marshal.UTF8Type', column_metadata=[ ttypes.ColumnDef( name='col1', validation_class='org.apache.cassandra.db.marshal.UTF8Type', index_type=ttypes.IndexType.KEYS, index_name='idxCol1') ], default_validation_class='org.apache.cassandra.db.marshal.BytesType' ), ] ) if self.version <= KS_RF_ATTRIBUTE: self.my_keyspace.replication_factor = 1 else: self.my_keyspace.strategy_options['replication_factor'] = '1' if self.version >= COUNTERS_SUPPORTED_API: self.my_keyspace.cf_defs.extend([ ttypes.CfDef( keyspace=KEYSPACE, name=COUNTER_CF, column_type='Standard', default_validation_class='org.apache.cassandra.db.marshal.CounterColumnType' ), ttypes.CfDef( keyspace=KEYSPACE, name=SUPERCOUNTER_CF, column_type='Super', default_validation_class='org.apache.cassandra.db.marshal.CounterColumnType' ), ]) yield self.client.system_add_keyspace(self.my_keyspace) yield self.client.set_keyspace(KEYSPACE) @defer.inlineCallbacks def tearDown(self): yield self.client.system_drop_keyspace(self.my_keyspace.name) self.cmanager.shutdown() for c in reactor.getDelayedCalls(): c.cancel() reactor.removeAll() @defer.inlineCallbacks def test_insert_get(self): yield self.client.insert('test', CF, 'testval', column=COLUMN) yield self.client.insert('test2', CF, 'testval2', column=COLUMN) yield self.client.insert('test', SCF, 'superval', column=COLUMN, super_column=SCOLUMN) yield self.client.insert('test2', SCF, 'superval2', column=COLUMN, super_column=SCOLUMN) res = yield self.client.get('test', CF, column=COLUMN) self.assertEqual(res.column.value, 'testval') res = yield self.client.get('test2', CF, column=COLUMN) self.assertEqual(res.column.value, 'testval2') res = yield self.client.get('test', SCF, column=COLUMN, super_column=SCOLUMN) self.assertEqual(res.column.value, 'superval') res = yield self.client.get('test2', SCF, column=COLUMN, super_column=SCOLUMN) self.assertEqual(res.column.value, 'superval2') @defer.inlineCallbacks def test_batch_insert_get_slice_and_count(self): yield self.client.batch_insert('test', CF, {COLUMN: 'test', COLUMN2: 'test2'}) yield self.client.batch_insert('test', SCF, {SCOLUMN: {COLUMN: 'test', COLUMN2: 'test2'}}) res = yield self.client.get_slice('test', CF, names=(COLUMN, COLUMN2)) self.assertEqual(res[0].column.value, 'test') self.assertEqual(res[1].column.value, 'test2') res = yield self.client.get_slice('test', SCF, names=(COLUMN, COLUMN2), super_column=SCOLUMN) self.assertEqual(res[0].column.value, 'test') self.assertEqual(res[1].column.value, 'test2') res = yield self.client.get_count('test', CF) self.assertEqual(res, 2) @defer.inlineCallbacks def test_batch_mutate_and_remove(self): yield self.client.batch_mutate({'test': {CF: {COLUMN: 'test', COLUMN2: 'test2'}, SCF: { SCOLUMN: { COLUMN: 'test', COLUMN2: 'test2'} } }, 'test2': {CF: {COLUMN: 'test', COLUMN2: 'test2'}, SCF: { SCOLUMN: { COLUMN: 'test', COLUMN2: 'test2'} } } }) res = yield self.client.get_slice('test', CF, names=(COLUMN, COLUMN2)) self.assertEqual(res[0].column.value, 'test') self.assertEqual(res[1].column.value, 'test2') res = yield self.client.get_slice('test2', CF, names=(COLUMN, COLUMN2)) self.assertEqual(res[0].column.value, 'test') self.assertEqual(res[1].column.value, 'test2') res = yield self.client.get_slice('test', SCF, names=(COLUMN, COLUMN2), super_column=SCOLUMN) self.assertEqual(res[0].column.value, 'test') self.assertEqual(res[1].column.value, 'test2') res = yield self.client.get_slice('test2', SCF, names=(COLUMN, COLUMN2), super_column=SCOLUMN) self.assertEqual(res[0].column.value, 'test') self.assertEqual(res[1].column.value, 'test2') yield self.client.batch_remove({CF: ['test', 'test2']}, names=['test', 'test2']) yield self.client.batch_remove({SCF: ['test', 'test2']}, names=['test', 'test2'], supercolumn=SCOLUMN) @defer.inlineCallbacks def test_batch_mutate_with_deletion(self): yield self.client.batch_mutate({'test': {CF: {COLUMN: 'test', COLUMN2: 'test2'}}}) res = yield self.client.get_slice('test', CF, names=(COLUMN, COLUMN2)) self.assertEqual(res[0].column.value, 'test') self.assertEqual(res[1].column.value, 'test2') yield self.client.batch_mutate({'test': {CF: {COLUMN: None, COLUMN2: 'test3'}}}) res = yield self.client.get_slice('test', CF, names=(COLUMN, COLUMN2)) self.assertEqual(len(res), 1) self.assertEqual(res[0].column.value, 'test3') @defer.inlineCallbacks def test_multiget_slice_remove(self): yield self.client.insert('test', CF, 'testval', column=COLUMN) yield self.client.insert('test', CF, 'testval', column=COLUMN2) yield self.client.insert('test2', CF, 'testval2', column=COLUMN) res = yield self.client.multiget(['test', 'test2'], CF, column=COLUMN) self.assertEqual(res['test'][0].column.value, 'testval') self.assertEqual(res['test2'][0].column.value, 'testval2') res = yield self.client.multiget_slice(['test', 'test2'], CF) self.assertEqual(res['test'][0].column.value, 'testval') self.assertEqual(res['test'][1].column.value, 'testval') self.assertEqual(res['test2'][0].column.value, 'testval2') yield self.client.remove('test', CF, column=COLUMN) yield self.client.remove('test2', CF, column=COLUMN) res = yield self.client.multiget(['test', 'test2'], CF, column=COLUMN) self.assertEqual(len(res['test']), 0) self.assertEqual(len(res['test2']), 0) @defer.inlineCallbacks def test_range_slices(self): yield self.client.insert('test', CF, 'testval', column=COLUMN) yield self.client.insert('test', CF, 'testval', column=COLUMN2) yield self.client.insert('test2', CF, 'testval2', column=COLUMN) ks = yield self.client.get_range_slices(CF, start='', finish='') keys = [k.key for k in ks] for key in ['test', 'test2']: self.assertIn(key, keys) @defer.inlineCallbacks def test_indexed_slices(self): yield self.client.insert('test1', IDX_CF, 'one', column='col1') yield self.client.insert('test2', IDX_CF, 'two', column='col1') yield self.client.insert('test3', IDX_CF, 'three', column='col1') expressions = [ttypes.IndexExpression('col1', ttypes.IndexOperator.EQ, 'two')] res = yield self.client.get_indexed_slices(IDX_CF, expressions, start_key='') self.assertEquals(res[0].columns[0].column.value,'two') @defer.inlineCallbacks def test_counter_add(self): if self.version < COUNTERS_SUPPORTED_API: raise unittest.SkipTest('Counters are not supported before 0.8') # test standard column counter yield self.client.add('test', COUNTER_CF, 1, column='col') res = yield self.client.get('test', COUNTER_CF, column='col') self.assertEquals(res.counter_column.value, 1) yield self.client.add('test', COUNTER_CF, 1, column='col') res = yield self.client.get('test', COUNTER_CF, column='col') self.assertEquals(res.counter_column.value, 2) # test super column counters yield self.client.add('test', SUPERCOUNTER_CF, 1, column='col', super_column='scol') res = yield self.client.get('test', SUPERCOUNTER_CF, column='col', super_column='scol') self.assertEquals(res.counter_column.value, 1) yield self.client.add('test', SUPERCOUNTER_CF, 1, column='col', super_column='scol') res = yield self.client.get('test', SUPERCOUNTER_CF, column='col', super_column='scol') self.assertEquals(res.counter_column.value, 2) @defer.inlineCallbacks def test_counter_remove(self): if self.version < COUNTERS_SUPPORTED_API: raise unittest.SkipTest('Counters are not supported before 0.8') # test standard column counter yield self.client.add('test', COUNTER_CF, 1, column='col') res = yield self.client.get('test', COUNTER_CF, column='col') self.assertEquals(res.counter_column.value, 1) yield self.client.remove_counter('test', COUNTER_CF, column='col') yield self.assertFailure(self.client.get('test', COUNTER_CF, column='col'), ttypes.NotFoundException) # test super column counters yield self.client.add('test', SUPERCOUNTER_CF, 1, column='col', super_column='scol') res = yield self.client.get('test', SUPERCOUNTER_CF, column='col', super_column='scol') self.assertEquals(res.counter_column.value, 1) yield self.client.remove_counter('test', SUPERCOUNTER_CF, column='col', super_column='scol') yield self.assertFailure(self.client.get('test', SUPERCOUNTER_CF, column='col', super_column='scol'), ttypes.NotFoundException) def sleep(self, secs): d = defer.Deferred() reactor.callLater(secs, d.callback, None) return d @defer.inlineCallbacks def test_ttls(self): yield self.client.insert('test_ttls', CF, 'testval', column=COLUMN, ttl=1) res = yield self.client.get('test_ttls', CF, column=COLUMN) self.assertEqual(res.column.value, 'testval') yield self.sleep(2) yield self.assertFailure(self.client.get('test_ttls', CF, column=COLUMN), ttypes.NotFoundException) yield self.client.batch_insert('test_ttls', CF, {COLUMN:'testval'}, ttl=1) res = yield self.client.get('test_ttls', CF, column=COLUMN) self.assertEqual(res.column.value, 'testval') yield self.sleep(2) yield self.assertFailure(self.client.get('test_ttls', CF, column=COLUMN), ttypes.NotFoundException) yield self.client.batch_mutate({'test_ttls': {CF: {COLUMN: 'testval'}}}, ttl=1) res = yield self.client.get('test_ttls', CF, column=COLUMN) self.assertEqual(res.column.value, 'testval') yield self.sleep(2) yield self.assertFailure(self.client.get('test_ttls', CF, column=COLUMN), ttypes.NotFoundException) def compare_keyspaces(self, ks1, ks2): self.assertEqual(ks1.name, ks2.name) self.assertEqual(ks1.strategy_class, ks2.strategy_class) self.assertEqual(ks1.cf_defs, ks2.cf_defs) def get_rf(ksdef): rf = ksdef.replication_factor if ksdef.strategy_options and \ 'replication_factor' in ksdef.strategy_options: rf = int(ksdef.strategy_options['replication_factor']) return rf def strat_opts_no_rf(ksdef): if not ksdef.strategy_options: return {} opts = ksdef.strategy_options.copy() if 'replication_factor' in ksdef.strategy_options: del opts['replication_factor'] return opts self.assertEqual(get_rf(ks1), get_rf(ks2)) self.assertEqual(strat_opts_no_rf(ks1), strat_opts_no_rf(ks2)) @defer.inlineCallbacks def test_keyspace_manipulation(self): try: yield self.client.system_drop_keyspace(T_KEYSPACE) except ttypes.InvalidRequestException: pass ksdef = ttypes.KsDef(name=T_KEYSPACE, strategy_class='org.apache.cassandra.locator.SimpleStrategy', strategy_options={}, cf_defs=[]) if self.version <= KS_RF_ATTRIBUTE: ksdef.replication_factor = 1 else: ksdef.strategy_options['replication_factor'] = '1' yield self.client.system_add_keyspace(ksdef) ks2 = yield self.client.describe_keyspace(T_KEYSPACE) self.compare_keyspaces(ksdef, ks2) if DO_SYSTEM_RENAMING: newname = T_KEYSPACE + '2' yield self.client.system_rename_keyspace(T_KEYSPACE, newname) ks2 = yield self.client.describe_keyspace(newname) ksdef.name = newname self.compare_keyspaces(ksdef, ks2) yield self.client.system_drop_keyspace(ksdef.name) yield self.assertFailure(self.client.describe_keyspace(T_KEYSPACE), ttypes.NotFoundException) if DO_SYSTEM_RENAMING: yield self.assertFailure(self.client.describe_keyspace(ksdef.name), ttypes.NotFoundException) @defer.inlineCallbacks def test_column_family_manipulation(self): # CfDef attributes present in all supported c*/thrift-api versions common_attrs = ( ('column_type', 'Standard'), ('comparator_type', 'org.apache.cassandra.db.marshal.BytesType'), ('comment', 'foo'), ('read_repair_chance', 1.0), ('column_metadata', []), ('gc_grace_seconds', 86400), ('default_validation_class', 'org.apache.cassandra.db.marshal.BytesType'), ('min_compaction_threshold', 5), ('max_compaction_threshold', 31), ) cfdef = ttypes.CfDef(KEYSPACE, T_CF) for attr, val in common_attrs: setattr(cfdef, attr, val) yield self.client.system_add_column_family(cfdef) ksdef = yield self.client.describe_keyspace(KEYSPACE) cfdefs = [c for c in ksdef.cf_defs if c.name == T_CF] self.assertEqual(len(cfdefs), 1) cfdef2 = cfdefs[0] for attr, val in common_attrs: val1 = getattr(cfdef, attr) val2 = getattr(cfdef2, attr) self.assertEqual(val1, val2, 'attribute %s mismatch: %r != %r' % (attr, val1, val2)) if DO_SYSTEM_RENAMING: newname = T_CF + '2' yield self.client.system_rename_column_family(T_CF, newname) ksdef = yield self.client.describe_keyspace(KEYSPACE) cfdef2 = [c for c in ksdef.cf_defs if c.name == newname][0] self.assertNotIn(T_CF, [c.name for c in ksdef.cf_defs]) cfdef.name = newname self.assertEqual(cfdef, cfdef2) yield self.client.system_drop_column_family(cfdef.name) ksdef = yield self.client.describe_keyspace(KEYSPACE) self.assertNotIn(cfdef.name, [c.name for c in ksdef.cf_defs]) @defer.inlineCallbacks def test_describes(self): name = yield self.client.describe_cluster_name() self.assertIsInstance(name, str) self.assertNotEqual(name, '') partitioner = yield self.client.describe_partitioner() self.assert_(partitioner.startswith('org.apache.cassandra.'), msg='partitioner is %r' % partitioner) snitch = yield self.client.describe_snitch() self.assert_(snitch.startswith('org.apache.cassandra.'), msg='snitch is %r' % snitch) version = yield self.client.describe_version() self.assertIsInstance(version, str) self.assertIn('.', version) schemavers = yield self.client.describe_schema_versions() self.assertIsInstance(schemavers, dict) self.assertNotEqual(schemavers, {}) ring = yield self.client.describe_ring(KEYSPACE) self.assertIsInstance(ring, list) self.assertNotEqual(ring, []) for r in ring: self.assertIsInstance(r.start_token, str) self.assertIsInstance(r.end_token, str) self.assertIsInstance(r.endpoints, list) self.assertNotEqual(r.endpoints, []) for ep in r.endpoints: self.assertIsInstance(ep, str) @defer.inlineCallbacks def test_errback(self): yield self.client.remove('poiqwe', CF) try: yield self.client.get('poiqwe', CF, column='foo') except Exception, e: pass
class CassandraClientTest(unittest.TestCase): def setUp(self): self.cmanager = ManagedCassandraClientFactory() self.client = CassandraClient(self.cmanager, KEYSPACE) for i in xrange(CONNS): reactor.connectTCP(HOST, PORT, self.cmanager) return self.cmanager.deferred @defer.inlineCallbacks def tearDown(self): yield self.client.remove('test', CF) yield self.client.remove('test2', CF) yield self.client.remove('test', SCF) yield self.client.remove('test2', SCF) self.cmanager.shutdown() for c in reactor.getDelayedCalls(): c.cancel() reactor.removeAll() @defer.inlineCallbacks def test_insert_get(self): yield self.client.insert('test', ColumnPath(CF, None, COLUMN), 'testval') yield self.client.insert('test2', CF, 'testval2', column=COLUMN) yield self.client.insert('test', ColumnPath(SCF, SCOLUMN, COLUMN), 'superval') yield self.client.insert('test2', SCF, 'superval2', column=COLUMN, super_column=SCOLUMN) res = yield self.client.get('test', CF, column=COLUMN) self.assert_(res.column.value == 'testval') res = yield self.client.get('test2', CF, column=COLUMN) self.assert_(res.column.value == 'testval2') res = yield self.client.get('test', SCF, column=COLUMN, super_column=SCOLUMN) self.assert_(res.column.value == 'superval') res = yield self.client.get('test2', SCF, column=COLUMN, super_column=SCOLUMN) self.assert_(res.column.value == 'superval2') @defer.inlineCallbacks def test_batch_insert_get_slice_and_count(self): yield self.client.batch_insert('test', CF, {COLUMN: 'test', COLUMN2: 'test2'}) yield self.client.batch_insert('test', SCF, {SCOLUMN: {COLUMN: 'test', COLUMN2: 'test2'}}) res = yield self.client.get_slice('test', CF, names=(COLUMN, COLUMN2)) self.assert_(res[0].column.value == 'test') self.assert_(res[1].column.value == 'test2') res = yield self.client.get_slice('test', SCF, names=(COLUMN, COLUMN2), super_column=SCOLUMN) self.assert_(res[0].column.value == 'test') self.assert_(res[1].column.value == 'test2') res = yield self.client.get_count('test', CF) self.assert_(res == 2) @defer.inlineCallbacks def test_multiget_slice_remove(self): yield self.client.insert('test', CF, 'testval', column=COLUMN) yield self.client.insert('test', CF, 'testval', column=COLUMN2) yield self.client.insert('test2', CF, 'testval2', column=COLUMN) res = yield self.client.multiget(['test', 'test2'], CF, column=COLUMN) self.assert_(res['test'].column.value == 'testval') self.assert_(res['test2'].column.value == 'testval2') res = yield self.client.multiget_slice(['test', 'test2'], CF) self.assert_(res['test'][0].column.value == 'testval') self.assert_(res['test'][1].column.value == 'testval') self.assert_(res['test2'][0].column.value == 'testval2') yield self.client.remove('test', CF, column=COLUMN) yield self.client.remove('test2', CF, column=COLUMN) res = yield self.client.multiget(['test', 'test2'], CF, column=COLUMN) self.assert_(res['test'].column == None) self.assert_(res['test2'].column == None)
class CassandraSchemaManager(object): """Manages creation and destruction of cassandra schemas. Useful for both testing and production """ def __init__(self, keyspace_def, error_if_existing=False): self.keyspace_def = keyspace_def self.error_if_existing = error_if_existing self.created_keyspace = False self.created_cfs = [] self.client = None self.manager = None self.connector = None def connect(self, host=None, port=9160, username=None, password=None): if not host: host, port = get_host_port() if username or password: if not (username and password): raise CassandraConfigurationError( "Specify both username and password or neither") else: username, password = get_credentials() authz = dict(username=username, password=password) self.manager = ManagedCassandraClientFactory(credentials=authz, check_api_version=True) self.connector = reactor.connectTCP(host, port, self.manager) self.client = CassandraClient(self.manager) def disconnect(self): if self.manager: self.manager.shutdown() if self.connector: self.connector.disconnect() @timeout(DEFAULT_CASSANDRA_TIMEOUT) @defer.inlineCallbacks def create(self, truncate=False): if not self.client: self.connect() keyspace = self.keyspace_def try: existing = yield self.client.describe_keyspace(keyspace.name) except NotFoundException: existing = None # keyspace already exists if existing: yield self.client.set_keyspace(keyspace.name) _compare_ks_properties(existing, keyspace) existing_cfs = dict((cf.name, cf) for cf in existing.cf_defs) for cf in keyspace.cf_defs: if cf.name in existing_cfs: if truncate: # in truncate mode we drop and readd any existing CFs. yield self.client.system_drop_column_family(cf.name) yield self.client.system_add_column_family(cf) else: _compare_cf_properties(existing_cfs[cf.name], cf) else: if cf.keyspace != keyspace.name: raise CassandraSchemaError( "CF %s has wrong keyspace name", cf.name) self.created_cfs.append(cf.name) yield self.client.system_add_column_family(cf) else: self.created_keyspace = True yield self.client.system_add_keyspace(keyspace) yield self.client.set_keyspace(keyspace.name) @timeout(DEFAULT_CASSANDRA_TIMEOUT) @defer.inlineCallbacks def teardown(self): if self.created_keyspace: yield self.client.system_drop_keyspace(self.keyspace_def.name) elif self.created_cfs: for cf in self.created_cfs: yield self.client.system_drop_column_family(cf)
def setUp(self): self.cmanager = ManagedCassandraClientFactory() self.client = CassandraClient(self.cmanager, KEYSPACE) for i in xrange(CONNS): reactor.connectTCP(HOST, PORT, self.cmanager) return self.cmanager.deferred
class CassandraClientTest(unittest.TestCase): def setUp(self): self.cmanager = ManagedCassandraClientFactory() self.client = CassandraClient(self.cmanager, KEYSPACE) for i in xrange(CONNS): reactor.connectTCP(HOST, PORT, self.cmanager) return self.cmanager.deferred @defer.inlineCallbacks def tearDown(self): yield self.client.remove('test', CF) yield self.client.remove('test2', CF) yield self.client.remove('test', SCF) yield self.client.remove('test2', SCF) self.cmanager.shutdown() for c in reactor.getDelayedCalls(): c.cancel() reactor.removeAll() @defer.inlineCallbacks def test_insert_get(self): yield self.client.insert('test', ColumnPath(CF, None, COLUMN), 'testval') yield self.client.insert('test2', CF, 'testval2', column=COLUMN) yield self.client.insert('test', ColumnPath(SCF, SCOLUMN, COLUMN), 'superval') yield self.client.insert('test2', SCF, 'superval2', column=COLUMN, super_column=SCOLUMN) res = yield self.client.get('test', CF, column=COLUMN) self.assert_(res.column.value == 'testval') res = yield self.client.get('test2', CF, column=COLUMN) self.assert_(res.column.value == 'testval2') res = yield self.client.get('test', SCF, column=COLUMN, super_column=SCOLUMN) self.assert_(res.column.value == 'superval') res = yield self.client.get('test2', SCF, column=COLUMN, super_column=SCOLUMN) self.assert_(res.column.value == 'superval2') @defer.inlineCallbacks def test_batch_insert_get_slice_and_count(self): yield self.client.batch_insert('test', CF, {COLUMN: 'test', COLUMN2: 'test2'}) yield self.client.batch_insert('test', SCF, {SCOLUMN: {COLUMN: 'test', COLUMN2: 'test2'}}) res = yield self.client.get_slice('test', CF, names=(COLUMN, COLUMN2)) self.assert_(res[0].column.value == 'test') self.assert_(res[1].column.value == 'test2') res = yield self.client.get_slice('test', SCF, names=(COLUMN, COLUMN2), super_column=SCOLUMN) self.assert_(res[0].column.value == 'test') self.assert_(res[1].column.value == 'test2') res = yield self.client.get_count('test', CF) self.assert_(res == 2) @defer.inlineCallbacks def test_batch_mutate_and_remove(self): yield self.client.batch_mutate({'test': {CF: {COLUMN: 'test', COLUMN2: 'test2'}, SCF: { SCOLUMN: { COLUMN: 'test', COLUMN2: 'test2'} } }, 'test2': {CF: {COLUMN: 'test', COLUMN2: 'test2'}, SCF: { SCOLUMN: { COLUMN: 'test', COLUMN2: 'test2'} } } }) res = yield self.client.get_slice('test', CF, names=(COLUMN, COLUMN2)) self.assert_(res[0].column.value == 'test') self.assert_(res[1].column.value == 'test2') res = yield self.client.get_slice('test2', CF, names=(COLUMN, COLUMN2)) self.assert_(res[0].column.value == 'test') self.assert_(res[1].column.value == 'test2') res = yield self.client.get_slice('test', SCF, names=(COLUMN, COLUMN2), super_column=SCOLUMN) self.assert_(res[0].column.value == 'test') self.assert_(res[1].column.value == 'test2') res = yield self.client.get_slice('test2', SCF, names=(COLUMN, COLUMN2), super_column=SCOLUMN) self.assert_(res[0].column.value == 'test') self.assert_(res[1].column.value == 'test2') yield self.client.batch_remove({CF: ['test', 'test2']}, names=['test', 'test2']) yield self.client.batch_remove({SCF: ['test', 'test2']}, names=['test', 'test2'], supercolumn=SCOLUMN) @defer.inlineCallbacks def test_batch_mutate_with_deletion(self): yield self.client.batch_mutate({'test': {CF: {COLUMN: 'test', COLUMN2: 'test2'}}}) res = yield self.client.get_slice('test', CF, names=(COLUMN, COLUMN2)) self.assert_(res[0].column.value == 'test') self.assert_(res[1].column.value == 'test2') yield self.client.batch_mutate({'test': {CF: {COLUMN: None, COLUMN2: 'test3'}}}) res = yield self.client.get_slice('test', CF, names=(COLUMN, COLUMN2)) self.assert_(len(res) == 1) self.assert_(res[0].column.value == 'test3') @defer.inlineCallbacks def test_multiget_slice_remove(self): yield self.client.insert('test', CF, 'testval', column=COLUMN) yield self.client.insert('test', CF, 'testval', column=COLUMN2) yield self.client.insert('test2', CF, 'testval2', column=COLUMN) res = yield self.client.multiget(['test', 'test2'], CF, column=COLUMN) self.assert_(res['test'].column.value == 'testval') self.assert_(res['test2'].column.value == 'testval2') res = yield self.client.multiget_slice(['test', 'test2'], CF) self.assert_(res['test'][0].column.value == 'testval') self.assert_(res['test'][1].column.value == 'testval') self.assert_(res['test2'][0].column.value == 'testval2') yield self.client.remove('test', CF, column=COLUMN) yield self.client.remove('test2', CF, column=COLUMN) res = yield self.client.multiget(['test', 'test2'], CF, column=COLUMN) self.assert_(res['test'].column == None) self.assert_(res['test2'].column == None) @defer.inlineCallbacks def test_range_slices(self): yield self.client.insert('test', CF, 'testval', column=COLUMN) yield self.client.insert('test', CF, 'testval', column=COLUMN2) yield self.client.insert('test2', CF, 'testval2', column=COLUMN) ks = yield self.client.get_range_slices(CF, start='', finish='') keys = [k.key for k in ks] for key in ['test', 'test2']: self.assert_(key in keys) @defer.inlineCallbacks def test_errback(self): yield self.client.remove('poiqwe', CF) try: yield self.client.get('poiqwe', CF, column='foo') except Exception, e: pass