def do_this(): for keyspace in keyspaces: factory = ManagedCassandraClientFactory(keyspace) reactor.connectTCP("localhost", 9160, factory) client = CassandraClient(factory) k = yield client.describe_keyspace(keyspace) for cf in [c.name for c in k.cf_defs]: n = 0 db_rows = yield client.get_range_slices(column_family=cf, count=10000000) with gzip.GzipFile("%s.%s.csv.gz" % (keyspace, cf), "w") as f: out = csv.DictWriter(f, fieldnames=["keyspace", "cf", "key", "col", "val"]) out.writeheader() for row in db_rows: for col in row.columns: out.writerow({"keyspace": keyspace, "cf": cf, "key": row.key.encode("string_escape"), "col": col.column.name, "val": col.column.value.encode("string_escape")}) n += 1 print "Successfully backed up %d rows from %s.%s" % (n, keyspace, cf) reactor.stop()
class CassandraClientTest(unittest.TestCase): @defer.inlineCallbacks def setUp(self): self.cmanager = ManagedCassandraClientFactory(keyspace='system') self.client = CassandraClient(self.cmanager) for i in xrange(CONNS): reactor.connectTCP(HOST, PORT, self.cmanager) yield self.cmanager.deferred remote_ver = yield self.client.describe_version() self.version = getAPIVersion(remote_ver) self.my_keyspace = KsDef( name=KEYSPACE, strategy_class='org.apache.cassandra.locator.SimpleStrategy', strategy_options={'replication_factor': '1'}, cf_defs=[ CfDef( keyspace=KEYSPACE, name=CF, column_type='Standard' ), CfDef( keyspace=KEYSPACE, name=SCF, column_type='Super' ), CfDef( keyspace=KEYSPACE, name=IDX_CF, column_type='Standard', comparator_type='org.apache.cassandra.db.marshal.UTF8Type', column_metadata=[ ColumnDef( name='col1', validation_class='org.apache.cassandra.db.marshal.UTF8Type', index_type=IndexType.KEYS, index_name='idxCol1') ], default_validation_class='org.apache.cassandra.db.marshal.BytesType' ), ] ) if self.version == CASSANDRA_08_VERSION: self.my_keyspace.cf_defs.extend([ CfDef( keyspace=KEYSPACE, name=COUNTER_CF, column_type='Standard', default_validation_class='org.apache.cassandra.db.marshal.CounterColumnType' ), CfDef( keyspace=KEYSPACE, name=SUPERCOUNTER_CF, column_type='Super', default_validation_class='org.apache.cassandra.db.marshal.CounterColumnType' ), ]) yield self.client.system_add_keyspace(self.my_keyspace) yield self.client.set_keyspace(KEYSPACE) @defer.inlineCallbacks def tearDown(self): yield self.client.system_drop_keyspace(self.my_keyspace.name) self.cmanager.shutdown() for c in reactor.getDelayedCalls(): c.cancel() reactor.removeAll() @defer.inlineCallbacks def test_insert_get(self): yield self.client.insert('test', CF, 'testval', column=COLUMN) yield self.client.insert('test2', CF, 'testval2', column=COLUMN) yield self.client.insert('test', SCF, 'superval', column=COLUMN, super_column=SCOLUMN) yield self.client.insert('test2', SCF, 'superval2', column=COLUMN, super_column=SCOLUMN) res = yield self.client.get('test', CF, column=COLUMN) self.assertEqual(res.column.value, 'testval') res = yield self.client.get('test2', CF, column=COLUMN) self.assertEqual(res.column.value, 'testval2') res = yield self.client.get('test', SCF, column=COLUMN, super_column=SCOLUMN) self.assertEqual(res.column.value, 'superval') res = yield self.client.get('test2', SCF, column=COLUMN, super_column=SCOLUMN) self.assertEqual(res.column.value, 'superval2') @defer.inlineCallbacks def test_batch_insert_get_slice_and_count(self): yield self.client.batch_insert('test', CF, {COLUMN: 'test', COLUMN2: 'test2'}) yield self.client.batch_insert('test', SCF, {SCOLUMN: {COLUMN: 'test', COLUMN2: 'test2'}}) res = yield self.client.get_slice('test', CF, names=(COLUMN, COLUMN2)) self.assertEqual(res[0].column.value, 'test') self.assertEqual(res[1].column.value, 'test2') res = yield self.client.get_slice('test', SCF, names=(COLUMN, COLUMN2), super_column=SCOLUMN) self.assertEqual(res[0].column.value, 'test') self.assertEqual(res[1].column.value, 'test2') res = yield self.client.get_count('test', CF) self.assertEqual(res, 2) @defer.inlineCallbacks def test_batch_mutate_and_remove(self): yield self.client.batch_mutate({'test': {CF: {COLUMN: 'test', COLUMN2: 'test2'}, SCF: { SCOLUMN: { COLUMN: 'test', COLUMN2: 'test2'} } }, 'test2': {CF: {COLUMN: 'test', COLUMN2: 'test2'}, SCF: { SCOLUMN: { COLUMN: 'test', COLUMN2: 'test2'} } } }) res = yield self.client.get_slice('test', CF, names=(COLUMN, COLUMN2)) self.assertEqual(res[0].column.value, 'test') self.assertEqual(res[1].column.value, 'test2') res = yield self.client.get_slice('test2', CF, names=(COLUMN, COLUMN2)) self.assertEqual(res[0].column.value, 'test') self.assertEqual(res[1].column.value, 'test2') res = yield self.client.get_slice('test', SCF, names=(COLUMN, COLUMN2), super_column=SCOLUMN) self.assertEqual(res[0].column.value, 'test') self.assertEqual(res[1].column.value, 'test2') res = yield self.client.get_slice('test2', SCF, names=(COLUMN, COLUMN2), super_column=SCOLUMN) self.assertEqual(res[0].column.value, 'test') self.assertEqual(res[1].column.value, 'test2') yield self.client.batch_remove({CF: ['test', 'test2']}, names=['test', 'test2']) yield self.client.batch_remove({SCF: ['test', 'test2']}, names=['test', 'test2'], supercolumn=SCOLUMN) @defer.inlineCallbacks def test_batch_mutate_with_deletion(self): yield self.client.batch_mutate({'test': {CF: {COLUMN: 'test', COLUMN2: 'test2'}}}) res = yield self.client.get_slice('test', CF, names=(COLUMN, COLUMN2)) self.assertEqual(res[0].column.value, 'test') self.assertEqual(res[1].column.value, 'test2') yield self.client.batch_mutate({'test': {CF: {COLUMN: None, COLUMN2: 'test3'}}}) res = yield self.client.get_slice('test', CF, names=(COLUMN, COLUMN2)) self.assertEqual(len(res), 1) self.assertEqual(res[0].column.value, 'test3') @defer.inlineCallbacks def test_multiget_slice_remove(self): yield self.client.insert('test', CF, 'testval', column=COLUMN) yield self.client.insert('test', CF, 'testval', column=COLUMN2) yield self.client.insert('test2', CF, 'testval2', column=COLUMN) res = yield self.client.multiget(['test', 'test2'], CF, column=COLUMN) self.assertEqual(res['test'][0].column.value, 'testval') self.assertEqual(res['test2'][0].column.value, 'testval2') res = yield self.client.multiget_slice(['test', 'test2'], CF) self.assertEqual(res['test'][0].column.value, 'testval') self.assertEqual(res['test'][1].column.value, 'testval') self.assertEqual(res['test2'][0].column.value, 'testval2') yield self.client.remove('test', CF, column=COLUMN) yield self.client.remove('test2', CF, column=COLUMN) res = yield self.client.multiget(['test', 'test2'], CF, column=COLUMN) self.assertEqual(len(res['test']), 0) self.assertEqual(len(res['test2']), 0) @defer.inlineCallbacks def test_range_slices(self): yield self.client.insert('test', CF, 'testval', column=COLUMN) yield self.client.insert('test', CF, 'testval', column=COLUMN2) yield self.client.insert('test2', CF, 'testval2', column=COLUMN) ks = yield self.client.get_range_slices(CF, start='', finish='') keys = [k.key for k in ks] for key in ['test', 'test2']: self.assertIn(key, keys) @defer.inlineCallbacks def test_indexed_slices(self): yield self.client.insert('test1', IDX_CF, 'one', column='col1') yield self.client.insert('test2', IDX_CF, 'two', column='col1') yield self.client.insert('test3', IDX_CF, 'three', column='col1') expressions = [IndexExpression('col1', IndexOperator.EQ, 'two')] res = yield self.client.get_indexed_slices(IDX_CF, expressions, start_key='') self.assertEquals(res[0].columns[0].column.value,'two') @defer.inlineCallbacks def test_counter_add(self): if self.version != CASSANDRA_08_VERSION: raise unittest.SkipTest('Counters are not supported in 0.7') # test standard column counter yield self.client.add('test', COUNTER_CF, 1, column='col') res = yield self.client.get('test', COUNTER_CF, column='col') self.assertEquals(res.counter_column.value, 1) yield self.client.add('test', COUNTER_CF, 1, column='col') res = yield self.client.get('test', COUNTER_CF, column='col') self.assertEquals(res.counter_column.value, 2) # test super column counters yield self.client.add('test', SUPERCOUNTER_CF, 1, column='col', super_column='scol') res = yield self.client.get('test', SUPERCOUNTER_CF, column='col', super_column='scol') self.assertEquals(res.counter_column.value, 1) yield self.client.add('test', SUPERCOUNTER_CF, 1, column='col', super_column='scol') res = yield self.client.get('test', SUPERCOUNTER_CF, column='col', super_column='scol') self.assertEquals(res.counter_column.value, 2) @defer.inlineCallbacks def test_counter_remove(self): if self.version != CASSANDRA_08_VERSION: raise unittest.SkipTest('Counters are not supported in 0.7') # test standard column counter yield self.client.add('test', COUNTER_CF, 1, column='col') res = yield self.client.get('test', COUNTER_CF, column='col') self.assertEquals(res.counter_column.value, 1) yield self.client.remove_counter('test', COUNTER_CF, column='col') yield self.assertFailure(self.client.get('test', COUNTER_CF, column='col'), NotFoundException) # test super column counters yield self.client.add('test', SUPERCOUNTER_CF, 1, column='col', super_column='scol') res = yield self.client.get('test', SUPERCOUNTER_CF, column='col', super_column='scol') self.assertEquals(res.counter_column.value, 1) yield self.client.remove_counter('test', SUPERCOUNTER_CF, column='col', super_column='scol') yield self.assertFailure(self.client.get('test', SUPERCOUNTER_CF, column='col', super_column='scol'), NotFoundException) @defer.inlineCallbacks def test_cql(self): if self.version != CASSANDRA_08_VERSION: raise unittest.SkipTest('CQL is not supported in 0.7') yield self.client.insert('test', CF, 'testval', column='col1') res = yield self.client.get('test', CF, column='col1') self.assertEquals(res.column.value, 'testval') query = 'SELECT * from %s where KEY = %s' % (CF, 'test'.encode('hex')) uncompressed_result = yield self.client.execute_cql_query(query, Compression.NONE) self.assertEquals(uncompressed_result.rows[0].columns[0].name, 'col1') self.assertEquals(uncompressed_result.rows[0].columns[0].value, 'testval') compressed_query = zlib.compress(query) compressed_result = yield self.client.execute_cql_query(compressed_query, Compression.GZIP) self.assertEquals(uncompressed_result, compressed_result) def sleep(self, secs): d = defer.Deferred() reactor.callLater(secs, d.callback, None) return d @defer.inlineCallbacks def test_ttls(self): yield self.client.insert('test_ttls', CF, 'testval', column=COLUMN, ttl=1) res = yield self.client.get('test_ttls', CF, column=COLUMN) self.assertEqual(res.column.value, 'testval') yield self.sleep(2) yield self.assertFailure(self.client.get('test_ttls', CF, column=COLUMN), NotFoundException) yield self.client.batch_insert('test_ttls', CF, {COLUMN:'testval'}, ttl=1) res = yield self.client.get('test_ttls', CF, column=COLUMN) self.assertEqual(res.column.value, 'testval') yield self.sleep(2) yield self.assertFailure(self.client.get('test_ttls', CF, column=COLUMN), NotFoundException) yield self.client.batch_mutate({'test_ttls': {CF: {COLUMN: 'testval'}}}, ttl=1) res = yield self.client.get('test_ttls', CF, column=COLUMN) self.assertEqual(res.column.value, 'testval') yield self.sleep(2) yield self.assertFailure(self.client.get('test_ttls', CF, column=COLUMN), NotFoundException) def compare_keyspaces(self, ks1, ks2): self.assertEqual(ks1.name, ks2.name) self.assertEqual(ks1.strategy_class, ks2.strategy_class) self.assertEqual(ks1.cf_defs, ks2.cf_defs) def get_rf(ksdef): rf = ksdef.replication_factor if ksdef.strategy_options and \ 'replication_factor' in ksdef.strategy_options: rf = int(ksdef.strategy_options['replication_factor']) return rf def strat_opts_no_rf(ksdef): if not ksdef.strategy_options: return {} opts = ksdef.strategy_options.copy() if 'replication_factor' in ksdef.strategy_options: del opts['replication_factor'] return opts self.assertEqual(get_rf(ks1), get_rf(ks2)) self.assertEqual(strat_opts_no_rf(ks1), strat_opts_no_rf(ks2)) @defer.inlineCallbacks def test_keyspace_manipulation(self): try: yield self.client.system_drop_keyspace(T_KEYSPACE) except InvalidRequestException: pass ksdef = KsDef(name=T_KEYSPACE, strategy_class='org.apache.cassandra.locator.SimpleStrategy', strategy_options={'replication_factor': '1'}, cf_defs=[]) yield self.client.system_add_keyspace(ksdef) ks2 = yield self.client.describe_keyspace(T_KEYSPACE) self.compare_keyspaces(ksdef, ks2) if DO_SYSTEM_RENAMING: newname = T_KEYSPACE + '2' yield self.client.system_rename_keyspace(T_KEYSPACE, newname) ks2 = yield self.client.describe_keyspace(newname) ksdef.name = newname self.compare_keyspaces(ksdef, ks2) yield self.client.system_drop_keyspace(ksdef.name) yield self.assertFailure(self.client.describe_keyspace(T_KEYSPACE), NotFoundException) if DO_SYSTEM_RENAMING: yield self.assertFailure(self.client.describe_keyspace(ksdef.name), NotFoundException) @defer.inlineCallbacks def test_column_family_manipulation(self): cfdef = CfDef(KEYSPACE, T_CF, column_type='Standard', comparator_type='org.apache.cassandra.db.marshal.BytesType', comment='foo', row_cache_size=0.0, key_cache_size=200000.0, read_repair_chance=1.0, column_metadata=[], gc_grace_seconds=86400, default_validation_class='org.apache.cassandra.db.marshal.BytesType', key_validation_class='org.apache.cassandra.db.marshal.BytesType', min_compaction_threshold=5, max_compaction_threshold=31, row_cache_save_period_in_seconds=0, key_cache_save_period_in_seconds=3600, memtable_flush_after_mins=60, memtable_throughput_in_mb=249, memtable_operations_in_millions=1.1671875, replicate_on_write=False, merge_shards_chance=0.10000000000000001, row_cache_provider=None, key_alias=None, ) post_07_fields = ['replicate_on_write', 'merge_shards_chance', 'key_validation_class', 'row_cache_provider', 'key_alias'] yield self.client.system_add_column_family(cfdef) ksdef = yield self.client.describe_keyspace(KEYSPACE) cfdef2 = [c for c in ksdef.cf_defs if c.name == T_CF][0] for field in post_07_fields: # Most of these are ignored in 0.7, so we can't reliably compare them setattr(cfdef, field, None) setattr(cfdef2, field, None) # we don't know the id ahead of time. copy the new one so the equality # comparison won't fail cfdef.id = cfdef2.id self.assertEqual(cfdef, cfdef2) if DO_SYSTEM_RENAMING: newname = T_CF + '2' yield self.client.system_rename_column_family(T_CF, newname) ksdef = yield self.client.describe_keyspace(KEYSPACE) cfdef2 = [c for c in ksdef.cf_defs if c.name == newname][0] self.assertNotIn(T_CF, [c.name for c in ksdef.cf_defs]) cfdef.name = newname self.assertEqual(cfdef, cfdef2) yield self.client.system_drop_column_family(cfdef.name) ksdef = yield self.client.describe_keyspace(KEYSPACE) self.assertNotIn(cfdef.name, [c.name for c in ksdef.cf_defs]) @defer.inlineCallbacks def test_describes(self): name = yield self.client.describe_cluster_name() self.assertIsInstance(name, str) self.assertNotEqual(name, '') partitioner = yield self.client.describe_partitioner() self.assert_(partitioner.startswith('org.apache.cassandra.'), msg='partitioner is %r' % partitioner) snitch = yield self.client.describe_snitch() self.assert_(snitch.startswith('org.apache.cassandra.'), msg='snitch is %r' % snitch) version = yield self.client.describe_version() self.assertIsInstance(version, str) self.assertIn('.', version) schemavers = yield self.client.describe_schema_versions() self.assertIsInstance(schemavers, dict) self.assertNotEqual(schemavers, {}) ring = yield self.client.describe_ring(KEYSPACE) self.assertIsInstance(ring, list) self.assertNotEqual(ring, []) for r in ring: self.assertIsInstance(r.start_token, str) self.assertIsInstance(r.end_token, str) self.assertIsInstance(r.endpoints, list) self.assertNotEqual(r.endpoints, []) for ep in r.endpoints: self.assertIsInstance(ep, str) @defer.inlineCallbacks def test_errback(self): yield self.client.remove('poiqwe', CF) try: yield self.client.get('poiqwe', CF, column='foo') except Exception, e: pass
class CassandraClientTest(unittest.TestCase): @defer.inlineCallbacks def setUp(self): self.cmanager = ManagedCassandraClientFactory(keyspace='system') self.client = CassandraClient(self.cmanager) for i in xrange(CONNS): reactor.connectTCP(HOST, PORT, self.cmanager) yield self.cmanager.deferred remote_ver = yield self.client.describe_version() self.version = thrift_api_ver_to_cassandra_ver(remote_ver) self.my_keyspace = KsDef( name=KEYSPACE, strategy_class='org.apache.cassandra.locator.SimpleStrategy', strategy_options={'replication_factor': '1'}, cf_defs=[ CfDef(keyspace=KEYSPACE, name=CF, column_type='Standard'), CfDef(keyspace=KEYSPACE, name=SCF, column_type='Super'), CfDef( keyspace=KEYSPACE, name=IDX_CF, column_type='Standard', comparator_type='org.apache.cassandra.db.marshal.UTF8Type', column_metadata=[ ColumnDef(name='col1', validation_class= 'org.apache.cassandra.db.marshal.UTF8Type', index_type=IndexType.KEYS, index_name='idxCol1') ], default_validation_class= 'org.apache.cassandra.db.marshal.BytesType'), ]) if self.version == CASSANDRA_08_VERSION: self.my_keyspace.cf_defs.extend([ CfDef(keyspace=KEYSPACE, name=COUNTER_CF, column_type='Standard', default_validation_class= 'org.apache.cassandra.db.marshal.CounterColumnType'), CfDef(keyspace=KEYSPACE, name=SUPERCOUNTER_CF, column_type='Super', default_validation_class= 'org.apache.cassandra.db.marshal.CounterColumnType'), ]) yield self.client.system_add_keyspace(self.my_keyspace) yield self.client.set_keyspace(KEYSPACE) @defer.inlineCallbacks def tearDown(self): yield self.client.system_drop_keyspace(self.my_keyspace.name) self.cmanager.shutdown() for c in reactor.getDelayedCalls(): c.cancel() reactor.removeAll() @defer.inlineCallbacks def test_insert_get(self): yield self.client.insert('test', CF, 'testval', column=COLUMN) yield self.client.insert('test2', CF, 'testval2', column=COLUMN) yield self.client.insert('test', SCF, 'superval', column=COLUMN, super_column=SCOLUMN) yield self.client.insert('test2', SCF, 'superval2', column=COLUMN, super_column=SCOLUMN) res = yield self.client.get('test', CF, column=COLUMN) self.assertEqual(res.column.value, 'testval') res = yield self.client.get('test2', CF, column=COLUMN) self.assertEqual(res.column.value, 'testval2') res = yield self.client.get('test', SCF, column=COLUMN, super_column=SCOLUMN) self.assertEqual(res.column.value, 'superval') res = yield self.client.get('test2', SCF, column=COLUMN, super_column=SCOLUMN) self.assertEqual(res.column.value, 'superval2') @defer.inlineCallbacks def test_batch_insert_get_slice_and_count(self): yield self.client.batch_insert('test', CF, { COLUMN: 'test', COLUMN2: 'test2' }) yield self.client.batch_insert( 'test', SCF, {SCOLUMN: { COLUMN: 'test', COLUMN2: 'test2' }}) res = yield self.client.get_slice('test', CF, names=(COLUMN, COLUMN2)) self.assertEqual(res[0].column.value, 'test') self.assertEqual(res[1].column.value, 'test2') res = yield self.client.get_slice('test', SCF, names=(COLUMN, COLUMN2), super_column=SCOLUMN) self.assertEqual(res[0].column.value, 'test') self.assertEqual(res[1].column.value, 'test2') res = yield self.client.get_count('test', CF) self.assertEqual(res, 2) @defer.inlineCallbacks def test_batch_mutate_and_remove(self): yield self.client.batch_mutate({ 'test': { CF: { COLUMN: 'test', COLUMN2: 'test2' }, SCF: { SCOLUMN: { COLUMN: 'test', COLUMN2: 'test2' } } }, 'test2': { CF: { COLUMN: 'test', COLUMN2: 'test2' }, SCF: { SCOLUMN: { COLUMN: 'test', COLUMN2: 'test2' } } } }) res = yield self.client.get_slice('test', CF, names=(COLUMN, COLUMN2)) self.assertEqual(res[0].column.value, 'test') self.assertEqual(res[1].column.value, 'test2') res = yield self.client.get_slice('test2', CF, names=(COLUMN, COLUMN2)) self.assertEqual(res[0].column.value, 'test') self.assertEqual(res[1].column.value, 'test2') res = yield self.client.get_slice('test', SCF, names=(COLUMN, COLUMN2), super_column=SCOLUMN) self.assertEqual(res[0].column.value, 'test') self.assertEqual(res[1].column.value, 'test2') res = yield self.client.get_slice('test2', SCF, names=(COLUMN, COLUMN2), super_column=SCOLUMN) self.assertEqual(res[0].column.value, 'test') self.assertEqual(res[1].column.value, 'test2') yield self.client.batch_remove({CF: ['test', 'test2']}, names=['test', 'test2']) yield self.client.batch_remove({SCF: ['test', 'test2']}, names=['test', 'test2'], supercolumn=SCOLUMN) @defer.inlineCallbacks def test_batch_mutate_with_deletion(self): yield self.client.batch_mutate( {'test': { CF: { COLUMN: 'test', COLUMN2: 'test2' } }}) res = yield self.client.get_slice('test', CF, names=(COLUMN, COLUMN2)) self.assertEqual(res[0].column.value, 'test') self.assertEqual(res[1].column.value, 'test2') yield self.client.batch_mutate( {'test': { CF: { COLUMN: None, COLUMN2: 'test3' } }}) res = yield self.client.get_slice('test', CF, names=(COLUMN, COLUMN2)) self.assertEqual(len(res), 1) self.assertEqual(res[0].column.value, 'test3') @defer.inlineCallbacks def test_multiget_slice_remove(self): yield self.client.insert('test', CF, 'testval', column=COLUMN) yield self.client.insert('test', CF, 'testval', column=COLUMN2) yield self.client.insert('test2', CF, 'testval2', column=COLUMN) res = yield self.client.multiget(['test', 'test2'], CF, column=COLUMN) self.assertEqual(res['test'][0].column.value, 'testval') self.assertEqual(res['test2'][0].column.value, 'testval2') res = yield self.client.multiget_slice(['test', 'test2'], CF) self.assertEqual(res['test'][0].column.value, 'testval') self.assertEqual(res['test'][1].column.value, 'testval') self.assertEqual(res['test2'][0].column.value, 'testval2') yield self.client.remove('test', CF, column=COLUMN) yield self.client.remove('test2', CF, column=COLUMN) res = yield self.client.multiget(['test', 'test2'], CF, column=COLUMN) self.assertEqual(len(res['test']), 0) self.assertEqual(len(res['test2']), 0) @defer.inlineCallbacks def test_range_slices(self): yield self.client.insert('test', CF, 'testval', column=COLUMN) yield self.client.insert('test', CF, 'testval', column=COLUMN2) yield self.client.insert('test2', CF, 'testval2', column=COLUMN) ks = yield self.client.get_range_slices(CF, start='', finish='') keys = [k.key for k in ks] for key in ['test', 'test2']: self.assertIn(key, keys) @defer.inlineCallbacks def test_indexed_slices(self): yield self.client.insert('test1', IDX_CF, 'one', column='col1') yield self.client.insert('test2', IDX_CF, 'two', column='col1') yield self.client.insert('test3', IDX_CF, 'three', column='col1') expressions = [IndexExpression('col1', IndexOperator.EQ, 'two')] res = yield self.client.get_indexed_slices(IDX_CF, expressions, start_key='') self.assertEquals(res[0].columns[0].column.value, 'two') @defer.inlineCallbacks def test_counter_add(self): if self.version != CASSANDRA_08_VERSION: raise unittest.SkipTest('Counters are not supported in 0.7') # test standard column counter yield self.client.add('test', COUNTER_CF, 1, column='col') res = yield self.client.get('test', COUNTER_CF, column='col') self.assertEquals(res.counter_column.value, 1) yield self.client.add('test', COUNTER_CF, 1, column='col') res = yield self.client.get('test', COUNTER_CF, column='col') self.assertEquals(res.counter_column.value, 2) # test super column counters yield self.client.add('test', SUPERCOUNTER_CF, 1, column='col', super_column='scol') res = yield self.client.get('test', SUPERCOUNTER_CF, column='col', super_column='scol') self.assertEquals(res.counter_column.value, 1) yield self.client.add('test', SUPERCOUNTER_CF, 1, column='col', super_column='scol') res = yield self.client.get('test', SUPERCOUNTER_CF, column='col', super_column='scol') self.assertEquals(res.counter_column.value, 2) @defer.inlineCallbacks def test_counter_remove(self): if self.version != CASSANDRA_08_VERSION: raise unittest.SkipTest('Counters are not supported in 0.7') # test standard column counter yield self.client.add('test', COUNTER_CF, 1, column='col') res = yield self.client.get('test', COUNTER_CF, column='col') self.assertEquals(res.counter_column.value, 1) yield self.client.remove_counter('test', COUNTER_CF, column='col') yield self.assertFailure( self.client.get('test', COUNTER_CF, column='col'), NotFoundException) # test super column counters yield self.client.add('test', SUPERCOUNTER_CF, 1, column='col', super_column='scol') res = yield self.client.get('test', SUPERCOUNTER_CF, column='col', super_column='scol') self.assertEquals(res.counter_column.value, 1) yield self.client.remove_counter('test', SUPERCOUNTER_CF, column='col', super_column='scol') yield self.assertFailure( self.client.get('test', SUPERCOUNTER_CF, column='col', super_column='scol'), NotFoundException) def sleep(self, secs): d = defer.Deferred() reactor.callLater(secs, d.callback, None) return d @defer.inlineCallbacks def test_ttls(self): yield self.client.insert('test_ttls', CF, 'testval', column=COLUMN, ttl=1) res = yield self.client.get('test_ttls', CF, column=COLUMN) self.assertEqual(res.column.value, 'testval') yield self.sleep(2) yield self.assertFailure( self.client.get('test_ttls', CF, column=COLUMN), NotFoundException) yield self.client.batch_insert('test_ttls', CF, {COLUMN: 'testval'}, ttl=1) res = yield self.client.get('test_ttls', CF, column=COLUMN) self.assertEqual(res.column.value, 'testval') yield self.sleep(2) yield self.assertFailure( self.client.get('test_ttls', CF, column=COLUMN), NotFoundException) yield self.client.batch_mutate( {'test_ttls': { CF: { COLUMN: 'testval' } }}, ttl=1) res = yield self.client.get('test_ttls', CF, column=COLUMN) self.assertEqual(res.column.value, 'testval') yield self.sleep(2) yield self.assertFailure( self.client.get('test_ttls', CF, column=COLUMN), NotFoundException) def compare_keyspaces(self, ks1, ks2): self.assertEqual(ks1.name, ks2.name) self.assertEqual(ks1.strategy_class, ks2.strategy_class) self.assertEqual(ks1.cf_defs, ks2.cf_defs) def get_rf(ksdef): rf = ksdef.replication_factor if ksdef.strategy_options and \ 'replication_factor' in ksdef.strategy_options: rf = int(ksdef.strategy_options['replication_factor']) return rf def strat_opts_no_rf(ksdef): if not ksdef.strategy_options: return {} opts = ksdef.strategy_options.copy() if 'replication_factor' in ksdef.strategy_options: del opts['replication_factor'] return opts self.assertEqual(get_rf(ks1), get_rf(ks2)) self.assertEqual(strat_opts_no_rf(ks1), strat_opts_no_rf(ks2)) @defer.inlineCallbacks def test_keyspace_manipulation(self): try: yield self.client.system_drop_keyspace(T_KEYSPACE) except InvalidRequestException: pass ksdef = KsDef( name=T_KEYSPACE, strategy_class='org.apache.cassandra.locator.SimpleStrategy', strategy_options={'replication_factor': '1'}, cf_defs=[]) yield self.client.system_add_keyspace(ksdef) ks2 = yield self.client.describe_keyspace(T_KEYSPACE) self.compare_keyspaces(ksdef, ks2) if DO_SYSTEM_RENAMING: newname = T_KEYSPACE + '2' yield self.client.system_rename_keyspace(T_KEYSPACE, newname) ks2 = yield self.client.describe_keyspace(newname) ksdef.name = newname self.compare_keyspaces(ksdef, ks2) yield self.client.system_drop_keyspace(ksdef.name) yield self.assertFailure(self.client.describe_keyspace(T_KEYSPACE), NotFoundException) if DO_SYSTEM_RENAMING: yield self.assertFailure(self.client.describe_keyspace(ksdef.name), NotFoundException) @defer.inlineCallbacks def test_column_family_manipulation(self): cfdef = CfDef( KEYSPACE, T_CF, column_type='Standard', comparator_type='org.apache.cassandra.db.marshal.BytesType', comment='foo', row_cache_size=0.0, key_cache_size=200000.0, read_repair_chance=1.0, column_metadata=[], gc_grace_seconds=86400, default_validation_class= 'org.apache.cassandra.db.marshal.BytesType', key_validation_class='org.apache.cassandra.db.marshal.BytesType', min_compaction_threshold=5, max_compaction_threshold=31, row_cache_save_period_in_seconds=0, key_cache_save_period_in_seconds=3600, memtable_flush_after_mins=60, memtable_throughput_in_mb=249, memtable_operations_in_millions=1.1671875, replicate_on_write=False, merge_shards_chance=0.10000000000000001, row_cache_provider=None, key_alias=None, ) post_07_fields = [ 'replicate_on_write', 'merge_shards_chance', 'key_validation_class', 'row_cache_provider', 'key_alias' ] post_08_fields = [ 'memtable_throughput_in_mb', 'memtable_flush_after_mins', 'memtable_operations_in_millions' ] yield self.client.system_add_column_family(cfdef) ksdef = yield self.client.describe_keyspace(KEYSPACE) cfdef2 = [c for c in ksdef.cf_defs if c.name == T_CF][0] for field in post_07_fields: # Most of these are ignored in 0.7, so we can't reliably compare them setattr(cfdef, field, None) setattr(cfdef2, field, None) for field in post_08_fields: # These fields change from 0.8 to 1.0 setattr(cfdef, field, None) setattr(cfdef2, field, None) # we don't know the id ahead of time. copy the new one so the equality # comparison won't fail cfdef.id = cfdef2.id self.assertEqual(cfdef, cfdef2) if DO_SYSTEM_RENAMING: newname = T_CF + '2' yield self.client.system_rename_column_family(T_CF, newname) ksdef = yield self.client.describe_keyspace(KEYSPACE) cfdef2 = [c for c in ksdef.cf_defs if c.name == newname][0] self.assertNotIn(T_CF, [c.name for c in ksdef.cf_defs]) cfdef.name = newname self.assertEqual(cfdef, cfdef2) yield self.client.system_drop_column_family(cfdef.name) ksdef = yield self.client.describe_keyspace(KEYSPACE) self.assertNotIn(cfdef.name, [c.name for c in ksdef.cf_defs]) @defer.inlineCallbacks def test_describes(self): name = yield self.client.describe_cluster_name() self.assertIsInstance(name, str) self.assertNotEqual(name, '') partitioner = yield self.client.describe_partitioner() self.assert_(partitioner.startswith('org.apache.cassandra.'), msg='partitioner is %r' % partitioner) snitch = yield self.client.describe_snitch() self.assert_(snitch.startswith('org.apache.cassandra.'), msg='snitch is %r' % snitch) version = yield self.client.describe_version() self.assertIsInstance(version, str) self.assertIn('.', version) schemavers = yield self.client.describe_schema_versions() self.assertIsInstance(schemavers, dict) self.assertNotEqual(schemavers, {}) ring = yield self.client.describe_ring(KEYSPACE) self.assertIsInstance(ring, list) self.assertNotEqual(ring, []) for r in ring: self.assertIsInstance(r.start_token, str) self.assertIsInstance(r.end_token, str) self.assertIsInstance(r.endpoints, list) self.assertNotEqual(r.endpoints, []) for ep in r.endpoints: self.assertIsInstance(ep, str) @defer.inlineCallbacks def test_errback(self): yield self.client.remove('poiqwe', CF) try: yield self.client.get('poiqwe', CF, column='foo') except Exception, e: pass
class CassandraProvisionerStore(object): """ Provides high level provisioner storage operations for Cassandra """ # default size of paged fetches _PAGE_SIZE = 100 LAUNCH_CF_NAME = "ProvisionerLaunches" NODE_CF_NAME = "ProvisionerNodes" @classmethod def get_column_families(cls, keyspace=None, prefix=''): """Builds a list of column families needed by this store. @param keyspace Name of keyspace. If None, it must be added manually. @param prefix Optional prefix for cf names. Useful for testing. @retval list of CfDef objects """ launch_cf_name = prefix + cls.LAUNCH_CF_NAME node_cf_name = prefix + cls.NODE_CF_NAME return [ CfDef(keyspace, launch_cf_name, comparator_type='org.apache.cassandra.db.marshal.UTF8Type'), CfDef(keyspace, node_cf_name, comparator_type='org.apache.cassandra.db.marshal.UTF8Type') ] def __init__(self, host, port, username, password, keyspace, prefix=''): self._launch_column_family = prefix + self.LAUNCH_CF_NAME self._node_column_family = prefix + self.NODE_CF_NAME authz = {'username': username, 'password': password} self._manager = ManagedCassandraClientFactory(credentials=authz, check_api_version=True, keyspace=keyspace) self.client = CassandraClient(self._manager) self._host = host self._port = port self._connector = None def connect(self): self._connector = reactor.connectTCP(self._host, self._port, self._manager) def disconnect(self): self._manager.shutdown() if self._connector: self._connector.disconnect() self._connector = None @timeout(CASSANDRA_TIMEOUT) def put_launch(self, launch): """ @brief Stores a single launch record @param launch Launch record to store @retval Deferred for success """ launch_id = launch['launch_id'] state = launch['state'] value = json.dumps(launch) return self.client.insert(launch_id, self._launch_column_family, value, column=state) @timeout(CASSANDRA_TIMEOUT) @defer.inlineCallbacks def put_nodes(self, nodes): """ @brief Stores a set of node records @param nodes Iterable of node records @retval Deferred for success """ # could be more efficient with a batch_mutate for node in nodes: yield self.put_node(node) @timeout(CASSANDRA_TIMEOUT) def put_node(self, node): """ @brief Stores a node record @param node Node record @retval Deferred for success """ node_id = node['node_id'] state = node['state'] value = json.dumps(node) return self.client.insert(node_id, self._node_column_family, value, column=state) @timeout(CASSANDRA_TIMEOUT) def get_launch(self, launch_id, count=1): """ @brief Retrieves a launch record by id @param launch_id Id of launch record to retrieve @param count Number of launch state records to retrieve @retval Deferred record(s), or None. A list of records if count > 1 """ return self._get_record(launch_id, self._launch_column_family, count) @timeout(CASSANDRA_TIMEOUT) def get_launches(self, state=None, min_state=None, max_state=None): """ @brief Retrieves the latest record for all launches within a state range @param state Only retrieve nodes in this state. @param min_state Inclusive start bound @param max_state Inclusive end bound @retval Deferred list of launch records """ return self._get_records(self._launch_column_family, state=state, min_state=min_state, max_state=max_state) @timeout(CASSANDRA_TIMEOUT) def get_node(self, node_id, count=1): """ @brief Retrieves a launch record by id @param node_id Id of node record to retrieve @param count Number of node state records to retrieve @retval Deferred record(s), or None. A list of records if count > 1 """ return self._get_record(node_id, self._node_column_family, count) @timeout(CASSANDRA_TIMEOUT) def get_nodes(self, state=None, min_state=None, max_state=None): """ @brief Retrieves all launch record within a state range @param state Only retrieve nodes in this state. @param min_state Inclusive start bound. @param max_state Inclusive end bound @retval Deferred list of launch records """ return self._get_records(self._node_column_family, state=state, min_state=min_state, max_state=max_state) @defer.inlineCallbacks def _get_record(self, key, column_family, count): slice = yield self.client.get_slice(key, column_family, reverse=True, count=count) # we're probably only interested in the last record, in sorted order. # This is the latest state the object has recorded. records = [json.loads(column.column.value) for column in slice] if count == 1: if records: ret = records[0] else: ret = None else: ret = records defer.returnValue(ret) @defer.inlineCallbacks def _get_records(self, column_family, state=None, min_state=None, max_state=None, reverse=True): # overrides range arguments if state: min_state = max_state = state start = '' end = min_state or '' if not reverse: start, end = end, start # this is tricky. We are only concerned with the latest state record # (by sort order not necessarily time). So when we look for records # within a state range, we effectively must pull down the latest state # for each record, and filter them locally. This is slightly improved # when a first_state (or last when reverse=False) is specified as the # server can skip any records not >= that state. records = [] done = False start_key = '' iterations = 0 while not done: slices = yield self.client.get_range_slices(column_family, column_start=start, column_finish=end, reverse=reverse, column_count=1, start=start_key, count=self._PAGE_SIZE) skipped_one = False for slice in slices: if not skipped_one and iterations: # if this not the first batch, skip the first element as it # will be a dupe. skipped_one = True continue if not slice.columns: # rows without matching columns will still be returned continue record = json.loads(slice.columns[0].column.value) if not max_state or record['state'] <= max_state: if not min_state or record['state'] >= min_state: records.append(record) # page through results. by default only 100 are returned at a time if len(slices) == self._PAGE_SIZE: start_key = slices[-1].key else: done = True iterations += 1 defer.returnValue(records) def on_deactivate(self, *args, **kwargs): self._manager.shutdown() log.info('on_deactivate: Lose Connection TCP') def on_terminate(self, *args, **kwargs): self._manager.shutdown() log.info('on_terminate: Lose Connection TCP')
class CassandraClientTest(unittest.TestCase): @defer.inlineCallbacks def setUp(self): self.cmanager = ManagedCassandraClientFactory(keyspace='system') self.client = CassandraClient(self.cmanager) for i in xrange(CONNS): reactor.connectTCP(HOST, PORT, self.cmanager) yield self.cmanager.deferred self.my_keyspace = KsDef( name=KEYSPACE, strategy_class='org.apache.cassandra.locator.SimpleStrategy', replication_factor=1, cf_defs=[ CfDef( keyspace=KEYSPACE, name=CF, column_type='Standard' ), CfDef( keyspace=KEYSPACE, name=SCF, column_type='Super' ), CfDef( keyspace=KEYSPACE, name=IDX_CF, column_type='Standard', comparator_type='org.apache.cassandra.db.marshal.UTF8Type', column_metadata=[ ColumnDef( name='col1', validation_class='org.apache.cassandra.db.marshal.UTF8Type', index_type=IndexType.KEYS, index_name='idxCol1') ], default_validation_class='org.apache.cassandra.db.marshal.BytesType') ] ) yield self.client.system_add_keyspace(self.my_keyspace) yield self.client.set_keyspace(KEYSPACE) @defer.inlineCallbacks def tearDown(self): yield self.client.system_drop_keyspace(self.my_keyspace.name) self.cmanager.shutdown() for c in reactor.getDelayedCalls(): c.cancel() reactor.removeAll() @defer.inlineCallbacks def test_insert_get(self): yield self.client.insert('test', CF, 'testval', column=COLUMN) yield self.client.insert('test2', CF, 'testval2', column=COLUMN) yield self.client.insert('test', SCF, 'superval', column=COLUMN, super_column=SCOLUMN) yield self.client.insert('test2', SCF, 'superval2', column=COLUMN, super_column=SCOLUMN) res = yield self.client.get('test', CF, column=COLUMN) self.assertEqual(res.column.value, 'testval') res = yield self.client.get('test2', CF, column=COLUMN) self.assertEqual(res.column.value, 'testval2') res = yield self.client.get('test', SCF, column=COLUMN, super_column=SCOLUMN) self.assertEqual(res.column.value, 'superval') res = yield self.client.get('test2', SCF, column=COLUMN, super_column=SCOLUMN) self.assertEqual(res.column.value, 'superval2') @defer.inlineCallbacks def test_batch_insert_get_slice_and_count(self): yield self.client.batch_insert('test', CF, {COLUMN: 'test', COLUMN2: 'test2'}) yield self.client.batch_insert('test', SCF, {SCOLUMN: {COLUMN: 'test', COLUMN2: 'test2'}}) res = yield self.client.get_slice('test', CF, names=(COLUMN, COLUMN2)) self.assertEqual(res[0].column.value, 'test') self.assertEqual(res[1].column.value, 'test2') res = yield self.client.get_slice('test', SCF, names=(COLUMN, COLUMN2), super_column=SCOLUMN) self.assertEqual(res[0].column.value, 'test') self.assertEqual(res[1].column.value, 'test2') res = yield self.client.get_count('test', CF) self.assertEqual(res, 2) @defer.inlineCallbacks def test_batch_mutate_and_remove(self): yield self.client.batch_mutate({'test': {CF: {COLUMN: 'test', COLUMN2: 'test2'}, SCF: { SCOLUMN: { COLUMN: 'test', COLUMN2: 'test2'} } }, 'test2': {CF: {COLUMN: 'test', COLUMN2: 'test2'}, SCF: { SCOLUMN: { COLUMN: 'test', COLUMN2: 'test2'} } } }) res = yield self.client.get_slice('test', CF, names=(COLUMN, COLUMN2)) self.assertEqual(res[0].column.value, 'test') self.assertEqual(res[1].column.value, 'test2') res = yield self.client.get_slice('test2', CF, names=(COLUMN, COLUMN2)) self.assertEqual(res[0].column.value, 'test') self.assertEqual(res[1].column.value, 'test2') res = yield self.client.get_slice('test', SCF, names=(COLUMN, COLUMN2), super_column=SCOLUMN) self.assertEqual(res[0].column.value, 'test') self.assertEqual(res[1].column.value, 'test2') res = yield self.client.get_slice('test2', SCF, names=(COLUMN, COLUMN2), super_column=SCOLUMN) self.assertEqual(res[0].column.value, 'test') self.assertEqual(res[1].column.value, 'test2') yield self.client.batch_remove({CF: ['test', 'test2']}, names=['test', 'test2']) yield self.client.batch_remove({SCF: ['test', 'test2']}, names=['test', 'test2'], supercolumn=SCOLUMN) @defer.inlineCallbacks def test_batch_mutate_with_deletion(self): yield self.client.batch_mutate({'test': {CF: {COLUMN: 'test', COLUMN2: 'test2'}}}) res = yield self.client.get_slice('test', CF, names=(COLUMN, COLUMN2)) self.assertEqual(res[0].column.value, 'test') self.assertEqual(res[1].column.value, 'test2') yield self.client.batch_mutate({'test': {CF: {COLUMN: None, COLUMN2: 'test3'}}}) res = yield self.client.get_slice('test', CF, names=(COLUMN, COLUMN2)) self.assertEqual(len(res), 1) self.assertEqual(res[0].column.value, 'test3') @defer.inlineCallbacks def test_multiget_slice_remove(self): yield self.client.insert('test', CF, 'testval', column=COLUMN) yield self.client.insert('test', CF, 'testval', column=COLUMN2) yield self.client.insert('test2', CF, 'testval2', column=COLUMN) res = yield self.client.multiget(['test', 'test2'], CF, column=COLUMN) self.assertEqual(res['test'][0].column.value, 'testval') self.assertEqual(res['test2'][0].column.value, 'testval2') res = yield self.client.multiget_slice(['test', 'test2'], CF) self.assertEqual(res['test'][0].column.value, 'testval') self.assertEqual(res['test'][1].column.value, 'testval') self.assertEqual(res['test2'][0].column.value, 'testval2') yield self.client.remove('test', CF, column=COLUMN) yield self.client.remove('test2', CF, column=COLUMN) res = yield self.client.multiget(['test', 'test2'], CF, column=COLUMN) self.assertEqual(len(res['test']), 0) self.assertEqual(len(res['test2']), 0) @defer.inlineCallbacks def test_range_slices(self): yield self.client.insert('test', CF, 'testval', column=COLUMN) yield self.client.insert('test', CF, 'testval', column=COLUMN2) yield self.client.insert('test2', CF, 'testval2', column=COLUMN) ks = yield self.client.get_range_slices(CF, start='', finish='') keys = [k.key for k in ks] for key in ['test', 'test2']: self.assertIn(key, keys) @defer.inlineCallbacks def test_indexed_slices(self): yield self.client.insert('test1', IDX_CF, 'one', column='col1') yield self.client.insert('test2', IDX_CF, 'two', column='col1') yield self.client.insert('test3', IDX_CF, 'three', column='col1') expressions = [IndexExpression('col1', IndexOperator.EQ, 'two')] res = yield self.client.get_indexed_slices(IDX_CF, expressions, start_key='') self.assertEquals(res[0].columns[0].column.value,'two') def sleep(self, secs): d = defer.Deferred() reactor.callLater(secs, d.callback, None) return d @defer.inlineCallbacks def test_ttls(self): yield self.client.insert('test_ttls', CF, 'testval', column=COLUMN, ttl=1) res = yield self.client.get('test_ttls', CF, column=COLUMN) self.assertEqual(res.column.value, 'testval') yield self.sleep(2) yield self.assertFailure(self.client.get('test_ttls', CF, column=COLUMN), NotFoundException) yield self.client.batch_insert('test_ttls', CF, {COLUMN:'testval'}, ttl=1) res = yield self.client.get('test_ttls', CF, column=COLUMN) self.assertEqual(res.column.value, 'testval') yield self.sleep(2) yield self.assertFailure(self.client.get('test_ttls', CF, column=COLUMN), NotFoundException) yield self.client.batch_mutate({'test_ttls': {CF: {COLUMN: 'testval'}}}, ttl=1) res = yield self.client.get('test_ttls', CF, column=COLUMN) self.assertEqual(res.column.value, 'testval') yield self.sleep(2) yield self.assertFailure(self.client.get('test_ttls', CF, column=COLUMN), NotFoundException) @defer.inlineCallbacks def test_keyspace_manipulation(self): ksdef = KsDef(name=T_KEYSPACE, strategy_class='org.apache.cassandra.locator.SimpleStrategy', replication_factor=1, cf_defs=[]) yield self.client.system_add_keyspace(ksdef) ks2 = yield self.client.describe_keyspace(T_KEYSPACE) self.assertEqual(ksdef, ks2) if DO_SYSTEM_RENAMING: newname = T_KEYSPACE + '2' yield self.client.system_rename_keyspace(T_KEYSPACE, newname) ks2 = yield self.client.describe_keyspace(newname) ksdef.name = newname self.assertEqual(ksdef, ks2) yield self.client.system_drop_keyspace(ksdef.name) yield self.assertFailure(self.client.describe_keyspace(T_KEYSPACE), NotFoundException) if DO_SYSTEM_RENAMING: yield self.assertFailure(self.client.describe_keyspace(ksdef.name), NotFoundException) @defer.inlineCallbacks def test_column_family_manipulation(self): cfdef = CfDef(KEYSPACE, T_CF, column_type='Standard', comparator_type='org.apache.cassandra.db.marshal.BytesType', comment='foo', row_cache_size=0.0, key_cache_size=200000.0, read_repair_chance=1.0, column_metadata=[], gc_grace_seconds=86400, default_validation_class='org.apache.cassandra.db.marshal.BytesType', min_compaction_threshold=5, max_compaction_threshold=31, row_cache_save_period_in_seconds=0, key_cache_save_period_in_seconds=3600, memtable_flush_after_mins=60, memtable_throughput_in_mb=249, memtable_operations_in_millions=1.1671875, ) yield self.client.system_add_column_family(cfdef) ksdef = yield self.client.describe_keyspace(KEYSPACE) cfdef2 = [c for c in ksdef.cf_defs if c.name == T_CF][0] # we don't know the id ahead of time. copy the new one so the equality # comparison won't fail cfdef.id = cfdef2.id self.assertEqual(cfdef, cfdef2) if DO_SYSTEM_RENAMING: newname = T_CF + '2' yield self.client.system_rename_column_family(T_CF, newname) ksdef = yield self.client.describe_keyspace(KEYSPACE) cfdef2 = [c for c in ksdef.cf_defs if c.name == newname][0] self.assertNotIn(T_CF, [c.name for c in ksdef.cf_defs]) cfdef.name = newname self.assertEqual(cfdef, cfdef2) yield self.client.system_drop_column_family(cfdef.name) ksdef = yield self.client.describe_keyspace(KEYSPACE) self.assertNotIn(cfdef.name, [c.name for c in ksdef.cf_defs]) @defer.inlineCallbacks def test_describes(self): name = yield self.client.describe_cluster_name() self.assertIsInstance(name, str) self.assertNotEqual(name, '') partitioner = yield self.client.describe_partitioner() self.assert_(partitioner.startswith('org.apache.cassandra.'), msg='partitioner is %r' % partitioner) snitch = yield self.client.describe_snitch() self.assert_(snitch.startswith('org.apache.cassandra.'), msg='snitch is %r' % snitch) version = yield self.client.describe_version() self.assertIsInstance(version, str) self.assertIn('.', version) schemavers = yield self.client.describe_schema_versions() self.assertIsInstance(schemavers, dict) self.assertNotEqual(schemavers, {}) ring = yield self.client.describe_ring(KEYSPACE) self.assertIsInstance(ring, list) self.assertNotEqual(ring, []) for r in ring: self.assertIsInstance(r.start_token, str) self.assertIsInstance(r.end_token, str) self.assertIsInstance(r.endpoints, list) self.assertNotEqual(r.endpoints, []) for ep in r.endpoints: self.assertIsInstance(ep, str) @defer.inlineCallbacks def test_errback(self): yield self.client.remove('poiqwe', CF) try: yield self.client.get('poiqwe', CF, column='foo') except Exception, e: pass
class CassandraClientTest(unittest.TestCase): @defer.inlineCallbacks def setUp(self): self.cmanager = ManagedCassandraClientFactory(keyspace='system') self.client = CassandraClient(self.cmanager) for i in xrange(CONNS): reactor.connectTCP(HOST, PORT, self.cmanager) yield self.cmanager.deferred remote_ver = yield self.client.describe_version() self.version = tuple(map(int, remote_ver.split('.'))) self.my_keyspace = ttypes.KsDef( name=KEYSPACE, strategy_class='org.apache.cassandra.locator.SimpleStrategy', strategy_options={}, cf_defs=[ ttypes.CfDef( keyspace=KEYSPACE, name=CF, column_type='Standard' ), ttypes.CfDef( keyspace=KEYSPACE, name=SCF, column_type='Super' ), ttypes.CfDef( keyspace=KEYSPACE, name=IDX_CF, column_type='Standard', comparator_type='org.apache.cassandra.db.marshal.UTF8Type', column_metadata=[ ttypes.ColumnDef( name='col1', validation_class='org.apache.cassandra.db.marshal.UTF8Type', index_type=ttypes.IndexType.KEYS, index_name='idxCol1') ], default_validation_class='org.apache.cassandra.db.marshal.BytesType' ), ] ) if self.version <= KS_RF_ATTRIBUTE: self.my_keyspace.replication_factor = 1 else: self.my_keyspace.strategy_options['replication_factor'] = '1' if self.version >= COUNTERS_SUPPORTED_API: self.my_keyspace.cf_defs.extend([ ttypes.CfDef( keyspace=KEYSPACE, name=COUNTER_CF, column_type='Standard', default_validation_class='org.apache.cassandra.db.marshal.CounterColumnType' ), ttypes.CfDef( keyspace=KEYSPACE, name=SUPERCOUNTER_CF, column_type='Super', default_validation_class='org.apache.cassandra.db.marshal.CounterColumnType' ), ]) yield self.client.system_add_keyspace(self.my_keyspace) yield self.client.set_keyspace(KEYSPACE) @defer.inlineCallbacks def tearDown(self): yield self.client.system_drop_keyspace(self.my_keyspace.name) self.cmanager.shutdown() for c in reactor.getDelayedCalls(): c.cancel() reactor.removeAll() @defer.inlineCallbacks def test_insert_get(self): yield self.client.insert('test', CF, 'testval', column=COLUMN) yield self.client.insert('test2', CF, 'testval2', column=COLUMN) yield self.client.insert('test', SCF, 'superval', column=COLUMN, super_column=SCOLUMN) yield self.client.insert('test2', SCF, 'superval2', column=COLUMN, super_column=SCOLUMN) res = yield self.client.get('test', CF, column=COLUMN) self.assertEqual(res.column.value, 'testval') res = yield self.client.get('test2', CF, column=COLUMN) self.assertEqual(res.column.value, 'testval2') res = yield self.client.get('test', SCF, column=COLUMN, super_column=SCOLUMN) self.assertEqual(res.column.value, 'superval') res = yield self.client.get('test2', SCF, column=COLUMN, super_column=SCOLUMN) self.assertEqual(res.column.value, 'superval2') @defer.inlineCallbacks def test_batch_insert_get_slice_and_count(self): yield self.client.batch_insert('test', CF, {COLUMN: 'test', COLUMN2: 'test2'}) yield self.client.batch_insert('test', SCF, {SCOLUMN: {COLUMN: 'test', COLUMN2: 'test2'}}) res = yield self.client.get_slice('test', CF, names=(COLUMN, COLUMN2)) self.assertEqual(res[0].column.value, 'test') self.assertEqual(res[1].column.value, 'test2') res = yield self.client.get_slice('test', SCF, names=(COLUMN, COLUMN2), super_column=SCOLUMN) self.assertEqual(res[0].column.value, 'test') self.assertEqual(res[1].column.value, 'test2') res = yield self.client.get_count('test', CF) self.assertEqual(res, 2) @defer.inlineCallbacks def test_batch_mutate_and_remove(self): yield self.client.batch_mutate({'test': {CF: {COLUMN: 'test', COLUMN2: 'test2'}, SCF: { SCOLUMN: { COLUMN: 'test', COLUMN2: 'test2'} } }, 'test2': {CF: {COLUMN: 'test', COLUMN2: 'test2'}, SCF: { SCOLUMN: { COLUMN: 'test', COLUMN2: 'test2'} } } }) res = yield self.client.get_slice('test', CF, names=(COLUMN, COLUMN2)) self.assertEqual(res[0].column.value, 'test') self.assertEqual(res[1].column.value, 'test2') res = yield self.client.get_slice('test2', CF, names=(COLUMN, COLUMN2)) self.assertEqual(res[0].column.value, 'test') self.assertEqual(res[1].column.value, 'test2') res = yield self.client.get_slice('test', SCF, names=(COLUMN, COLUMN2), super_column=SCOLUMN) self.assertEqual(res[0].column.value, 'test') self.assertEqual(res[1].column.value, 'test2') res = yield self.client.get_slice('test2', SCF, names=(COLUMN, COLUMN2), super_column=SCOLUMN) self.assertEqual(res[0].column.value, 'test') self.assertEqual(res[1].column.value, 'test2') yield self.client.batch_remove({CF: ['test', 'test2']}, names=['test', 'test2']) yield self.client.batch_remove({SCF: ['test', 'test2']}, names=['test', 'test2'], supercolumn=SCOLUMN) @defer.inlineCallbacks def test_batch_mutate_with_deletion(self): yield self.client.batch_mutate({'test': {CF: {COLUMN: 'test', COLUMN2: 'test2'}}}) res = yield self.client.get_slice('test', CF, names=(COLUMN, COLUMN2)) self.assertEqual(res[0].column.value, 'test') self.assertEqual(res[1].column.value, 'test2') yield self.client.batch_mutate({'test': {CF: {COLUMN: None, COLUMN2: 'test3'}}}) res = yield self.client.get_slice('test', CF, names=(COLUMN, COLUMN2)) self.assertEqual(len(res), 1) self.assertEqual(res[0].column.value, 'test3') @defer.inlineCallbacks def test_multiget_slice_remove(self): yield self.client.insert('test', CF, 'testval', column=COLUMN) yield self.client.insert('test', CF, 'testval', column=COLUMN2) yield self.client.insert('test2', CF, 'testval2', column=COLUMN) res = yield self.client.multiget(['test', 'test2'], CF, column=COLUMN) self.assertEqual(res['test'][0].column.value, 'testval') self.assertEqual(res['test2'][0].column.value, 'testval2') res = yield self.client.multiget_slice(['test', 'test2'], CF) self.assertEqual(res['test'][0].column.value, 'testval') self.assertEqual(res['test'][1].column.value, 'testval') self.assertEqual(res['test2'][0].column.value, 'testval2') yield self.client.remove('test', CF, column=COLUMN) yield self.client.remove('test2', CF, column=COLUMN) res = yield self.client.multiget(['test', 'test2'], CF, column=COLUMN) self.assertEqual(len(res['test']), 0) self.assertEqual(len(res['test2']), 0) @defer.inlineCallbacks def test_range_slices(self): yield self.client.insert('test', CF, 'testval', column=COLUMN) yield self.client.insert('test', CF, 'testval', column=COLUMN2) yield self.client.insert('test2', CF, 'testval2', column=COLUMN) ks = yield self.client.get_range_slices(CF, start='', finish='') keys = [k.key for k in ks] for key in ['test', 'test2']: self.assertIn(key, keys) @defer.inlineCallbacks def test_indexed_slices(self): yield self.client.insert('test1', IDX_CF, 'one', column='col1') yield self.client.insert('test2', IDX_CF, 'two', column='col1') yield self.client.insert('test3', IDX_CF, 'three', column='col1') expressions = [ttypes.IndexExpression('col1', ttypes.IndexOperator.EQ, 'two')] res = yield self.client.get_indexed_slices(IDX_CF, expressions, start_key='') self.assertEquals(res[0].columns[0].column.value,'two') @defer.inlineCallbacks def test_counter_add(self): if self.version < COUNTERS_SUPPORTED_API: raise unittest.SkipTest('Counters are not supported before 0.8') # test standard column counter yield self.client.add('test', COUNTER_CF, 1, column='col') res = yield self.client.get('test', COUNTER_CF, column='col') self.assertEquals(res.counter_column.value, 1) yield self.client.add('test', COUNTER_CF, 1, column='col') res = yield self.client.get('test', COUNTER_CF, column='col') self.assertEquals(res.counter_column.value, 2) # test super column counters yield self.client.add('test', SUPERCOUNTER_CF, 1, column='col', super_column='scol') res = yield self.client.get('test', SUPERCOUNTER_CF, column='col', super_column='scol') self.assertEquals(res.counter_column.value, 1) yield self.client.add('test', SUPERCOUNTER_CF, 1, column='col', super_column='scol') res = yield self.client.get('test', SUPERCOUNTER_CF, column='col', super_column='scol') self.assertEquals(res.counter_column.value, 2) @defer.inlineCallbacks def test_counter_remove(self): if self.version < COUNTERS_SUPPORTED_API: raise unittest.SkipTest('Counters are not supported before 0.8') # test standard column counter yield self.client.add('test', COUNTER_CF, 1, column='col') res = yield self.client.get('test', COUNTER_CF, column='col') self.assertEquals(res.counter_column.value, 1) yield self.client.remove_counter('test', COUNTER_CF, column='col') yield self.assertFailure(self.client.get('test', COUNTER_CF, column='col'), ttypes.NotFoundException) # test super column counters yield self.client.add('test', SUPERCOUNTER_CF, 1, column='col', super_column='scol') res = yield self.client.get('test', SUPERCOUNTER_CF, column='col', super_column='scol') self.assertEquals(res.counter_column.value, 1) yield self.client.remove_counter('test', SUPERCOUNTER_CF, column='col', super_column='scol') yield self.assertFailure(self.client.get('test', SUPERCOUNTER_CF, column='col', super_column='scol'), ttypes.NotFoundException) def sleep(self, secs): d = defer.Deferred() reactor.callLater(secs, d.callback, None) return d @defer.inlineCallbacks def test_ttls(self): yield self.client.insert('test_ttls', CF, 'testval', column=COLUMN, ttl=1) res = yield self.client.get('test_ttls', CF, column=COLUMN) self.assertEqual(res.column.value, 'testval') yield self.sleep(2) yield self.assertFailure(self.client.get('test_ttls', CF, column=COLUMN), ttypes.NotFoundException) yield self.client.batch_insert('test_ttls', CF, {COLUMN:'testval'}, ttl=1) res = yield self.client.get('test_ttls', CF, column=COLUMN) self.assertEqual(res.column.value, 'testval') yield self.sleep(2) yield self.assertFailure(self.client.get('test_ttls', CF, column=COLUMN), ttypes.NotFoundException) yield self.client.batch_mutate({'test_ttls': {CF: {COLUMN: 'testval'}}}, ttl=1) res = yield self.client.get('test_ttls', CF, column=COLUMN) self.assertEqual(res.column.value, 'testval') yield self.sleep(2) yield self.assertFailure(self.client.get('test_ttls', CF, column=COLUMN), ttypes.NotFoundException) def compare_keyspaces(self, ks1, ks2): self.assertEqual(ks1.name, ks2.name) self.assertEqual(ks1.strategy_class, ks2.strategy_class) self.assertEqual(ks1.cf_defs, ks2.cf_defs) def get_rf(ksdef): rf = ksdef.replication_factor if ksdef.strategy_options and \ 'replication_factor' in ksdef.strategy_options: rf = int(ksdef.strategy_options['replication_factor']) return rf def strat_opts_no_rf(ksdef): if not ksdef.strategy_options: return {} opts = ksdef.strategy_options.copy() if 'replication_factor' in ksdef.strategy_options: del opts['replication_factor'] return opts self.assertEqual(get_rf(ks1), get_rf(ks2)) self.assertEqual(strat_opts_no_rf(ks1), strat_opts_no_rf(ks2)) @defer.inlineCallbacks def test_keyspace_manipulation(self): try: yield self.client.system_drop_keyspace(T_KEYSPACE) except ttypes.InvalidRequestException: pass ksdef = ttypes.KsDef(name=T_KEYSPACE, strategy_class='org.apache.cassandra.locator.SimpleStrategy', strategy_options={}, cf_defs=[]) if self.version <= KS_RF_ATTRIBUTE: ksdef.replication_factor = 1 else: ksdef.strategy_options['replication_factor'] = '1' yield self.client.system_add_keyspace(ksdef) ks2 = yield self.client.describe_keyspace(T_KEYSPACE) self.compare_keyspaces(ksdef, ks2) if DO_SYSTEM_RENAMING: newname = T_KEYSPACE + '2' yield self.client.system_rename_keyspace(T_KEYSPACE, newname) ks2 = yield self.client.describe_keyspace(newname) ksdef.name = newname self.compare_keyspaces(ksdef, ks2) yield self.client.system_drop_keyspace(ksdef.name) yield self.assertFailure(self.client.describe_keyspace(T_KEYSPACE), ttypes.NotFoundException) if DO_SYSTEM_RENAMING: yield self.assertFailure(self.client.describe_keyspace(ksdef.name), ttypes.NotFoundException) @defer.inlineCallbacks def test_column_family_manipulation(self): # CfDef attributes present in all supported c*/thrift-api versions common_attrs = ( ('column_type', 'Standard'), ('comparator_type', 'org.apache.cassandra.db.marshal.BytesType'), ('comment', 'foo'), ('read_repair_chance', 1.0), ('column_metadata', []), ('gc_grace_seconds', 86400), ('default_validation_class', 'org.apache.cassandra.db.marshal.BytesType'), ('min_compaction_threshold', 5), ('max_compaction_threshold', 31), ) cfdef = ttypes.CfDef(KEYSPACE, T_CF) for attr, val in common_attrs: setattr(cfdef, attr, val) yield self.client.system_add_column_family(cfdef) ksdef = yield self.client.describe_keyspace(KEYSPACE) cfdefs = [c for c in ksdef.cf_defs if c.name == T_CF] self.assertEqual(len(cfdefs), 1) cfdef2 = cfdefs[0] for attr, val in common_attrs: val1 = getattr(cfdef, attr) val2 = getattr(cfdef2, attr) self.assertEqual(val1, val2, 'attribute %s mismatch: %r != %r' % (attr, val1, val2)) if DO_SYSTEM_RENAMING: newname = T_CF + '2' yield self.client.system_rename_column_family(T_CF, newname) ksdef = yield self.client.describe_keyspace(KEYSPACE) cfdef2 = [c for c in ksdef.cf_defs if c.name == newname][0] self.assertNotIn(T_CF, [c.name for c in ksdef.cf_defs]) cfdef.name = newname self.assertEqual(cfdef, cfdef2) yield self.client.system_drop_column_family(cfdef.name) ksdef = yield self.client.describe_keyspace(KEYSPACE) self.assertNotIn(cfdef.name, [c.name for c in ksdef.cf_defs]) @defer.inlineCallbacks def test_describes(self): name = yield self.client.describe_cluster_name() self.assertIsInstance(name, str) self.assertNotEqual(name, '') partitioner = yield self.client.describe_partitioner() self.assert_(partitioner.startswith('org.apache.cassandra.'), msg='partitioner is %r' % partitioner) snitch = yield self.client.describe_snitch() self.assert_(snitch.startswith('org.apache.cassandra.'), msg='snitch is %r' % snitch) version = yield self.client.describe_version() self.assertIsInstance(version, str) self.assertIn('.', version) schemavers = yield self.client.describe_schema_versions() self.assertIsInstance(schemavers, dict) self.assertNotEqual(schemavers, {}) ring = yield self.client.describe_ring(KEYSPACE) self.assertIsInstance(ring, list) self.assertNotEqual(ring, []) for r in ring: self.assertIsInstance(r.start_token, str) self.assertIsInstance(r.end_token, str) self.assertIsInstance(r.endpoints, list) self.assertNotEqual(r.endpoints, []) for ep in r.endpoints: self.assertIsInstance(ep, str) @defer.inlineCallbacks def test_errback(self): yield self.client.remove('poiqwe', CF) try: yield self.client.get('poiqwe', CF, column='foo') except Exception, e: pass
class CassandraProvisionerStore(object): """ Provides high level provisioner storage operations for Cassandra """ # default size of paged fetches _PAGE_SIZE = 100 LAUNCH_CF_NAME = "ProvisionerLaunches" NODE_CF_NAME = "ProvisionerNodes" @classmethod def get_column_families(cls, keyspace=None, prefix=''): """Builds a list of column families needed by this store. @param keyspace Name of keyspace. If None, it must be added manually. @param prefix Optional prefix for cf names. Useful for testing. @retval list of CfDef objects """ launch_cf_name = prefix + cls.LAUNCH_CF_NAME node_cf_name = prefix + cls.NODE_CF_NAME return [CfDef(keyspace, launch_cf_name, comparator_type='org.apache.cassandra.db.marshal.UTF8Type'), CfDef(keyspace, node_cf_name, comparator_type='org.apache.cassandra.db.marshal.UTF8Type')] def __init__(self, host, port, username, password, keyspace, prefix=''): self._launch_column_family = prefix + self.LAUNCH_CF_NAME self._node_column_family = prefix + self.NODE_CF_NAME authz= {'username': username, 'password': password} self._manager = ManagedCassandraClientFactory( credentials=authz, check_api_version=True, keyspace=keyspace) self.client = CassandraClient(self._manager) self._host = host self._port = port self._connector = None def connect(self): self._connector = reactor.connectTCP(self._host, self._port, self._manager) def disconnect(self): self._manager.shutdown() if self._connector: self._connector.disconnect() self._connector = None @timeout(CASSANDRA_TIMEOUT) def put_launch(self, launch): """ @brief Stores a single launch record @param launch Launch record to store @retval Deferred for success """ launch_id = launch['launch_id'] state = launch['state'] value = json.dumps(launch) return self.client.insert(launch_id, self._launch_column_family, value, column=state) @timeout(CASSANDRA_TIMEOUT) @defer.inlineCallbacks def put_nodes(self, nodes): """ @brief Stores a set of node records @param nodes Iterable of node records @retval Deferred for success """ # could be more efficient with a batch_mutate for node in nodes: yield self.put_node(node) @timeout(CASSANDRA_TIMEOUT) def put_node(self, node): """ @brief Stores a node record @param node Node record @retval Deferred for success """ node_id = node['node_id'] state = node['state'] value = json.dumps(node) return self.client.insert(node_id, self._node_column_family, value, column=state) @timeout(CASSANDRA_TIMEOUT) def get_launch(self, launch_id, count=1): """ @brief Retrieves a launch record by id @param launch_id Id of launch record to retrieve @param count Number of launch state records to retrieve @retval Deferred record(s), or None. A list of records if count > 1 """ return self._get_record(launch_id, self._launch_column_family, count) @timeout(CASSANDRA_TIMEOUT) def get_launches(self, state=None, min_state=None, max_state=None): """ @brief Retrieves the latest record for all launches within a state range @param state Only retrieve nodes in this state. @param min_state Inclusive start bound @param max_state Inclusive end bound @retval Deferred list of launch records """ return self._get_records(self._launch_column_family, state=state, min_state=min_state, max_state=max_state) @timeout(CASSANDRA_TIMEOUT) def get_node(self, node_id, count=1): """ @brief Retrieves a launch record by id @param node_id Id of node record to retrieve @param count Number of node state records to retrieve @retval Deferred record(s), or None. A list of records if count > 1 """ return self._get_record(node_id, self._node_column_family, count) @timeout(CASSANDRA_TIMEOUT) def get_nodes(self, state=None, min_state=None, max_state=None): """ @brief Retrieves all launch record within a state range @param state Only retrieve nodes in this state. @param min_state Inclusive start bound. @param max_state Inclusive end bound @retval Deferred list of launch records """ return self._get_records(self._node_column_family, state=state, min_state=min_state, max_state=max_state) @defer.inlineCallbacks def _get_record(self, key, column_family, count): slice = yield self.client.get_slice(key, column_family, reverse=True, count=count) # we're probably only interested in the last record, in sorted order. # This is the latest state the object has recorded. records = [json.loads(column.column.value) for column in slice] if count == 1: if records: ret = records[0] else: ret = None else: ret = records defer.returnValue(ret) @defer.inlineCallbacks def _get_records(self, column_family, state=None, min_state=None, max_state=None, reverse=True): # overrides range arguments if state: min_state = max_state = state start = '' end = min_state or '' if not reverse: start, end = end, start # this is tricky. We are only concerned with the latest state record # (by sort order not necessarily time). So when we look for records # within a state range, we effectively must pull down the latest state # for each record, and filter them locally. This is slightly improved # when a first_state (or last when reverse=False) is specified as the # server can skip any records not >= that state. records = [] done = False start_key = '' iterations = 0 while not done: slices = yield self.client.get_range_slices(column_family, column_start=start, column_finish=end, reverse=reverse, column_count=1, start=start_key, count=self._PAGE_SIZE) skipped_one = False for slice in slices: if not skipped_one and iterations: # if this not the first batch, skip the first element as it # will be a dupe. skipped_one = True continue if not slice.columns: # rows without matching columns will still be returned continue record = json.loads(slice.columns[0].column.value) if not max_state or record['state'] <= max_state: if not min_state or record['state'] >= min_state: records.append(record) # page through results. by default only 100 are returned at a time if len(slices) == self._PAGE_SIZE: start_key = slices[-1].key else: done = True iterations += 1 defer.returnValue(records) def on_deactivate(self, *args, **kwargs): self._manager.shutdown() log.info('on_deactivate: Lose Connection TCP') def on_terminate(self, *args, **kwargs): self._manager.shutdown() log.info('on_terminate: Lose Connection TCP')
class CassandraClientTest(unittest.TestCase): def setUp(self): self.cmanager = ManagedCassandraClientFactory() self.client = CassandraClient(self.cmanager, KEYSPACE) for i in xrange(CONNS): reactor.connectTCP(HOST, PORT, self.cmanager) return self.cmanager.deferred @defer.inlineCallbacks def tearDown(self): yield self.client.remove('test', CF) yield self.client.remove('test2', CF) yield self.client.remove('test', SCF) yield self.client.remove('test2', SCF) self.cmanager.shutdown() for c in reactor.getDelayedCalls(): c.cancel() reactor.removeAll() @defer.inlineCallbacks def test_insert_get(self): yield self.client.insert('test', ColumnPath(CF, None, COLUMN), 'testval') yield self.client.insert('test2', CF, 'testval2', column=COLUMN) yield self.client.insert('test', ColumnPath(SCF, SCOLUMN, COLUMN), 'superval') yield self.client.insert('test2', SCF, 'superval2', column=COLUMN, super_column=SCOLUMN) res = yield self.client.get('test', CF, column=COLUMN) self.assert_(res.column.value == 'testval') res = yield self.client.get('test2', CF, column=COLUMN) self.assert_(res.column.value == 'testval2') res = yield self.client.get('test', SCF, column=COLUMN, super_column=SCOLUMN) self.assert_(res.column.value == 'superval') res = yield self.client.get('test2', SCF, column=COLUMN, super_column=SCOLUMN) self.assert_(res.column.value == 'superval2') @defer.inlineCallbacks def test_batch_insert_get_slice_and_count(self): yield self.client.batch_insert('test', CF, {COLUMN: 'test', COLUMN2: 'test2'}) yield self.client.batch_insert('test', SCF, {SCOLUMN: {COLUMN: 'test', COLUMN2: 'test2'}}) res = yield self.client.get_slice('test', CF, names=(COLUMN, COLUMN2)) self.assert_(res[0].column.value == 'test') self.assert_(res[1].column.value == 'test2') res = yield self.client.get_slice('test', SCF, names=(COLUMN, COLUMN2), super_column=SCOLUMN) self.assert_(res[0].column.value == 'test') self.assert_(res[1].column.value == 'test2') res = yield self.client.get_count('test', CF) self.assert_(res == 2) @defer.inlineCallbacks def test_batch_mutate_and_remove(self): yield self.client.batch_mutate({'test': {CF: {COLUMN: 'test', COLUMN2: 'test2'}, SCF: { SCOLUMN: { COLUMN: 'test', COLUMN2: 'test2'} } }, 'test2': {CF: {COLUMN: 'test', COLUMN2: 'test2'}, SCF: { SCOLUMN: { COLUMN: 'test', COLUMN2: 'test2'} } } }) res = yield self.client.get_slice('test', CF, names=(COLUMN, COLUMN2)) self.assert_(res[0].column.value == 'test') self.assert_(res[1].column.value == 'test2') res = yield self.client.get_slice('test2', CF, names=(COLUMN, COLUMN2)) self.assert_(res[0].column.value == 'test') self.assert_(res[1].column.value == 'test2') res = yield self.client.get_slice('test', SCF, names=(COLUMN, COLUMN2), super_column=SCOLUMN) self.assert_(res[0].column.value == 'test') self.assert_(res[1].column.value == 'test2') res = yield self.client.get_slice('test2', SCF, names=(COLUMN, COLUMN2), super_column=SCOLUMN) self.assert_(res[0].column.value == 'test') self.assert_(res[1].column.value == 'test2') yield self.client.batch_remove({CF: ['test', 'test2']}, names=['test', 'test2']) yield self.client.batch_remove({SCF: ['test', 'test2']}, names=['test', 'test2'], supercolumn=SCOLUMN) @defer.inlineCallbacks def test_batch_mutate_with_deletion(self): yield self.client.batch_mutate({'test': {CF: {COLUMN: 'test', COLUMN2: 'test2'}}}) res = yield self.client.get_slice('test', CF, names=(COLUMN, COLUMN2)) self.assert_(res[0].column.value == 'test') self.assert_(res[1].column.value == 'test2') yield self.client.batch_mutate({'test': {CF: {COLUMN: None, COLUMN2: 'test3'}}}) res = yield self.client.get_slice('test', CF, names=(COLUMN, COLUMN2)) self.assert_(len(res) == 1) self.assert_(res[0].column.value == 'test3') @defer.inlineCallbacks def test_multiget_slice_remove(self): yield self.client.insert('test', CF, 'testval', column=COLUMN) yield self.client.insert('test', CF, 'testval', column=COLUMN2) yield self.client.insert('test2', CF, 'testval2', column=COLUMN) res = yield self.client.multiget(['test', 'test2'], CF, column=COLUMN) self.assert_(res['test'].column.value == 'testval') self.assert_(res['test2'].column.value == 'testval2') res = yield self.client.multiget_slice(['test', 'test2'], CF) self.assert_(res['test'][0].column.value == 'testval') self.assert_(res['test'][1].column.value == 'testval') self.assert_(res['test2'][0].column.value == 'testval2') yield self.client.remove('test', CF, column=COLUMN) yield self.client.remove('test2', CF, column=COLUMN) res = yield self.client.multiget(['test', 'test2'], CF, column=COLUMN) self.assert_(res['test'].column == None) self.assert_(res['test2'].column == None) @defer.inlineCallbacks def test_range_slices(self): yield self.client.insert('test', CF, 'testval', column=COLUMN) yield self.client.insert('test', CF, 'testval', column=COLUMN2) yield self.client.insert('test2', CF, 'testval2', column=COLUMN) ks = yield self.client.get_range_slices(CF, start='', finish='') keys = [k.key for k in ks] for key in ['test', 'test2']: self.assert_(key in keys) @defer.inlineCallbacks def test_errback(self): yield self.client.remove('poiqwe', CF) try: yield self.client.get('poiqwe', CF, column='foo') except Exception, e: pass