def do_this():
for keyspace in keyspaces:
factory = ManagedCassandraClientFactory(keyspace)
reactor.connectTCP("localhost", 9160, factory)
client = CassandraClient(factory)
k = yield client.describe_keyspace(keyspace)
for cf in [c.name for c in k.cf_defs]:
n = 0
db_rows = yield client.get_range_slices(column_family=cf, count=10000000)
with gzip.GzipFile("%s.%s.csv.gz" % (keyspace, cf), "w") as f:
out = csv.DictWriter(f, fieldnames=["keyspace", "cf", "key", "col", "val"])
out.writeheader()
for row in db_rows:
for col in row.columns:
out.writerow({"keyspace": keyspace,
"cf": cf,
"key": row.key.encode("string_escape"),
"col": col.column.name,
"val": col.column.value.encode("string_escape")})
n += 1
print "Successfully backed up %d rows from %s.%s" % (n, keyspace, cf)
reactor.stop()
class CassandraClientTest(unittest.TestCase):
@defer.inlineCallbacks
def setUp(self):
self.cmanager = ManagedCassandraClientFactory(keyspace='system')
self.client = CassandraClient(self.cmanager)
for i in xrange(CONNS):
reactor.connectTCP(HOST, PORT, self.cmanager)
yield self.cmanager.deferred
remote_ver = yield self.client.describe_version()
self.version = getAPIVersion(remote_ver)
self.my_keyspace = KsDef(
name=KEYSPACE,
strategy_class='org.apache.cassandra.locator.SimpleStrategy',
strategy_options={'replication_factor': '1'},
cf_defs=[
CfDef(
keyspace=KEYSPACE,
name=CF,
column_type='Standard'
),
CfDef(
keyspace=KEYSPACE,
name=SCF,
column_type='Super'
),
CfDef(
keyspace=KEYSPACE,
name=IDX_CF,
column_type='Standard',
comparator_type='org.apache.cassandra.db.marshal.UTF8Type',
column_metadata=[
ColumnDef(
name='col1',
validation_class='org.apache.cassandra.db.marshal.UTF8Type',
index_type=IndexType.KEYS,
index_name='idxCol1')
],
default_validation_class='org.apache.cassandra.db.marshal.BytesType'
),
]
)
if self.version == CASSANDRA_08_VERSION:
self.my_keyspace.cf_defs.extend([
CfDef(
keyspace=KEYSPACE,
name=COUNTER_CF,
column_type='Standard',
default_validation_class='org.apache.cassandra.db.marshal.CounterColumnType'
),
CfDef(
keyspace=KEYSPACE,
name=SUPERCOUNTER_CF,
column_type='Super',
default_validation_class='org.apache.cassandra.db.marshal.CounterColumnType'
),
])
yield self.client.system_add_keyspace(self.my_keyspace)
yield self.client.set_keyspace(KEYSPACE)
@defer.inlineCallbacks
def tearDown(self):
yield self.client.system_drop_keyspace(self.my_keyspace.name)
self.cmanager.shutdown()
for c in reactor.getDelayedCalls():
c.cancel()
reactor.removeAll()
@defer.inlineCallbacks
def test_insert_get(self):
yield self.client.insert('test', CF, 'testval', column=COLUMN)
yield self.client.insert('test2', CF, 'testval2', column=COLUMN)
yield self.client.insert('test', SCF, 'superval', column=COLUMN, super_column=SCOLUMN)
yield self.client.insert('test2', SCF, 'superval2', column=COLUMN,
super_column=SCOLUMN)
res = yield self.client.get('test', CF, column=COLUMN)
self.assertEqual(res.column.value, 'testval')
res = yield self.client.get('test2', CF, column=COLUMN)
self.assertEqual(res.column.value, 'testval2')
res = yield self.client.get('test', SCF, column=COLUMN, super_column=SCOLUMN)
self.assertEqual(res.column.value, 'superval')
res = yield self.client.get('test2', SCF, column=COLUMN, super_column=SCOLUMN)
self.assertEqual(res.column.value, 'superval2')
@defer.inlineCallbacks
def test_batch_insert_get_slice_and_count(self):
yield self.client.batch_insert('test', CF,
{COLUMN: 'test', COLUMN2: 'test2'})
yield self.client.batch_insert('test', SCF,
{SCOLUMN: {COLUMN: 'test', COLUMN2: 'test2'}})
res = yield self.client.get_slice('test', CF, names=(COLUMN, COLUMN2))
self.assertEqual(res[0].column.value, 'test')
self.assertEqual(res[1].column.value, 'test2')
res = yield self.client.get_slice('test', SCF, names=(COLUMN, COLUMN2),
super_column=SCOLUMN)
self.assertEqual(res[0].column.value, 'test')
self.assertEqual(res[1].column.value, 'test2')
res = yield self.client.get_count('test', CF)
self.assertEqual(res, 2)
@defer.inlineCallbacks
def test_batch_mutate_and_remove(self):
yield self.client.batch_mutate({'test': {CF: {COLUMN: 'test', COLUMN2: 'test2'}, SCF: { SCOLUMN: { COLUMN: 'test', COLUMN2: 'test2'} } }, 'test2': {CF: {COLUMN: 'test', COLUMN2: 'test2'}, SCF: { SCOLUMN: { COLUMN: 'test', COLUMN2: 'test2'} } } })
res = yield self.client.get_slice('test', CF, names=(COLUMN, COLUMN2))
self.assertEqual(res[0].column.value, 'test')
self.assertEqual(res[1].column.value, 'test2')
res = yield self.client.get_slice('test2', CF, names=(COLUMN, COLUMN2))
self.assertEqual(res[0].column.value, 'test')
self.assertEqual(res[1].column.value, 'test2')
res = yield self.client.get_slice('test', SCF, names=(COLUMN, COLUMN2),
super_column=SCOLUMN)
self.assertEqual(res[0].column.value, 'test')
self.assertEqual(res[1].column.value, 'test2')
res = yield self.client.get_slice('test2', SCF, names=(COLUMN, COLUMN2),
super_column=SCOLUMN)
self.assertEqual(res[0].column.value, 'test')
self.assertEqual(res[1].column.value, 'test2')
yield self.client.batch_remove({CF: ['test', 'test2']}, names=['test', 'test2'])
yield self.client.batch_remove({SCF: ['test', 'test2']}, names=['test', 'test2'], supercolumn=SCOLUMN)
@defer.inlineCallbacks
def test_batch_mutate_with_deletion(self):
yield self.client.batch_mutate({'test': {CF: {COLUMN: 'test', COLUMN2: 'test2'}}})
res = yield self.client.get_slice('test', CF, names=(COLUMN, COLUMN2))
self.assertEqual(res[0].column.value, 'test')
self.assertEqual(res[1].column.value, 'test2')
yield self.client.batch_mutate({'test': {CF: {COLUMN: None, COLUMN2: 'test3'}}})
res = yield self.client.get_slice('test', CF, names=(COLUMN, COLUMN2))
self.assertEqual(len(res), 1)
self.assertEqual(res[0].column.value, 'test3')
@defer.inlineCallbacks
def test_multiget_slice_remove(self):
yield self.client.insert('test', CF, 'testval', column=COLUMN)
yield self.client.insert('test', CF, 'testval', column=COLUMN2)
yield self.client.insert('test2', CF, 'testval2', column=COLUMN)
res = yield self.client.multiget(['test', 'test2'], CF, column=COLUMN)
self.assertEqual(res['test'][0].column.value, 'testval')
self.assertEqual(res['test2'][0].column.value, 'testval2')
res = yield self.client.multiget_slice(['test', 'test2'], CF)
self.assertEqual(res['test'][0].column.value, 'testval')
self.assertEqual(res['test'][1].column.value, 'testval')
self.assertEqual(res['test2'][0].column.value, 'testval2')
yield self.client.remove('test', CF, column=COLUMN)
yield self.client.remove('test2', CF, column=COLUMN)
res = yield self.client.multiget(['test', 'test2'], CF, column=COLUMN)
self.assertEqual(len(res['test']), 0)
self.assertEqual(len(res['test2']), 0)
@defer.inlineCallbacks
def test_range_slices(self):
yield self.client.insert('test', CF, 'testval', column=COLUMN)
yield self.client.insert('test', CF, 'testval', column=COLUMN2)
yield self.client.insert('test2', CF, 'testval2', column=COLUMN)
ks = yield self.client.get_range_slices(CF, start='', finish='')
keys = [k.key for k in ks]
for key in ['test', 'test2']:
self.assertIn(key, keys)
@defer.inlineCallbacks
def test_indexed_slices(self):
yield self.client.insert('test1', IDX_CF, 'one', column='col1')
yield self.client.insert('test2', IDX_CF, 'two', column='col1')
yield self.client.insert('test3', IDX_CF, 'three', column='col1')
expressions = [IndexExpression('col1', IndexOperator.EQ, 'two')]
res = yield self.client.get_indexed_slices(IDX_CF, expressions, start_key='')
self.assertEquals(res[0].columns[0].column.value,'two')
@defer.inlineCallbacks
def test_counter_add(self):
if self.version != CASSANDRA_08_VERSION:
raise unittest.SkipTest('Counters are not supported in 0.7')
# test standard column counter
yield self.client.add('test', COUNTER_CF, 1, column='col')
res = yield self.client.get('test', COUNTER_CF, column='col')
self.assertEquals(res.counter_column.value, 1)
yield self.client.add('test', COUNTER_CF, 1, column='col')
res = yield self.client.get('test', COUNTER_CF, column='col')
self.assertEquals(res.counter_column.value, 2)
# test super column counters
yield self.client.add('test', SUPERCOUNTER_CF, 1, column='col', super_column='scol')
res = yield self.client.get('test', SUPERCOUNTER_CF, column='col', super_column='scol')
self.assertEquals(res.counter_column.value, 1)
yield self.client.add('test', SUPERCOUNTER_CF, 1, column='col', super_column='scol')
res = yield self.client.get('test', SUPERCOUNTER_CF, column='col', super_column='scol')
self.assertEquals(res.counter_column.value, 2)
@defer.inlineCallbacks
def test_counter_remove(self):
if self.version != CASSANDRA_08_VERSION:
raise unittest.SkipTest('Counters are not supported in 0.7')
# test standard column counter
yield self.client.add('test', COUNTER_CF, 1, column='col')
res = yield self.client.get('test', COUNTER_CF, column='col')
self.assertEquals(res.counter_column.value, 1)
yield self.client.remove_counter('test', COUNTER_CF, column='col')
yield self.assertFailure(self.client.get('test', COUNTER_CF, column='col'),
NotFoundException)
# test super column counters
yield self.client.add('test', SUPERCOUNTER_CF, 1, column='col', super_column='scol')
res = yield self.client.get('test', SUPERCOUNTER_CF, column='col', super_column='scol')
self.assertEquals(res.counter_column.value, 1)
yield self.client.remove_counter('test', SUPERCOUNTER_CF,
column='col', super_column='scol')
yield self.assertFailure(self.client.get('test', SUPERCOUNTER_CF,
column='col', super_column='scol'),
NotFoundException)
@defer.inlineCallbacks
def test_cql(self):
if self.version != CASSANDRA_08_VERSION:
raise unittest.SkipTest('CQL is not supported in 0.7')
yield self.client.insert('test', CF, 'testval', column='col1')
res = yield self.client.get('test', CF, column='col1')
self.assertEquals(res.column.value, 'testval')
query = 'SELECT * from %s where KEY = %s' % (CF, 'test'.encode('hex'))
uncompressed_result = yield self.client.execute_cql_query(query, Compression.NONE)
self.assertEquals(uncompressed_result.rows[0].columns[0].name, 'col1')
self.assertEquals(uncompressed_result.rows[0].columns[0].value, 'testval')
compressed_query = zlib.compress(query)
compressed_result = yield self.client.execute_cql_query(compressed_query, Compression.GZIP)
self.assertEquals(uncompressed_result, compressed_result)
def sleep(self, secs):
d = defer.Deferred()
reactor.callLater(secs, d.callback, None)
return d
@defer.inlineCallbacks
def test_ttls(self):
yield self.client.insert('test_ttls', CF, 'testval', column=COLUMN, ttl=1)
res = yield self.client.get('test_ttls', CF, column=COLUMN)
self.assertEqual(res.column.value, 'testval')
yield self.sleep(2)
yield self.assertFailure(self.client.get('test_ttls', CF, column=COLUMN), NotFoundException)
yield self.client.batch_insert('test_ttls', CF, {COLUMN:'testval'}, ttl=1)
res = yield self.client.get('test_ttls', CF, column=COLUMN)
self.assertEqual(res.column.value, 'testval')
yield self.sleep(2)
yield self.assertFailure(self.client.get('test_ttls', CF, column=COLUMN), NotFoundException)
yield self.client.batch_mutate({'test_ttls': {CF: {COLUMN: 'testval'}}}, ttl=1)
res = yield self.client.get('test_ttls', CF, column=COLUMN)
self.assertEqual(res.column.value, 'testval')
yield self.sleep(2)
yield self.assertFailure(self.client.get('test_ttls', CF, column=COLUMN), NotFoundException)
def compare_keyspaces(self, ks1, ks2):
self.assertEqual(ks1.name, ks2.name)
self.assertEqual(ks1.strategy_class, ks2.strategy_class)
self.assertEqual(ks1.cf_defs, ks2.cf_defs)
def get_rf(ksdef):
rf = ksdef.replication_factor
if ksdef.strategy_options and \
'replication_factor' in ksdef.strategy_options:
rf = int(ksdef.strategy_options['replication_factor'])
return rf
def strat_opts_no_rf(ksdef):
if not ksdef.strategy_options:
return {}
opts = ksdef.strategy_options.copy()
if 'replication_factor' in ksdef.strategy_options:
del opts['replication_factor']
return opts
self.assertEqual(get_rf(ks1), get_rf(ks2))
self.assertEqual(strat_opts_no_rf(ks1), strat_opts_no_rf(ks2))
@defer.inlineCallbacks
def test_keyspace_manipulation(self):
try:
yield self.client.system_drop_keyspace(T_KEYSPACE)
except InvalidRequestException:
pass
ksdef = KsDef(name=T_KEYSPACE, strategy_class='org.apache.cassandra.locator.SimpleStrategy', strategy_options={'replication_factor': '1'}, cf_defs=[])
yield self.client.system_add_keyspace(ksdef)
ks2 = yield self.client.describe_keyspace(T_KEYSPACE)
self.compare_keyspaces(ksdef, ks2)
if DO_SYSTEM_RENAMING:
newname = T_KEYSPACE + '2'
yield self.client.system_rename_keyspace(T_KEYSPACE, newname)
ks2 = yield self.client.describe_keyspace(newname)
ksdef.name = newname
self.compare_keyspaces(ksdef, ks2)
yield self.client.system_drop_keyspace(ksdef.name)
yield self.assertFailure(self.client.describe_keyspace(T_KEYSPACE), NotFoundException)
if DO_SYSTEM_RENAMING:
yield self.assertFailure(self.client.describe_keyspace(ksdef.name), NotFoundException)
@defer.inlineCallbacks
def test_column_family_manipulation(self):
cfdef = CfDef(KEYSPACE, T_CF,
column_type='Standard',
comparator_type='org.apache.cassandra.db.marshal.BytesType',
comment='foo',
row_cache_size=0.0,
key_cache_size=200000.0,
read_repair_chance=1.0,
column_metadata=[],
gc_grace_seconds=86400,
default_validation_class='org.apache.cassandra.db.marshal.BytesType',
key_validation_class='org.apache.cassandra.db.marshal.BytesType',
min_compaction_threshold=5,
max_compaction_threshold=31,
row_cache_save_period_in_seconds=0,
key_cache_save_period_in_seconds=3600,
memtable_flush_after_mins=60,
memtable_throughput_in_mb=249,
memtable_operations_in_millions=1.1671875,
replicate_on_write=False,
merge_shards_chance=0.10000000000000001,
row_cache_provider=None,
key_alias=None,
)
post_07_fields = ['replicate_on_write', 'merge_shards_chance',
'key_validation_class', 'row_cache_provider', 'key_alias']
yield self.client.system_add_column_family(cfdef)
ksdef = yield self.client.describe_keyspace(KEYSPACE)
cfdef2 = [c for c in ksdef.cf_defs if c.name == T_CF][0]
for field in post_07_fields:
# Most of these are ignored in 0.7, so we can't reliably compare them
setattr(cfdef, field, None)
setattr(cfdef2, field, None)
# we don't know the id ahead of time. copy the new one so the equality
# comparison won't fail
cfdef.id = cfdef2.id
self.assertEqual(cfdef, cfdef2)
if DO_SYSTEM_RENAMING:
newname = T_CF + '2'
yield self.client.system_rename_column_family(T_CF, newname)
ksdef = yield self.client.describe_keyspace(KEYSPACE)
cfdef2 = [c for c in ksdef.cf_defs if c.name == newname][0]
self.assertNotIn(T_CF, [c.name for c in ksdef.cf_defs])
cfdef.name = newname
self.assertEqual(cfdef, cfdef2)
yield self.client.system_drop_column_family(cfdef.name)
ksdef = yield self.client.describe_keyspace(KEYSPACE)
self.assertNotIn(cfdef.name, [c.name for c in ksdef.cf_defs])
@defer.inlineCallbacks
def test_describes(self):
name = yield self.client.describe_cluster_name()
self.assertIsInstance(name, str)
self.assertNotEqual(name, '')
partitioner = yield self.client.describe_partitioner()
self.assert_(partitioner.startswith('org.apache.cassandra.'),
msg='partitioner is %r' % partitioner)
snitch = yield self.client.describe_snitch()
self.assert_(snitch.startswith('org.apache.cassandra.'),
msg='snitch is %r' % snitch)
version = yield self.client.describe_version()
self.assertIsInstance(version, str)
self.assertIn('.', version)
schemavers = yield self.client.describe_schema_versions()
self.assertIsInstance(schemavers, dict)
self.assertNotEqual(schemavers, {})
ring = yield self.client.describe_ring(KEYSPACE)
self.assertIsInstance(ring, list)
self.assertNotEqual(ring, [])
for r in ring:
self.assertIsInstance(r.start_token, str)
self.assertIsInstance(r.end_token, str)
self.assertIsInstance(r.endpoints, list)
self.assertNotEqual(r.endpoints, [])
for ep in r.endpoints:
self.assertIsInstance(ep, str)
@defer.inlineCallbacks
def test_errback(self):
yield self.client.remove('poiqwe', CF)
try:
yield self.client.get('poiqwe', CF, column='foo')
except Exception, e:
pass
class CassandraClientTest(unittest.TestCase):
@defer.inlineCallbacks
def setUp(self):
self.cmanager = ManagedCassandraClientFactory(keyspace='system')
self.client = CassandraClient(self.cmanager)
for i in xrange(CONNS):
reactor.connectTCP(HOST, PORT, self.cmanager)
yield self.cmanager.deferred
remote_ver = yield self.client.describe_version()
self.version = thrift_api_ver_to_cassandra_ver(remote_ver)
self.my_keyspace = KsDef(
name=KEYSPACE,
strategy_class='org.apache.cassandra.locator.SimpleStrategy',
strategy_options={'replication_factor': '1'},
cf_defs=[
CfDef(keyspace=KEYSPACE, name=CF, column_type='Standard'),
CfDef(keyspace=KEYSPACE, name=SCF, column_type='Super'),
CfDef(
keyspace=KEYSPACE,
name=IDX_CF,
column_type='Standard',
comparator_type='org.apache.cassandra.db.marshal.UTF8Type',
column_metadata=[
ColumnDef(name='col1',
validation_class=
'org.apache.cassandra.db.marshal.UTF8Type',
index_type=IndexType.KEYS,
index_name='idxCol1')
],
default_validation_class=
'org.apache.cassandra.db.marshal.BytesType'),
])
if self.version == CASSANDRA_08_VERSION:
self.my_keyspace.cf_defs.extend([
CfDef(keyspace=KEYSPACE,
name=COUNTER_CF,
column_type='Standard',
default_validation_class=
'org.apache.cassandra.db.marshal.CounterColumnType'),
CfDef(keyspace=KEYSPACE,
name=SUPERCOUNTER_CF,
column_type='Super',
default_validation_class=
'org.apache.cassandra.db.marshal.CounterColumnType'),
])
yield self.client.system_add_keyspace(self.my_keyspace)
yield self.client.set_keyspace(KEYSPACE)
@defer.inlineCallbacks
def tearDown(self):
yield self.client.system_drop_keyspace(self.my_keyspace.name)
self.cmanager.shutdown()
for c in reactor.getDelayedCalls():
c.cancel()
reactor.removeAll()
@defer.inlineCallbacks
def test_insert_get(self):
yield self.client.insert('test', CF, 'testval', column=COLUMN)
yield self.client.insert('test2', CF, 'testval2', column=COLUMN)
yield self.client.insert('test',
SCF,
'superval',
column=COLUMN,
super_column=SCOLUMN)
yield self.client.insert('test2',
SCF,
'superval2',
column=COLUMN,
super_column=SCOLUMN)
res = yield self.client.get('test', CF, column=COLUMN)
self.assertEqual(res.column.value, 'testval')
res = yield self.client.get('test2', CF, column=COLUMN)
self.assertEqual(res.column.value, 'testval2')
res = yield self.client.get('test',
SCF,
column=COLUMN,
super_column=SCOLUMN)
self.assertEqual(res.column.value, 'superval')
res = yield self.client.get('test2',
SCF,
column=COLUMN,
super_column=SCOLUMN)
self.assertEqual(res.column.value, 'superval2')
@defer.inlineCallbacks
def test_batch_insert_get_slice_and_count(self):
yield self.client.batch_insert('test', CF, {
COLUMN: 'test',
COLUMN2: 'test2'
})
yield self.client.batch_insert(
'test', SCF, {SCOLUMN: {
COLUMN: 'test',
COLUMN2: 'test2'
}})
res = yield self.client.get_slice('test', CF, names=(COLUMN, COLUMN2))
self.assertEqual(res[0].column.value, 'test')
self.assertEqual(res[1].column.value, 'test2')
res = yield self.client.get_slice('test',
SCF,
names=(COLUMN, COLUMN2),
super_column=SCOLUMN)
self.assertEqual(res[0].column.value, 'test')
self.assertEqual(res[1].column.value, 'test2')
res = yield self.client.get_count('test', CF)
self.assertEqual(res, 2)
@defer.inlineCallbacks
def test_batch_mutate_and_remove(self):
yield self.client.batch_mutate({
'test': {
CF: {
COLUMN: 'test',
COLUMN2: 'test2'
},
SCF: {
SCOLUMN: {
COLUMN: 'test',
COLUMN2: 'test2'
}
}
},
'test2': {
CF: {
COLUMN: 'test',
COLUMN2: 'test2'
},
SCF: {
SCOLUMN: {
COLUMN: 'test',
COLUMN2: 'test2'
}
}
}
})
res = yield self.client.get_slice('test', CF, names=(COLUMN, COLUMN2))
self.assertEqual(res[0].column.value, 'test')
self.assertEqual(res[1].column.value, 'test2')
res = yield self.client.get_slice('test2', CF, names=(COLUMN, COLUMN2))
self.assertEqual(res[0].column.value, 'test')
self.assertEqual(res[1].column.value, 'test2')
res = yield self.client.get_slice('test',
SCF,
names=(COLUMN, COLUMN2),
super_column=SCOLUMN)
self.assertEqual(res[0].column.value, 'test')
self.assertEqual(res[1].column.value, 'test2')
res = yield self.client.get_slice('test2',
SCF,
names=(COLUMN, COLUMN2),
super_column=SCOLUMN)
self.assertEqual(res[0].column.value, 'test')
self.assertEqual(res[1].column.value, 'test2')
yield self.client.batch_remove({CF: ['test', 'test2']},
names=['test', 'test2'])
yield self.client.batch_remove({SCF: ['test', 'test2']},
names=['test', 'test2'],
supercolumn=SCOLUMN)
@defer.inlineCallbacks
def test_batch_mutate_with_deletion(self):
yield self.client.batch_mutate(
{'test': {
CF: {
COLUMN: 'test',
COLUMN2: 'test2'
}
}})
res = yield self.client.get_slice('test', CF, names=(COLUMN, COLUMN2))
self.assertEqual(res[0].column.value, 'test')
self.assertEqual(res[1].column.value, 'test2')
yield self.client.batch_mutate(
{'test': {
CF: {
COLUMN: None,
COLUMN2: 'test3'
}
}})
res = yield self.client.get_slice('test', CF, names=(COLUMN, COLUMN2))
self.assertEqual(len(res), 1)
self.assertEqual(res[0].column.value, 'test3')
@defer.inlineCallbacks
def test_multiget_slice_remove(self):
yield self.client.insert('test', CF, 'testval', column=COLUMN)
yield self.client.insert('test', CF, 'testval', column=COLUMN2)
yield self.client.insert('test2', CF, 'testval2', column=COLUMN)
res = yield self.client.multiget(['test', 'test2'], CF, column=COLUMN)
self.assertEqual(res['test'][0].column.value, 'testval')
self.assertEqual(res['test2'][0].column.value, 'testval2')
res = yield self.client.multiget_slice(['test', 'test2'], CF)
self.assertEqual(res['test'][0].column.value, 'testval')
self.assertEqual(res['test'][1].column.value, 'testval')
self.assertEqual(res['test2'][0].column.value, 'testval2')
yield self.client.remove('test', CF, column=COLUMN)
yield self.client.remove('test2', CF, column=COLUMN)
res = yield self.client.multiget(['test', 'test2'], CF, column=COLUMN)
self.assertEqual(len(res['test']), 0)
self.assertEqual(len(res['test2']), 0)
@defer.inlineCallbacks
def test_range_slices(self):
yield self.client.insert('test', CF, 'testval', column=COLUMN)
yield self.client.insert('test', CF, 'testval', column=COLUMN2)
yield self.client.insert('test2', CF, 'testval2', column=COLUMN)
ks = yield self.client.get_range_slices(CF, start='', finish='')
keys = [k.key for k in ks]
for key in ['test', 'test2']:
self.assertIn(key, keys)
@defer.inlineCallbacks
def test_indexed_slices(self):
yield self.client.insert('test1', IDX_CF, 'one', column='col1')
yield self.client.insert('test2', IDX_CF, 'two', column='col1')
yield self.client.insert('test3', IDX_CF, 'three', column='col1')
expressions = [IndexExpression('col1', IndexOperator.EQ, 'two')]
res = yield self.client.get_indexed_slices(IDX_CF,
expressions,
start_key='')
self.assertEquals(res[0].columns[0].column.value, 'two')
@defer.inlineCallbacks
def test_counter_add(self):
if self.version != CASSANDRA_08_VERSION:
raise unittest.SkipTest('Counters are not supported in 0.7')
# test standard column counter
yield self.client.add('test', COUNTER_CF, 1, column='col')
res = yield self.client.get('test', COUNTER_CF, column='col')
self.assertEquals(res.counter_column.value, 1)
yield self.client.add('test', COUNTER_CF, 1, column='col')
res = yield self.client.get('test', COUNTER_CF, column='col')
self.assertEquals(res.counter_column.value, 2)
# test super column counters
yield self.client.add('test',
SUPERCOUNTER_CF,
1,
column='col',
super_column='scol')
res = yield self.client.get('test',
SUPERCOUNTER_CF,
column='col',
super_column='scol')
self.assertEquals(res.counter_column.value, 1)
yield self.client.add('test',
SUPERCOUNTER_CF,
1,
column='col',
super_column='scol')
res = yield self.client.get('test',
SUPERCOUNTER_CF,
column='col',
super_column='scol')
self.assertEquals(res.counter_column.value, 2)
@defer.inlineCallbacks
def test_counter_remove(self):
if self.version != CASSANDRA_08_VERSION:
raise unittest.SkipTest('Counters are not supported in 0.7')
# test standard column counter
yield self.client.add('test', COUNTER_CF, 1, column='col')
res = yield self.client.get('test', COUNTER_CF, column='col')
self.assertEquals(res.counter_column.value, 1)
yield self.client.remove_counter('test', COUNTER_CF, column='col')
yield self.assertFailure(
self.client.get('test', COUNTER_CF, column='col'),
NotFoundException)
# test super column counters
yield self.client.add('test',
SUPERCOUNTER_CF,
1,
column='col',
super_column='scol')
res = yield self.client.get('test',
SUPERCOUNTER_CF,
column='col',
super_column='scol')
self.assertEquals(res.counter_column.value, 1)
yield self.client.remove_counter('test',
SUPERCOUNTER_CF,
column='col',
super_column='scol')
yield self.assertFailure(
self.client.get('test',
SUPERCOUNTER_CF,
column='col',
super_column='scol'), NotFoundException)
def sleep(self, secs):
d = defer.Deferred()
reactor.callLater(secs, d.callback, None)
return d
@defer.inlineCallbacks
def test_ttls(self):
yield self.client.insert('test_ttls',
CF,
'testval',
column=COLUMN,
ttl=1)
res = yield self.client.get('test_ttls', CF, column=COLUMN)
self.assertEqual(res.column.value, 'testval')
yield self.sleep(2)
yield self.assertFailure(
self.client.get('test_ttls', CF, column=COLUMN), NotFoundException)
yield self.client.batch_insert('test_ttls',
CF, {COLUMN: 'testval'},
ttl=1)
res = yield self.client.get('test_ttls', CF, column=COLUMN)
self.assertEqual(res.column.value, 'testval')
yield self.sleep(2)
yield self.assertFailure(
self.client.get('test_ttls', CF, column=COLUMN), NotFoundException)
yield self.client.batch_mutate(
{'test_ttls': {
CF: {
COLUMN: 'testval'
}
}}, ttl=1)
res = yield self.client.get('test_ttls', CF, column=COLUMN)
self.assertEqual(res.column.value, 'testval')
yield self.sleep(2)
yield self.assertFailure(
self.client.get('test_ttls', CF, column=COLUMN), NotFoundException)
def compare_keyspaces(self, ks1, ks2):
self.assertEqual(ks1.name, ks2.name)
self.assertEqual(ks1.strategy_class, ks2.strategy_class)
self.assertEqual(ks1.cf_defs, ks2.cf_defs)
def get_rf(ksdef):
rf = ksdef.replication_factor
if ksdef.strategy_options and \
'replication_factor' in ksdef.strategy_options:
rf = int(ksdef.strategy_options['replication_factor'])
return rf
def strat_opts_no_rf(ksdef):
if not ksdef.strategy_options:
return {}
opts = ksdef.strategy_options.copy()
if 'replication_factor' in ksdef.strategy_options:
del opts['replication_factor']
return opts
self.assertEqual(get_rf(ks1), get_rf(ks2))
self.assertEqual(strat_opts_no_rf(ks1), strat_opts_no_rf(ks2))
@defer.inlineCallbacks
def test_keyspace_manipulation(self):
try:
yield self.client.system_drop_keyspace(T_KEYSPACE)
except InvalidRequestException:
pass
ksdef = KsDef(
name=T_KEYSPACE,
strategy_class='org.apache.cassandra.locator.SimpleStrategy',
strategy_options={'replication_factor': '1'},
cf_defs=[])
yield self.client.system_add_keyspace(ksdef)
ks2 = yield self.client.describe_keyspace(T_KEYSPACE)
self.compare_keyspaces(ksdef, ks2)
if DO_SYSTEM_RENAMING:
newname = T_KEYSPACE + '2'
yield self.client.system_rename_keyspace(T_KEYSPACE, newname)
ks2 = yield self.client.describe_keyspace(newname)
ksdef.name = newname
self.compare_keyspaces(ksdef, ks2)
yield self.client.system_drop_keyspace(ksdef.name)
yield self.assertFailure(self.client.describe_keyspace(T_KEYSPACE),
NotFoundException)
if DO_SYSTEM_RENAMING:
yield self.assertFailure(self.client.describe_keyspace(ksdef.name),
NotFoundException)
@defer.inlineCallbacks
def test_column_family_manipulation(self):
cfdef = CfDef(
KEYSPACE,
T_CF,
column_type='Standard',
comparator_type='org.apache.cassandra.db.marshal.BytesType',
comment='foo',
row_cache_size=0.0,
key_cache_size=200000.0,
read_repair_chance=1.0,
column_metadata=[],
gc_grace_seconds=86400,
default_validation_class=
'org.apache.cassandra.db.marshal.BytesType',
key_validation_class='org.apache.cassandra.db.marshal.BytesType',
min_compaction_threshold=5,
max_compaction_threshold=31,
row_cache_save_period_in_seconds=0,
key_cache_save_period_in_seconds=3600,
memtable_flush_after_mins=60,
memtable_throughput_in_mb=249,
memtable_operations_in_millions=1.1671875,
replicate_on_write=False,
merge_shards_chance=0.10000000000000001,
row_cache_provider=None,
key_alias=None,
)
post_07_fields = [
'replicate_on_write', 'merge_shards_chance',
'key_validation_class', 'row_cache_provider', 'key_alias'
]
post_08_fields = [
'memtable_throughput_in_mb', 'memtable_flush_after_mins',
'memtable_operations_in_millions'
]
yield self.client.system_add_column_family(cfdef)
ksdef = yield self.client.describe_keyspace(KEYSPACE)
cfdef2 = [c for c in ksdef.cf_defs if c.name == T_CF][0]
for field in post_07_fields:
# Most of these are ignored in 0.7, so we can't reliably compare them
setattr(cfdef, field, None)
setattr(cfdef2, field, None)
for field in post_08_fields:
# These fields change from 0.8 to 1.0
setattr(cfdef, field, None)
setattr(cfdef2, field, None)
# we don't know the id ahead of time. copy the new one so the equality
# comparison won't fail
cfdef.id = cfdef2.id
self.assertEqual(cfdef, cfdef2)
if DO_SYSTEM_RENAMING:
newname = T_CF + '2'
yield self.client.system_rename_column_family(T_CF, newname)
ksdef = yield self.client.describe_keyspace(KEYSPACE)
cfdef2 = [c for c in ksdef.cf_defs if c.name == newname][0]
self.assertNotIn(T_CF, [c.name for c in ksdef.cf_defs])
cfdef.name = newname
self.assertEqual(cfdef, cfdef2)
yield self.client.system_drop_column_family(cfdef.name)
ksdef = yield self.client.describe_keyspace(KEYSPACE)
self.assertNotIn(cfdef.name, [c.name for c in ksdef.cf_defs])
@defer.inlineCallbacks
def test_describes(self):
name = yield self.client.describe_cluster_name()
self.assertIsInstance(name, str)
self.assertNotEqual(name, '')
partitioner = yield self.client.describe_partitioner()
self.assert_(partitioner.startswith('org.apache.cassandra.'),
msg='partitioner is %r' % partitioner)
snitch = yield self.client.describe_snitch()
self.assert_(snitch.startswith('org.apache.cassandra.'),
msg='snitch is %r' % snitch)
version = yield self.client.describe_version()
self.assertIsInstance(version, str)
self.assertIn('.', version)
schemavers = yield self.client.describe_schema_versions()
self.assertIsInstance(schemavers, dict)
self.assertNotEqual(schemavers, {})
ring = yield self.client.describe_ring(KEYSPACE)
self.assertIsInstance(ring, list)
self.assertNotEqual(ring, [])
for r in ring:
self.assertIsInstance(r.start_token, str)
self.assertIsInstance(r.end_token, str)
self.assertIsInstance(r.endpoints, list)
self.assertNotEqual(r.endpoints, [])
for ep in r.endpoints:
self.assertIsInstance(ep, str)
@defer.inlineCallbacks
def test_errback(self):
yield self.client.remove('poiqwe', CF)
try:
yield self.client.get('poiqwe', CF, column='foo')
except Exception, e:
pass
class CassandraProvisionerStore(object):
"""
Provides high level provisioner storage operations for Cassandra
"""
# default size of paged fetches
_PAGE_SIZE = 100
LAUNCH_CF_NAME = "ProvisionerLaunches"
NODE_CF_NAME = "ProvisionerNodes"
@classmethod
def get_column_families(cls, keyspace=None, prefix=''):
"""Builds a list of column families needed by this store.
@param keyspace Name of keyspace. If None, it must be added manually.
@param prefix Optional prefix for cf names. Useful for testing.
@retval list of CfDef objects
"""
launch_cf_name = prefix + cls.LAUNCH_CF_NAME
node_cf_name = prefix + cls.NODE_CF_NAME
return [
CfDef(keyspace,
launch_cf_name,
comparator_type='org.apache.cassandra.db.marshal.UTF8Type'),
CfDef(keyspace,
node_cf_name,
comparator_type='org.apache.cassandra.db.marshal.UTF8Type')
]
def __init__(self, host, port, username, password, keyspace, prefix=''):
self._launch_column_family = prefix + self.LAUNCH_CF_NAME
self._node_column_family = prefix + self.NODE_CF_NAME
authz = {'username': username, 'password': password}
self._manager = ManagedCassandraClientFactory(credentials=authz,
check_api_version=True,
keyspace=keyspace)
self.client = CassandraClient(self._manager)
self._host = host
self._port = port
self._connector = None
def connect(self):
self._connector = reactor.connectTCP(self._host, self._port,
self._manager)
def disconnect(self):
self._manager.shutdown()
if self._connector:
self._connector.disconnect()
self._connector = None
@timeout(CASSANDRA_TIMEOUT)
def put_launch(self, launch):
"""
@brief Stores a single launch record
@param launch Launch record to store
@retval Deferred for success
"""
launch_id = launch['launch_id']
state = launch['state']
value = json.dumps(launch)
return self.client.insert(launch_id,
self._launch_column_family,
value,
column=state)
@timeout(CASSANDRA_TIMEOUT)
@defer.inlineCallbacks
def put_nodes(self, nodes):
"""
@brief Stores a set of node records
@param nodes Iterable of node records
@retval Deferred for success
"""
# could be more efficient with a batch_mutate
for node in nodes:
yield self.put_node(node)
@timeout(CASSANDRA_TIMEOUT)
def put_node(self, node):
"""
@brief Stores a node record
@param node Node record
@retval Deferred for success
"""
node_id = node['node_id']
state = node['state']
value = json.dumps(node)
return self.client.insert(node_id,
self._node_column_family,
value,
column=state)
@timeout(CASSANDRA_TIMEOUT)
def get_launch(self, launch_id, count=1):
"""
@brief Retrieves a launch record by id
@param launch_id Id of launch record to retrieve
@param count Number of launch state records to retrieve
@retval Deferred record(s), or None. A list of records if count > 1
"""
return self._get_record(launch_id, self._launch_column_family, count)
@timeout(CASSANDRA_TIMEOUT)
def get_launches(self, state=None, min_state=None, max_state=None):
"""
@brief Retrieves the latest record for all launches within a state range
@param state Only retrieve nodes in this state.
@param min_state Inclusive start bound
@param max_state Inclusive end bound
@retval Deferred list of launch records
"""
return self._get_records(self._launch_column_family,
state=state,
min_state=min_state,
max_state=max_state)
@timeout(CASSANDRA_TIMEOUT)
def get_node(self, node_id, count=1):
"""
@brief Retrieves a launch record by id
@param node_id Id of node record to retrieve
@param count Number of node state records to retrieve
@retval Deferred record(s), or None. A list of records if count > 1
"""
return self._get_record(node_id, self._node_column_family, count)
@timeout(CASSANDRA_TIMEOUT)
def get_nodes(self, state=None, min_state=None, max_state=None):
"""
@brief Retrieves all launch record within a state range
@param state Only retrieve nodes in this state.
@param min_state Inclusive start bound.
@param max_state Inclusive end bound
@retval Deferred list of launch records
"""
return self._get_records(self._node_column_family,
state=state,
min_state=min_state,
max_state=max_state)
@defer.inlineCallbacks
def _get_record(self, key, column_family, count):
slice = yield self.client.get_slice(key,
column_family,
reverse=True,
count=count)
# we're probably only interested in the last record, in sorted order.
# This is the latest state the object has recorded.
records = [json.loads(column.column.value) for column in slice]
if count == 1:
if records:
ret = records[0]
else:
ret = None
else:
ret = records
defer.returnValue(ret)
@defer.inlineCallbacks
def _get_records(self,
column_family,
state=None,
min_state=None,
max_state=None,
reverse=True):
# overrides range arguments
if state:
min_state = max_state = state
start = ''
end = min_state or ''
if not reverse:
start, end = end, start
# this is tricky. We are only concerned with the latest state record
# (by sort order not necessarily time). So when we look for records
# within a state range, we effectively must pull down the latest state
# for each record, and filter them locally. This is slightly improved
# when a first_state (or last when reverse=False) is specified as the
# server can skip any records not >= that state.
records = []
done = False
start_key = ''
iterations = 0
while not done:
slices = yield self.client.get_range_slices(column_family,
column_start=start,
column_finish=end,
reverse=reverse,
column_count=1,
start=start_key,
count=self._PAGE_SIZE)
skipped_one = False
for slice in slices:
if not skipped_one and iterations:
# if this not the first batch, skip the first element as it
# will be a dupe.
skipped_one = True
continue
if not slice.columns:
# rows without matching columns will still be returned
continue
record = json.loads(slice.columns[0].column.value)
if not max_state or record['state'] <= max_state:
if not min_state or record['state'] >= min_state:
records.append(record)
# page through results. by default only 100 are returned at a time
if len(slices) == self._PAGE_SIZE:
start_key = slices[-1].key
else:
done = True
iterations += 1
defer.returnValue(records)
def on_deactivate(self, *args, **kwargs):
self._manager.shutdown()
log.info('on_deactivate: Lose Connection TCP')
def on_terminate(self, *args, **kwargs):
self._manager.shutdown()
log.info('on_terminate: Lose Connection TCP')
class CassandraClientTest(unittest.TestCase):
@defer.inlineCallbacks
def setUp(self):
self.cmanager = ManagedCassandraClientFactory(keyspace='system')
self.client = CassandraClient(self.cmanager)
for i in xrange(CONNS):
reactor.connectTCP(HOST, PORT, self.cmanager)
yield self.cmanager.deferred
self.my_keyspace = KsDef(
name=KEYSPACE,
strategy_class='org.apache.cassandra.locator.SimpleStrategy',
replication_factor=1,
cf_defs=[
CfDef(
keyspace=KEYSPACE,
name=CF,
column_type='Standard'
),
CfDef(
keyspace=KEYSPACE,
name=SCF,
column_type='Super'
),
CfDef(
keyspace=KEYSPACE,
name=IDX_CF,
column_type='Standard',
comparator_type='org.apache.cassandra.db.marshal.UTF8Type',
column_metadata=[
ColumnDef(
name='col1',
validation_class='org.apache.cassandra.db.marshal.UTF8Type',
index_type=IndexType.KEYS,
index_name='idxCol1')
],
default_validation_class='org.apache.cassandra.db.marshal.BytesType')
]
)
yield self.client.system_add_keyspace(self.my_keyspace)
yield self.client.set_keyspace(KEYSPACE)
@defer.inlineCallbacks
def tearDown(self):
yield self.client.system_drop_keyspace(self.my_keyspace.name)
self.cmanager.shutdown()
for c in reactor.getDelayedCalls():
c.cancel()
reactor.removeAll()
@defer.inlineCallbacks
def test_insert_get(self):
yield self.client.insert('test', CF, 'testval', column=COLUMN)
yield self.client.insert('test2', CF, 'testval2', column=COLUMN)
yield self.client.insert('test', SCF, 'superval', column=COLUMN, super_column=SCOLUMN)
yield self.client.insert('test2', SCF, 'superval2', column=COLUMN,
super_column=SCOLUMN)
res = yield self.client.get('test', CF, column=COLUMN)
self.assertEqual(res.column.value, 'testval')
res = yield self.client.get('test2', CF, column=COLUMN)
self.assertEqual(res.column.value, 'testval2')
res = yield self.client.get('test', SCF, column=COLUMN, super_column=SCOLUMN)
self.assertEqual(res.column.value, 'superval')
res = yield self.client.get('test2', SCF, column=COLUMN, super_column=SCOLUMN)
self.assertEqual(res.column.value, 'superval2')
@defer.inlineCallbacks
def test_batch_insert_get_slice_and_count(self):
yield self.client.batch_insert('test', CF,
{COLUMN: 'test', COLUMN2: 'test2'})
yield self.client.batch_insert('test', SCF,
{SCOLUMN: {COLUMN: 'test', COLUMN2: 'test2'}})
res = yield self.client.get_slice('test', CF, names=(COLUMN, COLUMN2))
self.assertEqual(res[0].column.value, 'test')
self.assertEqual(res[1].column.value, 'test2')
res = yield self.client.get_slice('test', SCF, names=(COLUMN, COLUMN2),
super_column=SCOLUMN)
self.assertEqual(res[0].column.value, 'test')
self.assertEqual(res[1].column.value, 'test2')
res = yield self.client.get_count('test', CF)
self.assertEqual(res, 2)
@defer.inlineCallbacks
def test_batch_mutate_and_remove(self):
yield self.client.batch_mutate({'test': {CF: {COLUMN: 'test', COLUMN2: 'test2'}, SCF: { SCOLUMN: { COLUMN: 'test', COLUMN2: 'test2'} } }, 'test2': {CF: {COLUMN: 'test', COLUMN2: 'test2'}, SCF: { SCOLUMN: { COLUMN: 'test', COLUMN2: 'test2'} } } })
res = yield self.client.get_slice('test', CF, names=(COLUMN, COLUMN2))
self.assertEqual(res[0].column.value, 'test')
self.assertEqual(res[1].column.value, 'test2')
res = yield self.client.get_slice('test2', CF, names=(COLUMN, COLUMN2))
self.assertEqual(res[0].column.value, 'test')
self.assertEqual(res[1].column.value, 'test2')
res = yield self.client.get_slice('test', SCF, names=(COLUMN, COLUMN2),
super_column=SCOLUMN)
self.assertEqual(res[0].column.value, 'test')
self.assertEqual(res[1].column.value, 'test2')
res = yield self.client.get_slice('test2', SCF, names=(COLUMN, COLUMN2),
super_column=SCOLUMN)
self.assertEqual(res[0].column.value, 'test')
self.assertEqual(res[1].column.value, 'test2')
yield self.client.batch_remove({CF: ['test', 'test2']}, names=['test', 'test2'])
yield self.client.batch_remove({SCF: ['test', 'test2']}, names=['test', 'test2'], supercolumn=SCOLUMN)
@defer.inlineCallbacks
def test_batch_mutate_with_deletion(self):
yield self.client.batch_mutate({'test': {CF: {COLUMN: 'test', COLUMN2: 'test2'}}})
res = yield self.client.get_slice('test', CF, names=(COLUMN, COLUMN2))
self.assertEqual(res[0].column.value, 'test')
self.assertEqual(res[1].column.value, 'test2')
yield self.client.batch_mutate({'test': {CF: {COLUMN: None, COLUMN2: 'test3'}}})
res = yield self.client.get_slice('test', CF, names=(COLUMN, COLUMN2))
self.assertEqual(len(res), 1)
self.assertEqual(res[0].column.value, 'test3')
@defer.inlineCallbacks
def test_multiget_slice_remove(self):
yield self.client.insert('test', CF, 'testval', column=COLUMN)
yield self.client.insert('test', CF, 'testval', column=COLUMN2)
yield self.client.insert('test2', CF, 'testval2', column=COLUMN)
res = yield self.client.multiget(['test', 'test2'], CF, column=COLUMN)
self.assertEqual(res['test'][0].column.value, 'testval')
self.assertEqual(res['test2'][0].column.value, 'testval2')
res = yield self.client.multiget_slice(['test', 'test2'], CF)
self.assertEqual(res['test'][0].column.value, 'testval')
self.assertEqual(res['test'][1].column.value, 'testval')
self.assertEqual(res['test2'][0].column.value, 'testval2')
yield self.client.remove('test', CF, column=COLUMN)
yield self.client.remove('test2', CF, column=COLUMN)
res = yield self.client.multiget(['test', 'test2'], CF, column=COLUMN)
self.assertEqual(len(res['test']), 0)
self.assertEqual(len(res['test2']), 0)
@defer.inlineCallbacks
def test_range_slices(self):
yield self.client.insert('test', CF, 'testval', column=COLUMN)
yield self.client.insert('test', CF, 'testval', column=COLUMN2)
yield self.client.insert('test2', CF, 'testval2', column=COLUMN)
ks = yield self.client.get_range_slices(CF, start='', finish='')
keys = [k.key for k in ks]
for key in ['test', 'test2']:
self.assertIn(key, keys)
@defer.inlineCallbacks
def test_indexed_slices(self):
yield self.client.insert('test1', IDX_CF, 'one', column='col1')
yield self.client.insert('test2', IDX_CF, 'two', column='col1')
yield self.client.insert('test3', IDX_CF, 'three', column='col1')
expressions = [IndexExpression('col1', IndexOperator.EQ, 'two')]
res = yield self.client.get_indexed_slices(IDX_CF, expressions, start_key='')
self.assertEquals(res[0].columns[0].column.value,'two')
def sleep(self, secs):
d = defer.Deferred()
reactor.callLater(secs, d.callback, None)
return d
@defer.inlineCallbacks
def test_ttls(self):
yield self.client.insert('test_ttls', CF, 'testval', column=COLUMN, ttl=1)
res = yield self.client.get('test_ttls', CF, column=COLUMN)
self.assertEqual(res.column.value, 'testval')
yield self.sleep(2)
yield self.assertFailure(self.client.get('test_ttls', CF, column=COLUMN), NotFoundException)
yield self.client.batch_insert('test_ttls', CF, {COLUMN:'testval'}, ttl=1)
res = yield self.client.get('test_ttls', CF, column=COLUMN)
self.assertEqual(res.column.value, 'testval')
yield self.sleep(2)
yield self.assertFailure(self.client.get('test_ttls', CF, column=COLUMN), NotFoundException)
yield self.client.batch_mutate({'test_ttls': {CF: {COLUMN: 'testval'}}}, ttl=1)
res = yield self.client.get('test_ttls', CF, column=COLUMN)
self.assertEqual(res.column.value, 'testval')
yield self.sleep(2)
yield self.assertFailure(self.client.get('test_ttls', CF, column=COLUMN), NotFoundException)
@defer.inlineCallbacks
def test_keyspace_manipulation(self):
ksdef = KsDef(name=T_KEYSPACE, strategy_class='org.apache.cassandra.locator.SimpleStrategy', replication_factor=1, cf_defs=[])
yield self.client.system_add_keyspace(ksdef)
ks2 = yield self.client.describe_keyspace(T_KEYSPACE)
self.assertEqual(ksdef, ks2)
if DO_SYSTEM_RENAMING:
newname = T_KEYSPACE + '2'
yield self.client.system_rename_keyspace(T_KEYSPACE, newname)
ks2 = yield self.client.describe_keyspace(newname)
ksdef.name = newname
self.assertEqual(ksdef, ks2)
yield self.client.system_drop_keyspace(ksdef.name)
yield self.assertFailure(self.client.describe_keyspace(T_KEYSPACE), NotFoundException)
if DO_SYSTEM_RENAMING:
yield self.assertFailure(self.client.describe_keyspace(ksdef.name), NotFoundException)
@defer.inlineCallbacks
def test_column_family_manipulation(self):
cfdef = CfDef(KEYSPACE, T_CF,
column_type='Standard',
comparator_type='org.apache.cassandra.db.marshal.BytesType',
comment='foo',
row_cache_size=0.0,
key_cache_size=200000.0,
read_repair_chance=1.0,
column_metadata=[],
gc_grace_seconds=86400,
default_validation_class='org.apache.cassandra.db.marshal.BytesType',
min_compaction_threshold=5,
max_compaction_threshold=31,
row_cache_save_period_in_seconds=0,
key_cache_save_period_in_seconds=3600,
memtable_flush_after_mins=60,
memtable_throughput_in_mb=249,
memtable_operations_in_millions=1.1671875,
)
yield self.client.system_add_column_family(cfdef)
ksdef = yield self.client.describe_keyspace(KEYSPACE)
cfdef2 = [c for c in ksdef.cf_defs if c.name == T_CF][0]
# we don't know the id ahead of time. copy the new one so the equality
# comparison won't fail
cfdef.id = cfdef2.id
self.assertEqual(cfdef, cfdef2)
if DO_SYSTEM_RENAMING:
newname = T_CF + '2'
yield self.client.system_rename_column_family(T_CF, newname)
ksdef = yield self.client.describe_keyspace(KEYSPACE)
cfdef2 = [c for c in ksdef.cf_defs if c.name == newname][0]
self.assertNotIn(T_CF, [c.name for c in ksdef.cf_defs])
cfdef.name = newname
self.assertEqual(cfdef, cfdef2)
yield self.client.system_drop_column_family(cfdef.name)
ksdef = yield self.client.describe_keyspace(KEYSPACE)
self.assertNotIn(cfdef.name, [c.name for c in ksdef.cf_defs])
@defer.inlineCallbacks
def test_describes(self):
name = yield self.client.describe_cluster_name()
self.assertIsInstance(name, str)
self.assertNotEqual(name, '')
partitioner = yield self.client.describe_partitioner()
self.assert_(partitioner.startswith('org.apache.cassandra.'),
msg='partitioner is %r' % partitioner)
snitch = yield self.client.describe_snitch()
self.assert_(snitch.startswith('org.apache.cassandra.'),
msg='snitch is %r' % snitch)
version = yield self.client.describe_version()
self.assertIsInstance(version, str)
self.assertIn('.', version)
schemavers = yield self.client.describe_schema_versions()
self.assertIsInstance(schemavers, dict)
self.assertNotEqual(schemavers, {})
ring = yield self.client.describe_ring(KEYSPACE)
self.assertIsInstance(ring, list)
self.assertNotEqual(ring, [])
for r in ring:
self.assertIsInstance(r.start_token, str)
self.assertIsInstance(r.end_token, str)
self.assertIsInstance(r.endpoints, list)
self.assertNotEqual(r.endpoints, [])
for ep in r.endpoints:
self.assertIsInstance(ep, str)
@defer.inlineCallbacks
def test_errback(self):
yield self.client.remove('poiqwe', CF)
try:
yield self.client.get('poiqwe', CF, column='foo')
except Exception, e:
pass
class CassandraClientTest(unittest.TestCase):
@defer.inlineCallbacks
def setUp(self):
self.cmanager = ManagedCassandraClientFactory(keyspace='system')
self.client = CassandraClient(self.cmanager)
for i in xrange(CONNS):
reactor.connectTCP(HOST, PORT, self.cmanager)
yield self.cmanager.deferred
remote_ver = yield self.client.describe_version()
self.version = tuple(map(int, remote_ver.split('.')))
self.my_keyspace = ttypes.KsDef(
name=KEYSPACE,
strategy_class='org.apache.cassandra.locator.SimpleStrategy',
strategy_options={},
cf_defs=[
ttypes.CfDef(
keyspace=KEYSPACE,
name=CF,
column_type='Standard'
),
ttypes.CfDef(
keyspace=KEYSPACE,
name=SCF,
column_type='Super'
),
ttypes.CfDef(
keyspace=KEYSPACE,
name=IDX_CF,
column_type='Standard',
comparator_type='org.apache.cassandra.db.marshal.UTF8Type',
column_metadata=[
ttypes.ColumnDef(
name='col1',
validation_class='org.apache.cassandra.db.marshal.UTF8Type',
index_type=ttypes.IndexType.KEYS,
index_name='idxCol1')
],
default_validation_class='org.apache.cassandra.db.marshal.BytesType'
),
]
)
if self.version <= KS_RF_ATTRIBUTE:
self.my_keyspace.replication_factor = 1
else:
self.my_keyspace.strategy_options['replication_factor'] = '1'
if self.version >= COUNTERS_SUPPORTED_API:
self.my_keyspace.cf_defs.extend([
ttypes.CfDef(
keyspace=KEYSPACE,
name=COUNTER_CF,
column_type='Standard',
default_validation_class='org.apache.cassandra.db.marshal.CounterColumnType'
),
ttypes.CfDef(
keyspace=KEYSPACE,
name=SUPERCOUNTER_CF,
column_type='Super',
default_validation_class='org.apache.cassandra.db.marshal.CounterColumnType'
),
])
yield self.client.system_add_keyspace(self.my_keyspace)
yield self.client.set_keyspace(KEYSPACE)
@defer.inlineCallbacks
def tearDown(self):
yield self.client.system_drop_keyspace(self.my_keyspace.name)
self.cmanager.shutdown()
for c in reactor.getDelayedCalls():
c.cancel()
reactor.removeAll()
@defer.inlineCallbacks
def test_insert_get(self):
yield self.client.insert('test', CF, 'testval', column=COLUMN)
yield self.client.insert('test2', CF, 'testval2', column=COLUMN)
yield self.client.insert('test', SCF, 'superval', column=COLUMN, super_column=SCOLUMN)
yield self.client.insert('test2', SCF, 'superval2', column=COLUMN,
super_column=SCOLUMN)
res = yield self.client.get('test', CF, column=COLUMN)
self.assertEqual(res.column.value, 'testval')
res = yield self.client.get('test2', CF, column=COLUMN)
self.assertEqual(res.column.value, 'testval2')
res = yield self.client.get('test', SCF, column=COLUMN, super_column=SCOLUMN)
self.assertEqual(res.column.value, 'superval')
res = yield self.client.get('test2', SCF, column=COLUMN, super_column=SCOLUMN)
self.assertEqual(res.column.value, 'superval2')
@defer.inlineCallbacks
def test_batch_insert_get_slice_and_count(self):
yield self.client.batch_insert('test', CF,
{COLUMN: 'test', COLUMN2: 'test2'})
yield self.client.batch_insert('test', SCF,
{SCOLUMN: {COLUMN: 'test', COLUMN2: 'test2'}})
res = yield self.client.get_slice('test', CF, names=(COLUMN, COLUMN2))
self.assertEqual(res[0].column.value, 'test')
self.assertEqual(res[1].column.value, 'test2')
res = yield self.client.get_slice('test', SCF, names=(COLUMN, COLUMN2),
super_column=SCOLUMN)
self.assertEqual(res[0].column.value, 'test')
self.assertEqual(res[1].column.value, 'test2')
res = yield self.client.get_count('test', CF)
self.assertEqual(res, 2)
@defer.inlineCallbacks
def test_batch_mutate_and_remove(self):
yield self.client.batch_mutate({'test': {CF: {COLUMN: 'test', COLUMN2: 'test2'}, SCF: { SCOLUMN: { COLUMN: 'test', COLUMN2: 'test2'} } }, 'test2': {CF: {COLUMN: 'test', COLUMN2: 'test2'}, SCF: { SCOLUMN: { COLUMN: 'test', COLUMN2: 'test2'} } } })
res = yield self.client.get_slice('test', CF, names=(COLUMN, COLUMN2))
self.assertEqual(res[0].column.value, 'test')
self.assertEqual(res[1].column.value, 'test2')
res = yield self.client.get_slice('test2', CF, names=(COLUMN, COLUMN2))
self.assertEqual(res[0].column.value, 'test')
self.assertEqual(res[1].column.value, 'test2')
res = yield self.client.get_slice('test', SCF, names=(COLUMN, COLUMN2),
super_column=SCOLUMN)
self.assertEqual(res[0].column.value, 'test')
self.assertEqual(res[1].column.value, 'test2')
res = yield self.client.get_slice('test2', SCF, names=(COLUMN, COLUMN2),
super_column=SCOLUMN)
self.assertEqual(res[0].column.value, 'test')
self.assertEqual(res[1].column.value, 'test2')
yield self.client.batch_remove({CF: ['test', 'test2']}, names=['test', 'test2'])
yield self.client.batch_remove({SCF: ['test', 'test2']}, names=['test', 'test2'], supercolumn=SCOLUMN)
@defer.inlineCallbacks
def test_batch_mutate_with_deletion(self):
yield self.client.batch_mutate({'test': {CF: {COLUMN: 'test', COLUMN2: 'test2'}}})
res = yield self.client.get_slice('test', CF, names=(COLUMN, COLUMN2))
self.assertEqual(res[0].column.value, 'test')
self.assertEqual(res[1].column.value, 'test2')
yield self.client.batch_mutate({'test': {CF: {COLUMN: None, COLUMN2: 'test3'}}})
res = yield self.client.get_slice('test', CF, names=(COLUMN, COLUMN2))
self.assertEqual(len(res), 1)
self.assertEqual(res[0].column.value, 'test3')
@defer.inlineCallbacks
def test_multiget_slice_remove(self):
yield self.client.insert('test', CF, 'testval', column=COLUMN)
yield self.client.insert('test', CF, 'testval', column=COLUMN2)
yield self.client.insert('test2', CF, 'testval2', column=COLUMN)
res = yield self.client.multiget(['test', 'test2'], CF, column=COLUMN)
self.assertEqual(res['test'][0].column.value, 'testval')
self.assertEqual(res['test2'][0].column.value, 'testval2')
res = yield self.client.multiget_slice(['test', 'test2'], CF)
self.assertEqual(res['test'][0].column.value, 'testval')
self.assertEqual(res['test'][1].column.value, 'testval')
self.assertEqual(res['test2'][0].column.value, 'testval2')
yield self.client.remove('test', CF, column=COLUMN)
yield self.client.remove('test2', CF, column=COLUMN)
res = yield self.client.multiget(['test', 'test2'], CF, column=COLUMN)
self.assertEqual(len(res['test']), 0)
self.assertEqual(len(res['test2']), 0)
@defer.inlineCallbacks
def test_range_slices(self):
yield self.client.insert('test', CF, 'testval', column=COLUMN)
yield self.client.insert('test', CF, 'testval', column=COLUMN2)
yield self.client.insert('test2', CF, 'testval2', column=COLUMN)
ks = yield self.client.get_range_slices(CF, start='', finish='')
keys = [k.key for k in ks]
for key in ['test', 'test2']:
self.assertIn(key, keys)
@defer.inlineCallbacks
def test_indexed_slices(self):
yield self.client.insert('test1', IDX_CF, 'one', column='col1')
yield self.client.insert('test2', IDX_CF, 'two', column='col1')
yield self.client.insert('test3', IDX_CF, 'three', column='col1')
expressions = [ttypes.IndexExpression('col1', ttypes.IndexOperator.EQ, 'two')]
res = yield self.client.get_indexed_slices(IDX_CF, expressions, start_key='')
self.assertEquals(res[0].columns[0].column.value,'two')
@defer.inlineCallbacks
def test_counter_add(self):
if self.version < COUNTERS_SUPPORTED_API:
raise unittest.SkipTest('Counters are not supported before 0.8')
# test standard column counter
yield self.client.add('test', COUNTER_CF, 1, column='col')
res = yield self.client.get('test', COUNTER_CF, column='col')
self.assertEquals(res.counter_column.value, 1)
yield self.client.add('test', COUNTER_CF, 1, column='col')
res = yield self.client.get('test', COUNTER_CF, column='col')
self.assertEquals(res.counter_column.value, 2)
# test super column counters
yield self.client.add('test', SUPERCOUNTER_CF, 1, column='col', super_column='scol')
res = yield self.client.get('test', SUPERCOUNTER_CF, column='col', super_column='scol')
self.assertEquals(res.counter_column.value, 1)
yield self.client.add('test', SUPERCOUNTER_CF, 1, column='col', super_column='scol')
res = yield self.client.get('test', SUPERCOUNTER_CF, column='col', super_column='scol')
self.assertEquals(res.counter_column.value, 2)
@defer.inlineCallbacks
def test_counter_remove(self):
if self.version < COUNTERS_SUPPORTED_API:
raise unittest.SkipTest('Counters are not supported before 0.8')
# test standard column counter
yield self.client.add('test', COUNTER_CF, 1, column='col')
res = yield self.client.get('test', COUNTER_CF, column='col')
self.assertEquals(res.counter_column.value, 1)
yield self.client.remove_counter('test', COUNTER_CF, column='col')
yield self.assertFailure(self.client.get('test', COUNTER_CF, column='col'),
ttypes.NotFoundException)
# test super column counters
yield self.client.add('test', SUPERCOUNTER_CF, 1, column='col', super_column='scol')
res = yield self.client.get('test', SUPERCOUNTER_CF, column='col', super_column='scol')
self.assertEquals(res.counter_column.value, 1)
yield self.client.remove_counter('test', SUPERCOUNTER_CF,
column='col', super_column='scol')
yield self.assertFailure(self.client.get('test', SUPERCOUNTER_CF,
column='col', super_column='scol'),
ttypes.NotFoundException)
def sleep(self, secs):
d = defer.Deferred()
reactor.callLater(secs, d.callback, None)
return d
@defer.inlineCallbacks
def test_ttls(self):
yield self.client.insert('test_ttls', CF, 'testval', column=COLUMN, ttl=1)
res = yield self.client.get('test_ttls', CF, column=COLUMN)
self.assertEqual(res.column.value, 'testval')
yield self.sleep(2)
yield self.assertFailure(self.client.get('test_ttls', CF, column=COLUMN), ttypes.NotFoundException)
yield self.client.batch_insert('test_ttls', CF, {COLUMN:'testval'}, ttl=1)
res = yield self.client.get('test_ttls', CF, column=COLUMN)
self.assertEqual(res.column.value, 'testval')
yield self.sleep(2)
yield self.assertFailure(self.client.get('test_ttls', CF, column=COLUMN), ttypes.NotFoundException)
yield self.client.batch_mutate({'test_ttls': {CF: {COLUMN: 'testval'}}}, ttl=1)
res = yield self.client.get('test_ttls', CF, column=COLUMN)
self.assertEqual(res.column.value, 'testval')
yield self.sleep(2)
yield self.assertFailure(self.client.get('test_ttls', CF, column=COLUMN), ttypes.NotFoundException)
def compare_keyspaces(self, ks1, ks2):
self.assertEqual(ks1.name, ks2.name)
self.assertEqual(ks1.strategy_class, ks2.strategy_class)
self.assertEqual(ks1.cf_defs, ks2.cf_defs)
def get_rf(ksdef):
rf = ksdef.replication_factor
if ksdef.strategy_options and \
'replication_factor' in ksdef.strategy_options:
rf = int(ksdef.strategy_options['replication_factor'])
return rf
def strat_opts_no_rf(ksdef):
if not ksdef.strategy_options:
return {}
opts = ksdef.strategy_options.copy()
if 'replication_factor' in ksdef.strategy_options:
del opts['replication_factor']
return opts
self.assertEqual(get_rf(ks1), get_rf(ks2))
self.assertEqual(strat_opts_no_rf(ks1), strat_opts_no_rf(ks2))
@defer.inlineCallbacks
def test_keyspace_manipulation(self):
try:
yield self.client.system_drop_keyspace(T_KEYSPACE)
except ttypes.InvalidRequestException:
pass
ksdef = ttypes.KsDef(name=T_KEYSPACE, strategy_class='org.apache.cassandra.locator.SimpleStrategy', strategy_options={}, cf_defs=[])
if self.version <= KS_RF_ATTRIBUTE:
ksdef.replication_factor = 1
else:
ksdef.strategy_options['replication_factor'] = '1'
yield self.client.system_add_keyspace(ksdef)
ks2 = yield self.client.describe_keyspace(T_KEYSPACE)
self.compare_keyspaces(ksdef, ks2)
if DO_SYSTEM_RENAMING:
newname = T_KEYSPACE + '2'
yield self.client.system_rename_keyspace(T_KEYSPACE, newname)
ks2 = yield self.client.describe_keyspace(newname)
ksdef.name = newname
self.compare_keyspaces(ksdef, ks2)
yield self.client.system_drop_keyspace(ksdef.name)
yield self.assertFailure(self.client.describe_keyspace(T_KEYSPACE), ttypes.NotFoundException)
if DO_SYSTEM_RENAMING:
yield self.assertFailure(self.client.describe_keyspace(ksdef.name), ttypes.NotFoundException)
@defer.inlineCallbacks
def test_column_family_manipulation(self):
# CfDef attributes present in all supported c*/thrift-api versions
common_attrs = (
('column_type', 'Standard'),
('comparator_type', 'org.apache.cassandra.db.marshal.BytesType'),
('comment', 'foo'),
('read_repair_chance', 1.0),
('column_metadata', []),
('gc_grace_seconds', 86400),
('default_validation_class', 'org.apache.cassandra.db.marshal.BytesType'),
('min_compaction_threshold', 5),
('max_compaction_threshold', 31),
)
cfdef = ttypes.CfDef(KEYSPACE, T_CF)
for attr, val in common_attrs:
setattr(cfdef, attr, val)
yield self.client.system_add_column_family(cfdef)
ksdef = yield self.client.describe_keyspace(KEYSPACE)
cfdefs = [c for c in ksdef.cf_defs if c.name == T_CF]
self.assertEqual(len(cfdefs), 1)
cfdef2 = cfdefs[0]
for attr, val in common_attrs:
val1 = getattr(cfdef, attr)
val2 = getattr(cfdef2, attr)
self.assertEqual(val1, val2, 'attribute %s mismatch: %r != %r' % (attr, val1, val2))
if DO_SYSTEM_RENAMING:
newname = T_CF + '2'
yield self.client.system_rename_column_family(T_CF, newname)
ksdef = yield self.client.describe_keyspace(KEYSPACE)
cfdef2 = [c for c in ksdef.cf_defs if c.name == newname][0]
self.assertNotIn(T_CF, [c.name for c in ksdef.cf_defs])
cfdef.name = newname
self.assertEqual(cfdef, cfdef2)
yield self.client.system_drop_column_family(cfdef.name)
ksdef = yield self.client.describe_keyspace(KEYSPACE)
self.assertNotIn(cfdef.name, [c.name for c in ksdef.cf_defs])
@defer.inlineCallbacks
def test_describes(self):
name = yield self.client.describe_cluster_name()
self.assertIsInstance(name, str)
self.assertNotEqual(name, '')
partitioner = yield self.client.describe_partitioner()
self.assert_(partitioner.startswith('org.apache.cassandra.'),
msg='partitioner is %r' % partitioner)
snitch = yield self.client.describe_snitch()
self.assert_(snitch.startswith('org.apache.cassandra.'),
msg='snitch is %r' % snitch)
version = yield self.client.describe_version()
self.assertIsInstance(version, str)
self.assertIn('.', version)
schemavers = yield self.client.describe_schema_versions()
self.assertIsInstance(schemavers, dict)
self.assertNotEqual(schemavers, {})
ring = yield self.client.describe_ring(KEYSPACE)
self.assertIsInstance(ring, list)
self.assertNotEqual(ring, [])
for r in ring:
self.assertIsInstance(r.start_token, str)
self.assertIsInstance(r.end_token, str)
self.assertIsInstance(r.endpoints, list)
self.assertNotEqual(r.endpoints, [])
for ep in r.endpoints:
self.assertIsInstance(ep, str)
@defer.inlineCallbacks
def test_errback(self):
yield self.client.remove('poiqwe', CF)
try:
yield self.client.get('poiqwe', CF, column='foo')
except Exception, e:
pass
class CassandraProvisionerStore(object):
"""
Provides high level provisioner storage operations for Cassandra
"""
# default size of paged fetches
_PAGE_SIZE = 100
LAUNCH_CF_NAME = "ProvisionerLaunches"
NODE_CF_NAME = "ProvisionerNodes"
@classmethod
def get_column_families(cls, keyspace=None, prefix=''):
"""Builds a list of column families needed by this store.
@param keyspace Name of keyspace. If None, it must be added manually.
@param prefix Optional prefix for cf names. Useful for testing.
@retval list of CfDef objects
"""
launch_cf_name = prefix + cls.LAUNCH_CF_NAME
node_cf_name = prefix + cls.NODE_CF_NAME
return [CfDef(keyspace, launch_cf_name,
comparator_type='org.apache.cassandra.db.marshal.UTF8Type'),
CfDef(keyspace, node_cf_name,
comparator_type='org.apache.cassandra.db.marshal.UTF8Type')]
def __init__(self, host, port, username, password, keyspace, prefix=''):
self._launch_column_family = prefix + self.LAUNCH_CF_NAME
self._node_column_family = prefix + self.NODE_CF_NAME
authz= {'username': username, 'password': password}
self._manager = ManagedCassandraClientFactory(
credentials=authz, check_api_version=True, keyspace=keyspace)
self.client = CassandraClient(self._manager)
self._host = host
self._port = port
self._connector = None
def connect(self):
self._connector = reactor.connectTCP(self._host, self._port,
self._manager)
def disconnect(self):
self._manager.shutdown()
if self._connector:
self._connector.disconnect()
self._connector = None
@timeout(CASSANDRA_TIMEOUT)
def put_launch(self, launch):
"""
@brief Stores a single launch record
@param launch Launch record to store
@retval Deferred for success
"""
launch_id = launch['launch_id']
state = launch['state']
value = json.dumps(launch)
return self.client.insert(launch_id, self._launch_column_family,
value, column=state)
@timeout(CASSANDRA_TIMEOUT)
@defer.inlineCallbacks
def put_nodes(self, nodes):
"""
@brief Stores a set of node records
@param nodes Iterable of node records
@retval Deferred for success
"""
# could be more efficient with a batch_mutate
for node in nodes:
yield self.put_node(node)
@timeout(CASSANDRA_TIMEOUT)
def put_node(self, node):
"""
@brief Stores a node record
@param node Node record
@retval Deferred for success
"""
node_id = node['node_id']
state = node['state']
value = json.dumps(node)
return self.client.insert(node_id, self._node_column_family, value,
column=state)
@timeout(CASSANDRA_TIMEOUT)
def get_launch(self, launch_id, count=1):
"""
@brief Retrieves a launch record by id
@param launch_id Id of launch record to retrieve
@param count Number of launch state records to retrieve
@retval Deferred record(s), or None. A list of records if count > 1
"""
return self._get_record(launch_id, self._launch_column_family, count)
@timeout(CASSANDRA_TIMEOUT)
def get_launches(self, state=None, min_state=None, max_state=None):
"""
@brief Retrieves the latest record for all launches within a state range
@param state Only retrieve nodes in this state.
@param min_state Inclusive start bound
@param max_state Inclusive end bound
@retval Deferred list of launch records
"""
return self._get_records(self._launch_column_family,
state=state,
min_state=min_state,
max_state=max_state)
@timeout(CASSANDRA_TIMEOUT)
def get_node(self, node_id, count=1):
"""
@brief Retrieves a launch record by id
@param node_id Id of node record to retrieve
@param count Number of node state records to retrieve
@retval Deferred record(s), or None. A list of records if count > 1
"""
return self._get_record(node_id, self._node_column_family, count)
@timeout(CASSANDRA_TIMEOUT)
def get_nodes(self, state=None, min_state=None, max_state=None):
"""
@brief Retrieves all launch record within a state range
@param state Only retrieve nodes in this state.
@param min_state Inclusive start bound.
@param max_state Inclusive end bound
@retval Deferred list of launch records
"""
return self._get_records(self._node_column_family,
state=state,
min_state=min_state,
max_state=max_state)
@defer.inlineCallbacks
def _get_record(self, key, column_family, count):
slice = yield self.client.get_slice(key, column_family,
reverse=True, count=count)
# we're probably only interested in the last record, in sorted order.
# This is the latest state the object has recorded.
records = [json.loads(column.column.value) for column in slice]
if count == 1:
if records:
ret = records[0]
else:
ret = None
else:
ret = records
defer.returnValue(ret)
@defer.inlineCallbacks
def _get_records(self, column_family, state=None, min_state=None,
max_state=None, reverse=True):
# overrides range arguments
if state:
min_state = max_state = state
start = ''
end = min_state or ''
if not reverse:
start, end = end, start
# this is tricky. We are only concerned with the latest state record
# (by sort order not necessarily time). So when we look for records
# within a state range, we effectively must pull down the latest state
# for each record, and filter them locally. This is slightly improved
# when a first_state (or last when reverse=False) is specified as the
# server can skip any records not >= that state.
records = []
done = False
start_key = ''
iterations = 0
while not done:
slices = yield self.client.get_range_slices(column_family,
column_start=start,
column_finish=end,
reverse=reverse,
column_count=1,
start=start_key,
count=self._PAGE_SIZE)
skipped_one = False
for slice in slices:
if not skipped_one and iterations:
# if this not the first batch, skip the first element as it
# will be a dupe.
skipped_one = True
continue
if not slice.columns:
# rows without matching columns will still be returned
continue
record = json.loads(slice.columns[0].column.value)
if not max_state or record['state'] <= max_state:
if not min_state or record['state'] >= min_state:
records.append(record)
# page through results. by default only 100 are returned at a time
if len(slices) == self._PAGE_SIZE:
start_key = slices[-1].key
else:
done = True
iterations += 1
defer.returnValue(records)
def on_deactivate(self, *args, **kwargs):
self._manager.shutdown()
log.info('on_deactivate: Lose Connection TCP')
def on_terminate(self, *args, **kwargs):
self._manager.shutdown()
log.info('on_terminate: Lose Connection TCP')
class CassandraClientTest(unittest.TestCase):
def setUp(self):
self.cmanager = ManagedCassandraClientFactory()
self.client = CassandraClient(self.cmanager, KEYSPACE)
for i in xrange(CONNS):
reactor.connectTCP(HOST, PORT, self.cmanager)
return self.cmanager.deferred
@defer.inlineCallbacks
def tearDown(self):
yield self.client.remove('test', CF)
yield self.client.remove('test2', CF)
yield self.client.remove('test', SCF)
yield self.client.remove('test2', SCF)
self.cmanager.shutdown()
for c in reactor.getDelayedCalls():
c.cancel()
reactor.removeAll()
@defer.inlineCallbacks
def test_insert_get(self):
yield self.client.insert('test', ColumnPath(CF, None, COLUMN), 'testval')
yield self.client.insert('test2', CF, 'testval2', column=COLUMN)
yield self.client.insert('test', ColumnPath(SCF, SCOLUMN, COLUMN), 'superval')
yield self.client.insert('test2', SCF, 'superval2', column=COLUMN,
super_column=SCOLUMN)
res = yield self.client.get('test', CF, column=COLUMN)
self.assert_(res.column.value == 'testval')
res = yield self.client.get('test2', CF, column=COLUMN)
self.assert_(res.column.value == 'testval2')
res = yield self.client.get('test', SCF, column=COLUMN, super_column=SCOLUMN)
self.assert_(res.column.value == 'superval')
res = yield self.client.get('test2', SCF, column=COLUMN, super_column=SCOLUMN)
self.assert_(res.column.value == 'superval2')
@defer.inlineCallbacks
def test_batch_insert_get_slice_and_count(self):
yield self.client.batch_insert('test', CF,
{COLUMN: 'test', COLUMN2: 'test2'})
yield self.client.batch_insert('test', SCF,
{SCOLUMN: {COLUMN: 'test', COLUMN2: 'test2'}})
res = yield self.client.get_slice('test', CF, names=(COLUMN, COLUMN2))
self.assert_(res[0].column.value == 'test')
self.assert_(res[1].column.value == 'test2')
res = yield self.client.get_slice('test', SCF, names=(COLUMN, COLUMN2),
super_column=SCOLUMN)
self.assert_(res[0].column.value == 'test')
self.assert_(res[1].column.value == 'test2')
res = yield self.client.get_count('test', CF)
self.assert_(res == 2)
@defer.inlineCallbacks
def test_batch_mutate_and_remove(self):
yield self.client.batch_mutate({'test': {CF: {COLUMN: 'test', COLUMN2: 'test2'}, SCF: { SCOLUMN: { COLUMN: 'test', COLUMN2: 'test2'} } }, 'test2': {CF: {COLUMN: 'test', COLUMN2: 'test2'}, SCF: { SCOLUMN: { COLUMN: 'test', COLUMN2: 'test2'} } } })
res = yield self.client.get_slice('test', CF, names=(COLUMN, COLUMN2))
self.assert_(res[0].column.value == 'test')
self.assert_(res[1].column.value == 'test2')
res = yield self.client.get_slice('test2', CF, names=(COLUMN, COLUMN2))
self.assert_(res[0].column.value == 'test')
self.assert_(res[1].column.value == 'test2')
res = yield self.client.get_slice('test', SCF, names=(COLUMN, COLUMN2),
super_column=SCOLUMN)
self.assert_(res[0].column.value == 'test')
self.assert_(res[1].column.value == 'test2')
res = yield self.client.get_slice('test2', SCF, names=(COLUMN, COLUMN2),
super_column=SCOLUMN)
self.assert_(res[0].column.value == 'test')
self.assert_(res[1].column.value == 'test2')
yield self.client.batch_remove({CF: ['test', 'test2']}, names=['test', 'test2'])
yield self.client.batch_remove({SCF: ['test', 'test2']}, names=['test', 'test2'], supercolumn=SCOLUMN)
@defer.inlineCallbacks
def test_batch_mutate_with_deletion(self):
yield self.client.batch_mutate({'test': {CF: {COLUMN: 'test', COLUMN2: 'test2'}}})
res = yield self.client.get_slice('test', CF, names=(COLUMN, COLUMN2))
self.assert_(res[0].column.value == 'test')
self.assert_(res[1].column.value == 'test2')
yield self.client.batch_mutate({'test': {CF: {COLUMN: None, COLUMN2: 'test3'}}})
res = yield self.client.get_slice('test', CF, names=(COLUMN, COLUMN2))
self.assert_(len(res) == 1)
self.assert_(res[0].column.value == 'test3')
@defer.inlineCallbacks
def test_multiget_slice_remove(self):
yield self.client.insert('test', CF, 'testval', column=COLUMN)
yield self.client.insert('test', CF, 'testval', column=COLUMN2)
yield self.client.insert('test2', CF, 'testval2', column=COLUMN)
res = yield self.client.multiget(['test', 'test2'], CF, column=COLUMN)
self.assert_(res['test'].column.value == 'testval')
self.assert_(res['test2'].column.value == 'testval2')
res = yield self.client.multiget_slice(['test', 'test2'], CF)
self.assert_(res['test'][0].column.value == 'testval')
self.assert_(res['test'][1].column.value == 'testval')
self.assert_(res['test2'][0].column.value == 'testval2')
yield self.client.remove('test', CF, column=COLUMN)
yield self.client.remove('test2', CF, column=COLUMN)
res = yield self.client.multiget(['test', 'test2'], CF, column=COLUMN)
self.assert_(res['test'].column == None)
self.assert_(res['test2'].column == None)
@defer.inlineCallbacks
def test_range_slices(self):
yield self.client.insert('test', CF, 'testval', column=COLUMN)
yield self.client.insert('test', CF, 'testval', column=COLUMN2)
yield self.client.insert('test2', CF, 'testval2', column=COLUMN)
ks = yield self.client.get_range_slices(CF, start='', finish='')
keys = [k.key for k in ks]
for key in ['test', 'test2']:
self.assert_(key in keys)
@defer.inlineCallbacks
def test_errback(self):
yield self.client.remove('poiqwe', CF)
try:
yield self.client.get('poiqwe', CF, column='foo')
except Exception, e:
pass
