def test_initial_connection_failure(self):
cmanager = ManagedCassandraClientFactory()
client = CassandraClient(cmanager)
d = cmanager.deferred
reactor.connectTCP('nonexistent-host.000-', PORT, cmanager)
yield self.assertFailure(d, error.DNSLookupError)
cmanager.shutdown()
def test_initial_connection_failure(self):
cmanager = ManagedCassandraClientFactory()
client = CassandraClient(cmanager)
d = cmanager.deferred
reactor.connectTCP('nonexistent.example.com', PORT, cmanager)
yield self.assertFailure(d, error.DNSLookupError)
cmanager.shutdown()
def __init__(self, controller_name, host, port, username, password,
keyspace, instance_factory, sensor_item_factory,
prefix=''):
self.controller_name = str(controller_name)
# keep a set of known instances and sensors so we can save on
# unnecessary inserts to the controller instance/sensor lists
self.seen_instances = set()
self.seen_sensors = set()
self.instance_factory = instance_factory
self.sensor_item_factory = sensor_item_factory
authorization_dictionary = {'username': username, 'password': password}
self.manager = ManagedCassandraClientFactory(
credentials=authorization_dictionary,
check_api_version=True, keyspace=keyspace)
TCPConnection.__init__(self, host, port, self.manager)
self.client = CassandraClient(self.manager)
self.instance_cf = prefix + self.INSTANCE_CF_NAME
self.instance_id_cf = prefix + self.INSTANCE_ID_CF_NAME
self.sensor_cf = prefix + self.SENSOR_CF_NAME
self.sensor_id_cf = prefix + self.SENSOR_ID_CF_NAME
self.config_cf = prefix + self.CONFIG_CF_NAME
def test_api_mismatch(self):
cmanager = ManagedCassandraClientFactory(check_api_version=True)
constants.VERSION = '0.0.0'
client = CassandraClient(cmanager)
d = cmanager.deferred
conn = reactor.connectTCP(HOST, PORT, cmanager)
yield self.assertFailure(d, APIMismatch)
cmanager.shutdown()
def test_api_match(self):
for version in [translate.CASSANDRA_07_VERSION, translate.CASSANDRA_08_VERSION, None]:
cmanager = ManagedCassandraClientFactory(api_version=version)
client = CassandraClient(cmanager)
d = cmanager.deferred
conn = reactor.connectTCP(HOST, PORT, cmanager)
yield d
# do something innocuous, make sure connection is good
yield client.describe_schema_versions()
yield cmanager.shutdown()
def test_api_match(self):
cmanager = ManagedCassandraClientFactory(require_api_version=None)
client = CassandraClient(cmanager)
d = cmanager.deferred
conn = reactor.connectTCP(HOST, PORT, cmanager)
yield d
yield client.describe_schema_versions()
api_ver = cmanager._protos[0].api_version
yield cmanager.shutdown()
# try with the right version explicitly required
cmanager = ManagedCassandraClientFactory(require_api_version=api_ver)
client = CassandraClient(cmanager)
d = cmanager.deferred
conn = reactor.connectTCP(HOST, PORT, cmanager)
yield d
yield client.describe_schema_versions()
yield cmanager.shutdown()
# try with a mismatching version explicitly required
bad_ver = [
v for (_, v) in translate.supported_versions if v != api_ver
][0]
cmanager = ManagedCassandraClientFactory(require_api_version=bad_ver)
client = CassandraClient(cmanager)
d = cmanager.deferred
conn = reactor.connectTCP(HOST, PORT, cmanager)
yield self.assertFailure(d, translate.APIMismatch)
yield cmanager.shutdown()
def __init__(self, host, port, username, password, keyspace, prefix=''):
self._launch_column_family = prefix + self.LAUNCH_CF_NAME
self._node_column_family = prefix + self.NODE_CF_NAME
authz = {'username': username, 'password': password}
self._manager = ManagedCassandraClientFactory(credentials=authz,
check_api_version=True,
keyspace=keyspace)
self.client = CassandraClient(self._manager)
self._host = host
self._port = port
self._connector = None
def setUp(self):
self.cmanager = ManagedCassandraClientFactory(keyspace='system')
self.client = CassandraClient(self.cmanager)
for i in xrange(CONNS):
reactor.connectTCP(HOST, PORT, self.cmanager)
yield self.cmanager.deferred
remote_ver = yield self.client.describe_version()
self.version = thrift_api_ver_to_cassandra_ver(remote_ver)
self.my_keyspace = KsDef(
name=KEYSPACE,
strategy_class='org.apache.cassandra.locator.SimpleStrategy',
strategy_options={'replication_factor': '1'},
cf_defs=[
CfDef(keyspace=KEYSPACE, name=CF, column_type='Standard'),
CfDef(keyspace=KEYSPACE, name=SCF, column_type='Super'),
CfDef(
keyspace=KEYSPACE,
name=IDX_CF,
column_type='Standard',
comparator_type='org.apache.cassandra.db.marshal.UTF8Type',
column_metadata=[
ColumnDef(name='col1',
validation_class=
'org.apache.cassandra.db.marshal.UTF8Type',
index_type=IndexType.KEYS,
index_name='idxCol1')
],
default_validation_class=
'org.apache.cassandra.db.marshal.BytesType'),
])
if self.version == CASSANDRA_08_VERSION:
self.my_keyspace.cf_defs.extend([
CfDef(keyspace=KEYSPACE,
name=COUNTER_CF,
column_type='Standard',
default_validation_class=
'org.apache.cassandra.db.marshal.CounterColumnType'),
CfDef(keyspace=KEYSPACE,
name=SUPERCOUNTER_CF,
column_type='Super',
default_validation_class=
'org.apache.cassandra.db.marshal.CounterColumnType'),
])
yield self.client.system_add_keyspace(self.my_keyspace)
yield self.client.set_keyspace(KEYSPACE)
def setUp(self):
self.factory = ManagedCassandraClientFactory()
self.client = CassandraClient(self.factory, KEYSPACE)
self.conns = []
for i in xrange(CONNS):
conn = reactor.connectTCP(HOST, PORT, self.factory)
self.conns.append(conn)
def __init__(self, keyspace):
self._keyspace = keyspace
self.factory = ManagedCassandraClientFactory(keyspace)
host = settings.CASS_HOST.strip()
# Ensure that the host isn't blank
if host == '':
host = 'localhost'
addresses = socket.getaddrinfo(host, settings.CASS_PORT)
# If we are using IPv6, we need to provide an IPv6 address directly
# to twisted, as it doesn't support IPv6.
# addresses is a list of 5 tuple:
# (family, socktype, proto, cannonname, sockaddr)
shuffle(addresses)
if len(addresses) > 1 and addresses[0][0] == socket.AF_INET6:
# sockaddr is a 4 tuple:
# (address, port, flow, scope)
address = addresses[0][4][0]
else:
address = host
_log.debug("Cassandra is connecting to %s - for host %s", address,
settings.CASS_HOST)
reactor.connectTCP(address, settings.CASS_PORT, self.factory)
self.client = CassandraClient(self.factory)
def connect(self, host=None, port=9160, username=None, password=None):
if not host:
host, port = get_host_port()
if username or password:
if not (username and password):
raise CassandraConfigurationError(
"Specify both username and password or neither")
else:
username, password = get_credentials()
authz = dict(username=username, password=password)
self.manager = ManagedCassandraClientFactory(credentials=authz,
check_api_version=True)
self.connector = reactor.connectTCP(host, port, self.manager)
self.client = CassandraClient(self.manager)
def setUp(self):
self.cmanager = ManagedCassandraClientFactory(keyspace='system')
self.client = CassandraClient(self.cmanager)
for i in xrange(CONNS):
reactor.connectTCP(HOST, PORT, self.cmanager)
yield self.cmanager.deferred
remote_ver = yield self.client.describe_version()
self.version = getAPIVersion(remote_ver)
self.my_keyspace = KsDef(
name=KEYSPACE,
strategy_class='org.apache.cassandra.locator.SimpleStrategy',
strategy_options={'replication_factor': '1'},
cf_defs=[
CfDef(
keyspace=KEYSPACE,
name=CF,
column_type='Standard'
),
CfDef(
keyspace=KEYSPACE,
name=SCF,
column_type='Super'
),
CfDef(
keyspace=KEYSPACE,
name=IDX_CF,
column_type='Standard',
comparator_type='org.apache.cassandra.db.marshal.UTF8Type',
column_metadata=[
ColumnDef(
name='col1',
validation_class='org.apache.cassandra.db.marshal.UTF8Type',
index_type=IndexType.KEYS,
index_name='idxCol1')
],
default_validation_class='org.apache.cassandra.db.marshal.BytesType'
),
]
)
if self.version == CASSANDRA_08_VERSION:
self.my_keyspace.cf_defs.extend([
CfDef(
keyspace=KEYSPACE,
name=COUNTER_CF,
column_type='Standard',
default_validation_class='org.apache.cassandra.db.marshal.CounterColumnType'
),
CfDef(
keyspace=KEYSPACE,
name=SUPERCOUNTER_CF,
column_type='Super',
default_validation_class='org.apache.cassandra.db.marshal.CounterColumnType'
),
])
yield self.client.system_add_keyspace(self.my_keyspace)
yield self.client.set_keyspace(KEYSPACE)
def connect(keyspace='system',
hosts=['localhost:9160'],
connections=5,
consistency=ConsistencyLevel.ONE):
manager = ManagedCassandraClientFactory(keyspace=keyspace)
client = CassandraClient(manager, consistency=consistency)
for i in xrange(connections):
for host, port in [x.split(':') for x in hosts]:
reactor.connectTCP(host, int(port), manager)
return client
def main():
sasl_kwargs = {
"host": "thobbs-laptop2",
"service": "host",
"mechanism": "GSSAPI"}
f = ManagedCassandraClientFactory(keyspace='system', sasl_kwargs=sasl_kwargs)
reactor.connectTCP(HOST, PORT, f)
yield f.deferred
c = CassandraClient(f)
yield setup_schema(c)
yield dostuff(c)
def test_api_match(self):
cmanager = ManagedCassandraClientFactory(require_api_version=None)
client = CassandraClient(cmanager)
d = cmanager.deferred
conn = reactor.connectTCP(HOST, PORT, cmanager)
yield d
yield client.describe_schema_versions()
api_ver = cmanager._protos[0].api_version
yield cmanager.shutdown()
# try with the right version explicitly required
cmanager = ManagedCassandraClientFactory(require_api_version=api_ver)
client = CassandraClient(cmanager)
d = cmanager.deferred
conn = reactor.connectTCP(HOST, PORT, cmanager)
yield d
yield client.describe_schema_versions()
yield cmanager.shutdown()
# try with a mismatching version explicitly required
bad_ver = [v for (_, v) in translate.supported_versions if v != api_ver][0]
cmanager = ManagedCassandraClientFactory(require_api_version=bad_ver)
client = CassandraClient(cmanager)
d = cmanager.deferred
conn = reactor.connectTCP(HOST, PORT, cmanager)
yield self.assertFailure(d, translate.APIMismatch)
yield cmanager.shutdown()
def __init__(self, host, port, username, password, keyspace, prefix=''):
self._launch_column_family = prefix + self.LAUNCH_CF_NAME
self._node_column_family = prefix + self.NODE_CF_NAME
authz= {'username': username, 'password': password}
self._manager = ManagedCassandraClientFactory(
credentials=authz, check_api_version=True, keyspace=keyspace)
self.client = CassandraClient(self._manager)
self._host = host
self._port = port
self._connector = None
def connect(keyspace='system',
hosts=['localhost:9160'],
connections=5,
consistency=ConsistencyLevel.ONE):
manager = ManagedCassandraClientFactory()
client = PorcelainClient(manager, consistency=consistency)
for i in xrange(connections):
for host, port in [x.split(':') for x in hosts]:
reactor.connectTCP(host, int(port), manager)
yield manager.deferred
yield client.set_keyspace(keyspace)
defer.returnValue(client)
def create_application():
app_settings = {
"gzip": True,
"cookie_secret": settings.COOKIE_SECRET,
"debug": settings.CYCLONE_DEBUG,
}
application = cyclone.web.Application(api.get_routes(), **app_settings)
application.cassandra_factory = ManagedCassandraClientFactory(
settings.CASS_KEYSPACE)
reactor.connectTCP(settings.CASS_HOST, settings.CASS_PORT,
application.cassandra_factory)
# Initialize all modules
api.initialize(application)
return application
def connect(self, host=None, port=9160, username=None, password=None):
if not host:
host, port = get_host_port()
if username or password:
if not (username and password):
raise CassandraConfigurationError(
"Specify both username and password or neither")
else:
username, password = get_credentials()
authz = dict(username=username, password=password)
self.manager = ManagedCassandraClientFactory(credentials=authz,
check_api_version=True)
self.connector = reactor.connectTCP(host, port, self.manager)
self.client = CassandraClient(self.manager)
def do_this():
for gzfilename in sys.argv[1:]:
n = 0
with gzip.GzipFile(gzfilename) as f:
csv_file = csv.DictReader(f)
first = True
last_key = ''
for row in csv_file:
if first:
factory = ManagedCassandraClientFactory(row["keyspace"])
reactor.connectTCP("localhost", 9160, factory)
client = CassandraClient(factory)
first = False
yield client.insert(column_family=row["cf"], key=uuidify(row["key"].decode("string_escape")), column=row["col"], value=uuidify(row["val"].decode("string_escape")))
if last_key != row["key"].decode("string_escape"):
n += 1
last_key = row["key"].decode("string_escape")
print "Successfully restored %d rows from %s" % (n, gzfilename)
reactor.stop()
def setUp(self):
self.cmanager = ManagedCassandraClientFactory(keyspace='system')
self.client = CassandraClient(self.cmanager)
for i in xrange(CONNS):
reactor.connectTCP(HOST, PORT, self.cmanager)
yield self.cmanager.deferred
self.my_keyspace = KsDef(
name=KEYSPACE,
strategy_class='org.apache.cassandra.locator.SimpleStrategy',
replication_factor=1,
cf_defs=[
CfDef(
keyspace=KEYSPACE,
name=CF,
column_type='Standard'
),
CfDef(
keyspace=KEYSPACE,
name=SCF,
column_type='Super'
),
CfDef(
keyspace=KEYSPACE,
name=IDX_CF,
column_type='Standard',
comparator_type='org.apache.cassandra.db.marshal.UTF8Type',
column_metadata=[
ColumnDef(
name='col1',
validation_class='org.apache.cassandra.db.marshal.UTF8Type',
index_type=IndexType.KEYS,
index_name='idxCol1')
],
default_validation_class='org.apache.cassandra.db.marshal.BytesType')
]
)
yield self.client.system_add_keyspace(self.my_keyspace)
yield self.client.set_keyspace(KEYSPACE)
def test_api_check(self):
cmanager = ManagedCassandraClientFactory(check_api_version=False)
client = CassandraClient(cmanager)
conn = reactor.connectTCP(HOST, PORT, cmanager)
# we don't necessarily want to force an api match while testing;
# get the remote value and pretend ours matches, even if it doesn't
ver = yield client.describe_version()
cmanager.shutdown()
constants.VERSION = ver
cmanager = ManagedCassandraClientFactory(check_api_version=True)
client = CassandraClient(cmanager)
d = cmanager.deferred
conn = reactor.connectTCP(HOST, PORT, cmanager)
yield d
# do something innocuous, make sure connection is good
yield client.describe_schema_versions()
cmanager.shutdown()
class CassandraProvisionerStore(object):
"""
Provides high level provisioner storage operations for Cassandra
"""
# default size of paged fetches
_PAGE_SIZE = 100
LAUNCH_CF_NAME = "ProvisionerLaunches"
NODE_CF_NAME = "ProvisionerNodes"
@classmethod
def get_column_families(cls, keyspace=None, prefix=''):
"""Builds a list of column families needed by this store.
@param keyspace Name of keyspace. If None, it must be added manually.
@param prefix Optional prefix for cf names. Useful for testing.
@retval list of CfDef objects
"""
launch_cf_name = prefix + cls.LAUNCH_CF_NAME
node_cf_name = prefix + cls.NODE_CF_NAME
return [CfDef(keyspace, launch_cf_name,
comparator_type='org.apache.cassandra.db.marshal.UTF8Type'),
CfDef(keyspace, node_cf_name,
comparator_type='org.apache.cassandra.db.marshal.UTF8Type')]
def __init__(self, host, port, username, password, keyspace, prefix=''):
self._launch_column_family = prefix + self.LAUNCH_CF_NAME
self._node_column_family = prefix + self.NODE_CF_NAME
authz= {'username': username, 'password': password}
self._manager = ManagedCassandraClientFactory(
credentials=authz, check_api_version=True, keyspace=keyspace)
self.client = CassandraClient(self._manager)
self._host = host
self._port = port
self._connector = None
def connect(self):
self._connector = reactor.connectTCP(self._host, self._port,
self._manager)
def disconnect(self):
self._manager.shutdown()
if self._connector:
self._connector.disconnect()
self._connector = None
@timeout(CASSANDRA_TIMEOUT)
def put_launch(self, launch):
"""
@brief Stores a single launch record
@param launch Launch record to store
@retval Deferred for success
"""
launch_id = launch['launch_id']
state = launch['state']
value = json.dumps(launch)
return self.client.insert(launch_id, self._launch_column_family,
value, column=state)
@timeout(CASSANDRA_TIMEOUT)
@defer.inlineCallbacks
def put_nodes(self, nodes):
"""
@brief Stores a set of node records
@param nodes Iterable of node records
@retval Deferred for success
"""
# could be more efficient with a batch_mutate
for node in nodes:
yield self.put_node(node)
@timeout(CASSANDRA_TIMEOUT)
def put_node(self, node):
"""
@brief Stores a node record
@param node Node record
@retval Deferred for success
"""
node_id = node['node_id']
state = node['state']
value = json.dumps(node)
return self.client.insert(node_id, self._node_column_family, value,
column=state)
@timeout(CASSANDRA_TIMEOUT)
def get_launch(self, launch_id, count=1):
"""
@brief Retrieves a launch record by id
@param launch_id Id of launch record to retrieve
@param count Number of launch state records to retrieve
@retval Deferred record(s), or None. A list of records if count > 1
"""
return self._get_record(launch_id, self._launch_column_family, count)
@timeout(CASSANDRA_TIMEOUT)
def get_launches(self, state=None, min_state=None, max_state=None):
"""
@brief Retrieves the latest record for all launches within a state range
@param state Only retrieve nodes in this state.
@param min_state Inclusive start bound
@param max_state Inclusive end bound
@retval Deferred list of launch records
"""
return self._get_records(self._launch_column_family,
state=state,
min_state=min_state,
max_state=max_state)
@timeout(CASSANDRA_TIMEOUT)
def get_node(self, node_id, count=1):
"""
@brief Retrieves a launch record by id
@param node_id Id of node record to retrieve
@param count Number of node state records to retrieve
@retval Deferred record(s), or None. A list of records if count > 1
"""
return self._get_record(node_id, self._node_column_family, count)
@timeout(CASSANDRA_TIMEOUT)
def get_nodes(self, state=None, min_state=None, max_state=None):
"""
@brief Retrieves all launch record within a state range
@param state Only retrieve nodes in this state.
@param min_state Inclusive start bound.
@param max_state Inclusive end bound
@retval Deferred list of launch records
"""
return self._get_records(self._node_column_family,
state=state,
min_state=min_state,
max_state=max_state)
@defer.inlineCallbacks
def _get_record(self, key, column_family, count):
slice = yield self.client.get_slice(key, column_family,
reverse=True, count=count)
# we're probably only interested in the last record, in sorted order.
# This is the latest state the object has recorded.
records = [json.loads(column.column.value) for column in slice]
if count == 1:
if records:
ret = records[0]
else:
ret = None
else:
ret = records
defer.returnValue(ret)
@defer.inlineCallbacks
def _get_records(self, column_family, state=None, min_state=None,
max_state=None, reverse=True):
# overrides range arguments
if state:
min_state = max_state = state
start = ''
end = min_state or ''
if not reverse:
start, end = end, start
# this is tricky. We are only concerned with the latest state record
# (by sort order not necessarily time). So when we look for records
# within a state range, we effectively must pull down the latest state
# for each record, and filter them locally. This is slightly improved
# when a first_state (or last when reverse=False) is specified as the
# server can skip any records not >= that state.
records = []
done = False
start_key = ''
iterations = 0
while not done:
slices = yield self.client.get_range_slices(column_family,
column_start=start,
column_finish=end,
reverse=reverse,
column_count=1,
start=start_key,
count=self._PAGE_SIZE)
skipped_one = False
for slice in slices:
if not skipped_one and iterations:
# if this not the first batch, skip the first element as it
# will be a dupe.
skipped_one = True
continue
if not slice.columns:
# rows without matching columns will still be returned
continue
record = json.loads(slice.columns[0].column.value)
if not max_state or record['state'] <= max_state:
if not min_state or record['state'] >= min_state:
records.append(record)
# page through results. by default only 100 are returned at a time
if len(slices) == self._PAGE_SIZE:
start_key = slices[-1].key
else:
done = True
iterations += 1
defer.returnValue(records)
def on_deactivate(self, *args, **kwargs):
self._manager.shutdown()
log.info('on_deactivate: Lose Connection TCP')
def on_terminate(self, *args, **kwargs):
self._manager.shutdown()
log.info('on_terminate: Lose Connection TCP')
class CassandraClientTest(unittest.TestCase):
@defer.inlineCallbacks
def setUp(self):
self.cmanager = ManagedCassandraClientFactory(keyspace='system')
self.client = CassandraClient(self.cmanager)
for i in xrange(CONNS):
reactor.connectTCP(HOST, PORT, self.cmanager)
yield self.cmanager.deferred
remote_ver = yield self.client.describe_version()
self.version = getAPIVersion(remote_ver)
self.my_keyspace = KsDef(
name=KEYSPACE,
strategy_class='org.apache.cassandra.locator.SimpleStrategy',
strategy_options={'replication_factor': '1'},
cf_defs=[
CfDef(
keyspace=KEYSPACE,
name=CF,
column_type='Standard'
),
CfDef(
keyspace=KEYSPACE,
name=SCF,
column_type='Super'
),
CfDef(
keyspace=KEYSPACE,
name=IDX_CF,
column_type='Standard',
comparator_type='org.apache.cassandra.db.marshal.UTF8Type',
column_metadata=[
ColumnDef(
name='col1',
validation_class='org.apache.cassandra.db.marshal.UTF8Type',
index_type=IndexType.KEYS,
index_name='idxCol1')
],
default_validation_class='org.apache.cassandra.db.marshal.BytesType'
),
]
)
if self.version == CASSANDRA_08_VERSION:
self.my_keyspace.cf_defs.extend([
CfDef(
keyspace=KEYSPACE,
name=COUNTER_CF,
column_type='Standard',
default_validation_class='org.apache.cassandra.db.marshal.CounterColumnType'
),
CfDef(
keyspace=KEYSPACE,
name=SUPERCOUNTER_CF,
column_type='Super',
default_validation_class='org.apache.cassandra.db.marshal.CounterColumnType'
),
])
yield self.client.system_add_keyspace(self.my_keyspace)
yield self.client.set_keyspace(KEYSPACE)
@defer.inlineCallbacks
def tearDown(self):
yield self.client.system_drop_keyspace(self.my_keyspace.name)
self.cmanager.shutdown()
for c in reactor.getDelayedCalls():
c.cancel()
reactor.removeAll()
@defer.inlineCallbacks
def test_insert_get(self):
yield self.client.insert('test', CF, 'testval', column=COLUMN)
yield self.client.insert('test2', CF, 'testval2', column=COLUMN)
yield self.client.insert('test', SCF, 'superval', column=COLUMN, super_column=SCOLUMN)
yield self.client.insert('test2', SCF, 'superval2', column=COLUMN,
super_column=SCOLUMN)
res = yield self.client.get('test', CF, column=COLUMN)
self.assertEqual(res.column.value, 'testval')
res = yield self.client.get('test2', CF, column=COLUMN)
self.assertEqual(res.column.value, 'testval2')
res = yield self.client.get('test', SCF, column=COLUMN, super_column=SCOLUMN)
self.assertEqual(res.column.value, 'superval')
res = yield self.client.get('test2', SCF, column=COLUMN, super_column=SCOLUMN)
self.assertEqual(res.column.value, 'superval2')
@defer.inlineCallbacks
def test_batch_insert_get_slice_and_count(self):
yield self.client.batch_insert('test', CF,
{COLUMN: 'test', COLUMN2: 'test2'})
yield self.client.batch_insert('test', SCF,
{SCOLUMN: {COLUMN: 'test', COLUMN2: 'test2'}})
res = yield self.client.get_slice('test', CF, names=(COLUMN, COLUMN2))
self.assertEqual(res[0].column.value, 'test')
self.assertEqual(res[1].column.value, 'test2')
res = yield self.client.get_slice('test', SCF, names=(COLUMN, COLUMN2),
super_column=SCOLUMN)
self.assertEqual(res[0].column.value, 'test')
self.assertEqual(res[1].column.value, 'test2')
res = yield self.client.get_count('test', CF)
self.assertEqual(res, 2)
@defer.inlineCallbacks
def test_batch_mutate_and_remove(self):
yield self.client.batch_mutate({'test': {CF: {COLUMN: 'test', COLUMN2: 'test2'}, SCF: { SCOLUMN: { COLUMN: 'test', COLUMN2: 'test2'} } }, 'test2': {CF: {COLUMN: 'test', COLUMN2: 'test2'}, SCF: { SCOLUMN: { COLUMN: 'test', COLUMN2: 'test2'} } } })
res = yield self.client.get_slice('test', CF, names=(COLUMN, COLUMN2))
self.assertEqual(res[0].column.value, 'test')
self.assertEqual(res[1].column.value, 'test2')
res = yield self.client.get_slice('test2', CF, names=(COLUMN, COLUMN2))
self.assertEqual(res[0].column.value, 'test')
self.assertEqual(res[1].column.value, 'test2')
res = yield self.client.get_slice('test', SCF, names=(COLUMN, COLUMN2),
super_column=SCOLUMN)
self.assertEqual(res[0].column.value, 'test')
self.assertEqual(res[1].column.value, 'test2')
res = yield self.client.get_slice('test2', SCF, names=(COLUMN, COLUMN2),
super_column=SCOLUMN)
self.assertEqual(res[0].column.value, 'test')
self.assertEqual(res[1].column.value, 'test2')
yield self.client.batch_remove({CF: ['test', 'test2']}, names=['test', 'test2'])
yield self.client.batch_remove({SCF: ['test', 'test2']}, names=['test', 'test2'], supercolumn=SCOLUMN)
@defer.inlineCallbacks
def test_batch_mutate_with_deletion(self):
yield self.client.batch_mutate({'test': {CF: {COLUMN: 'test', COLUMN2: 'test2'}}})
res = yield self.client.get_slice('test', CF, names=(COLUMN, COLUMN2))
self.assertEqual(res[0].column.value, 'test')
self.assertEqual(res[1].column.value, 'test2')
yield self.client.batch_mutate({'test': {CF: {COLUMN: None, COLUMN2: 'test3'}}})
res = yield self.client.get_slice('test', CF, names=(COLUMN, COLUMN2))
self.assertEqual(len(res), 1)
self.assertEqual(res[0].column.value, 'test3')
@defer.inlineCallbacks
def test_multiget_slice_remove(self):
yield self.client.insert('test', CF, 'testval', column=COLUMN)
yield self.client.insert('test', CF, 'testval', column=COLUMN2)
yield self.client.insert('test2', CF, 'testval2', column=COLUMN)
res = yield self.client.multiget(['test', 'test2'], CF, column=COLUMN)
self.assertEqual(res['test'][0].column.value, 'testval')
self.assertEqual(res['test2'][0].column.value, 'testval2')
res = yield self.client.multiget_slice(['test', 'test2'], CF)
self.assertEqual(res['test'][0].column.value, 'testval')
self.assertEqual(res['test'][1].column.value, 'testval')
self.assertEqual(res['test2'][0].column.value, 'testval2')
yield self.client.remove('test', CF, column=COLUMN)
yield self.client.remove('test2', CF, column=COLUMN)
res = yield self.client.multiget(['test', 'test2'], CF, column=COLUMN)
self.assertEqual(len(res['test']), 0)
self.assertEqual(len(res['test2']), 0)
@defer.inlineCallbacks
def test_range_slices(self):
yield self.client.insert('test', CF, 'testval', column=COLUMN)
yield self.client.insert('test', CF, 'testval', column=COLUMN2)
yield self.client.insert('test2', CF, 'testval2', column=COLUMN)
ks = yield self.client.get_range_slices(CF, start='', finish='')
keys = [k.key for k in ks]
for key in ['test', 'test2']:
self.assertIn(key, keys)
@defer.inlineCallbacks
def test_indexed_slices(self):
yield self.client.insert('test1', IDX_CF, 'one', column='col1')
yield self.client.insert('test2', IDX_CF, 'two', column='col1')
yield self.client.insert('test3', IDX_CF, 'three', column='col1')
expressions = [IndexExpression('col1', IndexOperator.EQ, 'two')]
res = yield self.client.get_indexed_slices(IDX_CF, expressions, start_key='')
self.assertEquals(res[0].columns[0].column.value,'two')
@defer.inlineCallbacks
def test_counter_add(self):
if self.version != CASSANDRA_08_VERSION:
raise unittest.SkipTest('Counters are not supported in 0.7')
# test standard column counter
yield self.client.add('test', COUNTER_CF, 1, column='col')
res = yield self.client.get('test', COUNTER_CF, column='col')
self.assertEquals(res.counter_column.value, 1)
yield self.client.add('test', COUNTER_CF, 1, column='col')
res = yield self.client.get('test', COUNTER_CF, column='col')
self.assertEquals(res.counter_column.value, 2)
# test super column counters
yield self.client.add('test', SUPERCOUNTER_CF, 1, column='col', super_column='scol')
res = yield self.client.get('test', SUPERCOUNTER_CF, column='col', super_column='scol')
self.assertEquals(res.counter_column.value, 1)
yield self.client.add('test', SUPERCOUNTER_CF, 1, column='col', super_column='scol')
res = yield self.client.get('test', SUPERCOUNTER_CF, column='col', super_column='scol')
self.assertEquals(res.counter_column.value, 2)
@defer.inlineCallbacks
def test_counter_remove(self):
if self.version != CASSANDRA_08_VERSION:
raise unittest.SkipTest('Counters are not supported in 0.7')
# test standard column counter
yield self.client.add('test', COUNTER_CF, 1, column='col')
res = yield self.client.get('test', COUNTER_CF, column='col')
self.assertEquals(res.counter_column.value, 1)
yield self.client.remove_counter('test', COUNTER_CF, column='col')
yield self.assertFailure(self.client.get('test', COUNTER_CF, column='col'),
NotFoundException)
# test super column counters
yield self.client.add('test', SUPERCOUNTER_CF, 1, column='col', super_column='scol')
res = yield self.client.get('test', SUPERCOUNTER_CF, column='col', super_column='scol')
self.assertEquals(res.counter_column.value, 1)
yield self.client.remove_counter('test', SUPERCOUNTER_CF,
column='col', super_column='scol')
yield self.assertFailure(self.client.get('test', SUPERCOUNTER_CF,
column='col', super_column='scol'),
NotFoundException)
@defer.inlineCallbacks
def test_cql(self):
if self.version != CASSANDRA_08_VERSION:
raise unittest.SkipTest('CQL is not supported in 0.7')
yield self.client.insert('test', CF, 'testval', column='col1')
res = yield self.client.get('test', CF, column='col1')
self.assertEquals(res.column.value, 'testval')
query = 'SELECT * from %s where KEY = %s' % (CF, 'test'.encode('hex'))
uncompressed_result = yield self.client.execute_cql_query(query, Compression.NONE)
self.assertEquals(uncompressed_result.rows[0].columns[0].name, 'col1')
self.assertEquals(uncompressed_result.rows[0].columns[0].value, 'testval')
compressed_query = zlib.compress(query)
compressed_result = yield self.client.execute_cql_query(compressed_query, Compression.GZIP)
self.assertEquals(uncompressed_result, compressed_result)
def sleep(self, secs):
d = defer.Deferred()
reactor.callLater(secs, d.callback, None)
return d
@defer.inlineCallbacks
def test_ttls(self):
yield self.client.insert('test_ttls', CF, 'testval', column=COLUMN, ttl=1)
res = yield self.client.get('test_ttls', CF, column=COLUMN)
self.assertEqual(res.column.value, 'testval')
yield self.sleep(2)
yield self.assertFailure(self.client.get('test_ttls', CF, column=COLUMN), NotFoundException)
yield self.client.batch_insert('test_ttls', CF, {COLUMN:'testval'}, ttl=1)
res = yield self.client.get('test_ttls', CF, column=COLUMN)
self.assertEqual(res.column.value, 'testval')
yield self.sleep(2)
yield self.assertFailure(self.client.get('test_ttls', CF, column=COLUMN), NotFoundException)
yield self.client.batch_mutate({'test_ttls': {CF: {COLUMN: 'testval'}}}, ttl=1)
res = yield self.client.get('test_ttls', CF, column=COLUMN)
self.assertEqual(res.column.value, 'testval')
yield self.sleep(2)
yield self.assertFailure(self.client.get('test_ttls', CF, column=COLUMN), NotFoundException)
def compare_keyspaces(self, ks1, ks2):
self.assertEqual(ks1.name, ks2.name)
self.assertEqual(ks1.strategy_class, ks2.strategy_class)
self.assertEqual(ks1.cf_defs, ks2.cf_defs)
def get_rf(ksdef):
rf = ksdef.replication_factor
if ksdef.strategy_options and \
'replication_factor' in ksdef.strategy_options:
rf = int(ksdef.strategy_options['replication_factor'])
return rf
def strat_opts_no_rf(ksdef):
if not ksdef.strategy_options:
return {}
opts = ksdef.strategy_options.copy()
if 'replication_factor' in ksdef.strategy_options:
del opts['replication_factor']
return opts
self.assertEqual(get_rf(ks1), get_rf(ks2))
self.assertEqual(strat_opts_no_rf(ks1), strat_opts_no_rf(ks2))
@defer.inlineCallbacks
def test_keyspace_manipulation(self):
try:
yield self.client.system_drop_keyspace(T_KEYSPACE)
except InvalidRequestException:
pass
ksdef = KsDef(name=T_KEYSPACE, strategy_class='org.apache.cassandra.locator.SimpleStrategy', strategy_options={'replication_factor': '1'}, cf_defs=[])
yield self.client.system_add_keyspace(ksdef)
ks2 = yield self.client.describe_keyspace(T_KEYSPACE)
self.compare_keyspaces(ksdef, ks2)
if DO_SYSTEM_RENAMING:
newname = T_KEYSPACE + '2'
yield self.client.system_rename_keyspace(T_KEYSPACE, newname)
ks2 = yield self.client.describe_keyspace(newname)
ksdef.name = newname
self.compare_keyspaces(ksdef, ks2)
yield self.client.system_drop_keyspace(ksdef.name)
yield self.assertFailure(self.client.describe_keyspace(T_KEYSPACE), NotFoundException)
if DO_SYSTEM_RENAMING:
yield self.assertFailure(self.client.describe_keyspace(ksdef.name), NotFoundException)
@defer.inlineCallbacks
def test_column_family_manipulation(self):
cfdef = CfDef(KEYSPACE, T_CF,
column_type='Standard',
comparator_type='org.apache.cassandra.db.marshal.BytesType',
comment='foo',
row_cache_size=0.0,
key_cache_size=200000.0,
read_repair_chance=1.0,
column_metadata=[],
gc_grace_seconds=86400,
default_validation_class='org.apache.cassandra.db.marshal.BytesType',
key_validation_class='org.apache.cassandra.db.marshal.BytesType',
min_compaction_threshold=5,
max_compaction_threshold=31,
row_cache_save_period_in_seconds=0,
key_cache_save_period_in_seconds=3600,
memtable_flush_after_mins=60,
memtable_throughput_in_mb=249,
memtable_operations_in_millions=1.1671875,
replicate_on_write=False,
merge_shards_chance=0.10000000000000001,
row_cache_provider=None,
key_alias=None,
)
post_07_fields = ['replicate_on_write', 'merge_shards_chance',
'key_validation_class', 'row_cache_provider', 'key_alias']
yield self.client.system_add_column_family(cfdef)
ksdef = yield self.client.describe_keyspace(KEYSPACE)
cfdef2 = [c for c in ksdef.cf_defs if c.name == T_CF][0]
for field in post_07_fields:
# Most of these are ignored in 0.7, so we can't reliably compare them
setattr(cfdef, field, None)
setattr(cfdef2, field, None)
# we don't know the id ahead of time. copy the new one so the equality
# comparison won't fail
cfdef.id = cfdef2.id
self.assertEqual(cfdef, cfdef2)
if DO_SYSTEM_RENAMING:
newname = T_CF + '2'
yield self.client.system_rename_column_family(T_CF, newname)
ksdef = yield self.client.describe_keyspace(KEYSPACE)
cfdef2 = [c for c in ksdef.cf_defs if c.name == newname][0]
self.assertNotIn(T_CF, [c.name for c in ksdef.cf_defs])
cfdef.name = newname
self.assertEqual(cfdef, cfdef2)
yield self.client.system_drop_column_family(cfdef.name)
ksdef = yield self.client.describe_keyspace(KEYSPACE)
self.assertNotIn(cfdef.name, [c.name for c in ksdef.cf_defs])
@defer.inlineCallbacks
def test_describes(self):
name = yield self.client.describe_cluster_name()
self.assertIsInstance(name, str)
self.assertNotEqual(name, '')
partitioner = yield self.client.describe_partitioner()
self.assert_(partitioner.startswith('org.apache.cassandra.'),
msg='partitioner is %r' % partitioner)
snitch = yield self.client.describe_snitch()
self.assert_(snitch.startswith('org.apache.cassandra.'),
msg='snitch is %r' % snitch)
version = yield self.client.describe_version()
self.assertIsInstance(version, str)
self.assertIn('.', version)
schemavers = yield self.client.describe_schema_versions()
self.assertIsInstance(schemavers, dict)
self.assertNotEqual(schemavers, {})
ring = yield self.client.describe_ring(KEYSPACE)
self.assertIsInstance(ring, list)
self.assertNotEqual(ring, [])
for r in ring:
self.assertIsInstance(r.start_token, str)
self.assertIsInstance(r.end_token, str)
self.assertIsInstance(r.endpoints, list)
self.assertNotEqual(r.endpoints, [])
for ep in r.endpoints:
self.assertIsInstance(ep, str)
@defer.inlineCallbacks
def test_errback(self):
yield self.client.remove('poiqwe', CF)
try:
yield self.client.get('poiqwe', CF, column='foo')
except Exception, e:
pass
class CassandraSchemaManager(object):
"""Manages creation and destruction of cassandra schemas.
Useful for both testing and production
"""
def __init__(self, keyspace_def, error_if_existing=False):
self.keyspace_def = keyspace_def
self.error_if_existing=error_if_existing
self.created_keyspace = False
self.created_cfs = []
self.client = None
self.manager = None
self.connector = None
def connect(self, host=None, port=9160, username=None, password=None):
if not host:
host, port = get_host_port()
if username or password:
if not (username and password):
raise CassandraConfigurationError(
"Specify both username and password or neither")
else:
username, password = get_credentials()
authz = dict(username=username, password=password)
self.manager = ManagedCassandraClientFactory(credentials=authz,
check_api_version=True)
self.connector = reactor.connectTCP(host, port, self.manager)
self.client = CassandraClient(self.manager)
def disconnect(self):
if self.manager:
self.manager.shutdown()
if self.connector:
self.connector.disconnect()
@timeout(DEFAULT_CASSANDRA_TIMEOUT)
@defer.inlineCallbacks
def create(self, truncate=False):
if not self.client:
self.connect()
keyspace = self.keyspace_def
try:
existing = yield self.client.describe_keyspace(keyspace.name)
except NotFoundException:
existing = None
# keyspace already exists
if existing:
yield self.client.set_keyspace(keyspace.name)
_compare_ks_properties(existing, keyspace)
existing_cfs = dict((cf.name, cf) for cf in existing.cf_defs)
for cf in keyspace.cf_defs:
if cf.name in existing_cfs:
if truncate:
# in truncate mode we drop and readd any existing CFs.
yield self.client.system_drop_column_family(cf.name)
yield self.client.system_add_column_family(cf)
else:
_compare_cf_properties(existing_cfs[cf.name], cf)
else:
if cf.keyspace != keyspace.name:
raise CassandraSchemaError(
"CF %s has wrong keyspace name", cf.name)
self.created_cfs.append(cf.name)
yield self.client.system_add_column_family(cf)
else:
self.created_keyspace = True
yield self.client.system_add_keyspace(keyspace)
yield self.client.set_keyspace(keyspace.name)
@timeout(DEFAULT_CASSANDRA_TIMEOUT)
@defer.inlineCallbacks
def teardown(self):
if self.created_keyspace:
yield self.client.system_drop_keyspace(self.keyspace_def.name)
elif self.created_cfs:
for cf in self.created_cfs:
yield self.client.system_drop_column_family(cf)
class CassandraClientTest(unittest.TestCase):
def setUp(self):
self.factory = ManagedCassandraClientFactory()
self.client = CassandraClient(self.factory, KEYSPACE)
self.conns = []
for i in xrange(CONNS):
conn = reactor.connectTCP(HOST, PORT, self.factory)
self.conns.append(conn)
@defer.inlineCallbacks
def tearDown(self):
yield self.client.remove('test', CF)
yield self.client.remove('test2', CF)
yield self.client.remove('test', SCF)
yield self.client.remove('test2', SCF)
self.factory.shutDown()
for conn in self.conns:
yield conn.disconnect()
@defer.inlineCallbacks
def test_insert_get(self):
yield self.client.insert('test', ColumnPath(CF, None, COLUMN), 'testval')
yield self.client.insert('test2', CF, 'testval2', column=COLUMN)
yield self.client.insert('test', ColumnPath(SCF, SCOLUMN, COLUMN), 'superval')
yield self.client.insert('test2', SCF, 'superval2', column=COLUMN,
super_column=SCOLUMN)
res = yield self.client.get('test', CF, column=COLUMN)
self.assertEqual(res.column.value, 'testval')
res = yield self.client.get('test2', CF, column=COLUMN)
self.assertEqual(res.column.value, 'testval2')
res = yield self.client.get('test', SCF, column=COLUMN, super_column=SCOLUMN)
self.assertEqual(res.column.value, 'superval')
res = yield self.client.get('test2', SCF, column=COLUMN, super_column=SCOLUMN)
self.assertEqual(res.column.value, 'superval2')
@defer.inlineCallbacks
def test_batch_insert_get_slice_and_count(self):
yield self.client.batch_insert('test', CF,
{COLUMN: 'test', COLUMN2: 'test2'})
yield self.client.batch_insert('test', SCF,
{SCOLUMN: {COLUMN: 'test', COLUMN2: 'test2'}})
res = yield self.client.get_slice('test', CF, names=(COLUMN, COLUMN2))
self.assertEqual(res[0].column.value, 'test')
self.assertEqual(res[1].column.value, 'test2')
res = yield self.client.get_slice('test', SCF, names=(COLUMN, COLUMN2),
super_column=SCOLUMN)
self.assertEqual(res[0].column.value, 'test')
self.assertEqual(res[1].column.value, 'test2')
res = yield self.client.get_count('test', CF)
self.assertEqual(res, 2)
@defer.inlineCallbacks
def test_multiget_slice_remove(self):
yield self.client.insert('test', CF, 'testval', column=COLUMN)
yield self.client.insert('test', CF, 'testval', column=COLUMN2)
yield self.client.insert('test2', CF, 'testval2', column=COLUMN)
res = yield self.client.multiget(['test', 'test2'], CF, column=COLUMN)
self.assertEqual(res['test'].column.value, 'testval')
self.assertEqual(res['test2'].column.value, 'testval2')
res = yield self.client.multiget_slice(['test', 'test2'], CF)
self.assertEqual(res['test'][0].column.value, 'testval')
self.assertEqual(res['test'][1].column.value, 'testval')
self.assertEqual(res['test2'][0].column.value, 'testval2')
yield self.client.remove('test', CF, column=COLUMN)
yield self.client.remove('test2', CF, column=COLUMN)
res = yield self.client.multiget(['test', 'test2'], CF, column=COLUMN)
self.assertEqual(res['test'].column, None)
self.assertEqual(res['test2'].column, None)
class CassandraClientTest(unittest.TestCase):
@defer.inlineCallbacks
def setUp(self):
self.cmanager = ManagedCassandraClientFactory(keyspace='system')
self.client = CassandraClient(self.cmanager)
for i in xrange(CONNS):
reactor.connectTCP(HOST, PORT, self.cmanager)
yield self.cmanager.deferred
remote_ver = yield self.client.describe_version()
self.version = tuple(map(int, remote_ver.split('.')))
self.my_keyspace = ttypes.KsDef(
name=KEYSPACE,
strategy_class='org.apache.cassandra.locator.SimpleStrategy',
strategy_options={},
cf_defs=[
ttypes.CfDef(
keyspace=KEYSPACE,
name=CF,
column_type='Standard'
),
ttypes.CfDef(
keyspace=KEYSPACE,
name=SCF,
column_type='Super'
),
ttypes.CfDef(
keyspace=KEYSPACE,
name=IDX_CF,
column_type='Standard',
comparator_type='org.apache.cassandra.db.marshal.UTF8Type',
column_metadata=[
ttypes.ColumnDef(
name='col1',
validation_class='org.apache.cassandra.db.marshal.UTF8Type',
index_type=ttypes.IndexType.KEYS,
index_name='idxCol1')
],
default_validation_class='org.apache.cassandra.db.marshal.BytesType'
),
]
)
if self.version <= KS_RF_ATTRIBUTE:
self.my_keyspace.replication_factor = 1
else:
self.my_keyspace.strategy_options['replication_factor'] = '1'
if self.version >= COUNTERS_SUPPORTED_API:
self.my_keyspace.cf_defs.extend([
ttypes.CfDef(
keyspace=KEYSPACE,
name=COUNTER_CF,
column_type='Standard',
default_validation_class='org.apache.cassandra.db.marshal.CounterColumnType'
),
ttypes.CfDef(
keyspace=KEYSPACE,
name=SUPERCOUNTER_CF,
column_type='Super',
default_validation_class='org.apache.cassandra.db.marshal.CounterColumnType'
),
])
yield self.client.system_add_keyspace(self.my_keyspace)
yield self.client.set_keyspace(KEYSPACE)
@defer.inlineCallbacks
def tearDown(self):
yield self.client.system_drop_keyspace(self.my_keyspace.name)
self.cmanager.shutdown()
for c in reactor.getDelayedCalls():
c.cancel()
reactor.removeAll()
@defer.inlineCallbacks
def test_insert_get(self):
yield self.client.insert('test', CF, 'testval', column=COLUMN)
yield self.client.insert('test2', CF, 'testval2', column=COLUMN)
yield self.client.insert('test', SCF, 'superval', column=COLUMN, super_column=SCOLUMN)
yield self.client.insert('test2', SCF, 'superval2', column=COLUMN,
super_column=SCOLUMN)
res = yield self.client.get('test', CF, column=COLUMN)
self.assertEqual(res.column.value, 'testval')
res = yield self.client.get('test2', CF, column=COLUMN)
self.assertEqual(res.column.value, 'testval2')
res = yield self.client.get('test', SCF, column=COLUMN, super_column=SCOLUMN)
self.assertEqual(res.column.value, 'superval')
res = yield self.client.get('test2', SCF, column=COLUMN, super_column=SCOLUMN)
self.assertEqual(res.column.value, 'superval2')
@defer.inlineCallbacks
def test_batch_insert_get_slice_and_count(self):
yield self.client.batch_insert('test', CF,
{COLUMN: 'test', COLUMN2: 'test2'})
yield self.client.batch_insert('test', SCF,
{SCOLUMN: {COLUMN: 'test', COLUMN2: 'test2'}})
res = yield self.client.get_slice('test', CF, names=(COLUMN, COLUMN2))
self.assertEqual(res[0].column.value, 'test')
self.assertEqual(res[1].column.value, 'test2')
res = yield self.client.get_slice('test', SCF, names=(COLUMN, COLUMN2),
super_column=SCOLUMN)
self.assertEqual(res[0].column.value, 'test')
self.assertEqual(res[1].column.value, 'test2')
res = yield self.client.get_count('test', CF)
self.assertEqual(res, 2)
@defer.inlineCallbacks
def test_batch_mutate_and_remove(self):
yield self.client.batch_mutate({'test': {CF: {COLUMN: 'test', COLUMN2: 'test2'}, SCF: { SCOLUMN: { COLUMN: 'test', COLUMN2: 'test2'} } }, 'test2': {CF: {COLUMN: 'test', COLUMN2: 'test2'}, SCF: { SCOLUMN: { COLUMN: 'test', COLUMN2: 'test2'} } } })
res = yield self.client.get_slice('test', CF, names=(COLUMN, COLUMN2))
self.assertEqual(res[0].column.value, 'test')
self.assertEqual(res[1].column.value, 'test2')
res = yield self.client.get_slice('test2', CF, names=(COLUMN, COLUMN2))
self.assertEqual(res[0].column.value, 'test')
self.assertEqual(res[1].column.value, 'test2')
res = yield self.client.get_slice('test', SCF, names=(COLUMN, COLUMN2),
super_column=SCOLUMN)
self.assertEqual(res[0].column.value, 'test')
self.assertEqual(res[1].column.value, 'test2')
res = yield self.client.get_slice('test2', SCF, names=(COLUMN, COLUMN2),
super_column=SCOLUMN)
self.assertEqual(res[0].column.value, 'test')
self.assertEqual(res[1].column.value, 'test2')
yield self.client.batch_remove({CF: ['test', 'test2']}, names=['test', 'test2'])
yield self.client.batch_remove({SCF: ['test', 'test2']}, names=['test', 'test2'], supercolumn=SCOLUMN)
@defer.inlineCallbacks
def test_batch_mutate_with_deletion(self):
yield self.client.batch_mutate({'test': {CF: {COLUMN: 'test', COLUMN2: 'test2'}}})
res = yield self.client.get_slice('test', CF, names=(COLUMN, COLUMN2))
self.assertEqual(res[0].column.value, 'test')
self.assertEqual(res[1].column.value, 'test2')
yield self.client.batch_mutate({'test': {CF: {COLUMN: None, COLUMN2: 'test3'}}})
res = yield self.client.get_slice('test', CF, names=(COLUMN, COLUMN2))
self.assertEqual(len(res), 1)
self.assertEqual(res[0].column.value, 'test3')
@defer.inlineCallbacks
def test_multiget_slice_remove(self):
yield self.client.insert('test', CF, 'testval', column=COLUMN)
yield self.client.insert('test', CF, 'testval', column=COLUMN2)
yield self.client.insert('test2', CF, 'testval2', column=COLUMN)
res = yield self.client.multiget(['test', 'test2'], CF, column=COLUMN)
self.assertEqual(res['test'][0].column.value, 'testval')
self.assertEqual(res['test2'][0].column.value, 'testval2')
res = yield self.client.multiget_slice(['test', 'test2'], CF)
self.assertEqual(res['test'][0].column.value, 'testval')
self.assertEqual(res['test'][1].column.value, 'testval')
self.assertEqual(res['test2'][0].column.value, 'testval2')
yield self.client.remove('test', CF, column=COLUMN)
yield self.client.remove('test2', CF, column=COLUMN)
res = yield self.client.multiget(['test', 'test2'], CF, column=COLUMN)
self.assertEqual(len(res['test']), 0)
self.assertEqual(len(res['test2']), 0)
@defer.inlineCallbacks
def test_range_slices(self):
yield self.client.insert('test', CF, 'testval', column=COLUMN)
yield self.client.insert('test', CF, 'testval', column=COLUMN2)
yield self.client.insert('test2', CF, 'testval2', column=COLUMN)
ks = yield self.client.get_range_slices(CF, start='', finish='')
keys = [k.key for k in ks]
for key in ['test', 'test2']:
self.assertIn(key, keys)
@defer.inlineCallbacks
def test_indexed_slices(self):
yield self.client.insert('test1', IDX_CF, 'one', column='col1')
yield self.client.insert('test2', IDX_CF, 'two', column='col1')
yield self.client.insert('test3', IDX_CF, 'three', column='col1')
expressions = [ttypes.IndexExpression('col1', ttypes.IndexOperator.EQ, 'two')]
res = yield self.client.get_indexed_slices(IDX_CF, expressions, start_key='')
self.assertEquals(res[0].columns[0].column.value,'two')
@defer.inlineCallbacks
def test_counter_add(self):
if self.version < COUNTERS_SUPPORTED_API:
raise unittest.SkipTest('Counters are not supported before 0.8')
# test standard column counter
yield self.client.add('test', COUNTER_CF, 1, column='col')
res = yield self.client.get('test', COUNTER_CF, column='col')
self.assertEquals(res.counter_column.value, 1)
yield self.client.add('test', COUNTER_CF, 1, column='col')
res = yield self.client.get('test', COUNTER_CF, column='col')
self.assertEquals(res.counter_column.value, 2)
# test super column counters
yield self.client.add('test', SUPERCOUNTER_CF, 1, column='col', super_column='scol')
res = yield self.client.get('test', SUPERCOUNTER_CF, column='col', super_column='scol')
self.assertEquals(res.counter_column.value, 1)
yield self.client.add('test', SUPERCOUNTER_CF, 1, column='col', super_column='scol')
res = yield self.client.get('test', SUPERCOUNTER_CF, column='col', super_column='scol')
self.assertEquals(res.counter_column.value, 2)
@defer.inlineCallbacks
def test_counter_remove(self):
if self.version < COUNTERS_SUPPORTED_API:
raise unittest.SkipTest('Counters are not supported before 0.8')
# test standard column counter
yield self.client.add('test', COUNTER_CF, 1, column='col')
res = yield self.client.get('test', COUNTER_CF, column='col')
self.assertEquals(res.counter_column.value, 1)
yield self.client.remove_counter('test', COUNTER_CF, column='col')
yield self.assertFailure(self.client.get('test', COUNTER_CF, column='col'),
ttypes.NotFoundException)
# test super column counters
yield self.client.add('test', SUPERCOUNTER_CF, 1, column='col', super_column='scol')
res = yield self.client.get('test', SUPERCOUNTER_CF, column='col', super_column='scol')
self.assertEquals(res.counter_column.value, 1)
yield self.client.remove_counter('test', SUPERCOUNTER_CF,
column='col', super_column='scol')
yield self.assertFailure(self.client.get('test', SUPERCOUNTER_CF,
column='col', super_column='scol'),
ttypes.NotFoundException)
def sleep(self, secs):
d = defer.Deferred()
reactor.callLater(secs, d.callback, None)
return d
@defer.inlineCallbacks
def test_ttls(self):
yield self.client.insert('test_ttls', CF, 'testval', column=COLUMN, ttl=1)
res = yield self.client.get('test_ttls', CF, column=COLUMN)
self.assertEqual(res.column.value, 'testval')
yield self.sleep(2)
yield self.assertFailure(self.client.get('test_ttls', CF, column=COLUMN), ttypes.NotFoundException)
yield self.client.batch_insert('test_ttls', CF, {COLUMN:'testval'}, ttl=1)
res = yield self.client.get('test_ttls', CF, column=COLUMN)
self.assertEqual(res.column.value, 'testval')
yield self.sleep(2)
yield self.assertFailure(self.client.get('test_ttls', CF, column=COLUMN), ttypes.NotFoundException)
yield self.client.batch_mutate({'test_ttls': {CF: {COLUMN: 'testval'}}}, ttl=1)
res = yield self.client.get('test_ttls', CF, column=COLUMN)
self.assertEqual(res.column.value, 'testval')
yield self.sleep(2)
yield self.assertFailure(self.client.get('test_ttls', CF, column=COLUMN), ttypes.NotFoundException)
def compare_keyspaces(self, ks1, ks2):
self.assertEqual(ks1.name, ks2.name)
self.assertEqual(ks1.strategy_class, ks2.strategy_class)
self.assertEqual(ks1.cf_defs, ks2.cf_defs)
def get_rf(ksdef):
rf = ksdef.replication_factor
if ksdef.strategy_options and \
'replication_factor' in ksdef.strategy_options:
rf = int(ksdef.strategy_options['replication_factor'])
return rf
def strat_opts_no_rf(ksdef):
if not ksdef.strategy_options:
return {}
opts = ksdef.strategy_options.copy()
if 'replication_factor' in ksdef.strategy_options:
del opts['replication_factor']
return opts
self.assertEqual(get_rf(ks1), get_rf(ks2))
self.assertEqual(strat_opts_no_rf(ks1), strat_opts_no_rf(ks2))
@defer.inlineCallbacks
def test_keyspace_manipulation(self):
try:
yield self.client.system_drop_keyspace(T_KEYSPACE)
except ttypes.InvalidRequestException:
pass
ksdef = ttypes.KsDef(name=T_KEYSPACE, strategy_class='org.apache.cassandra.locator.SimpleStrategy', strategy_options={}, cf_defs=[])
if self.version <= KS_RF_ATTRIBUTE:
ksdef.replication_factor = 1
else:
ksdef.strategy_options['replication_factor'] = '1'
yield self.client.system_add_keyspace(ksdef)
ks2 = yield self.client.describe_keyspace(T_KEYSPACE)
self.compare_keyspaces(ksdef, ks2)
if DO_SYSTEM_RENAMING:
newname = T_KEYSPACE + '2'
yield self.client.system_rename_keyspace(T_KEYSPACE, newname)
ks2 = yield self.client.describe_keyspace(newname)
ksdef.name = newname
self.compare_keyspaces(ksdef, ks2)
yield self.client.system_drop_keyspace(ksdef.name)
yield self.assertFailure(self.client.describe_keyspace(T_KEYSPACE), ttypes.NotFoundException)
if DO_SYSTEM_RENAMING:
yield self.assertFailure(self.client.describe_keyspace(ksdef.name), ttypes.NotFoundException)
@defer.inlineCallbacks
def test_column_family_manipulation(self):
# CfDef attributes present in all supported c*/thrift-api versions
common_attrs = (
('column_type', 'Standard'),
('comparator_type', 'org.apache.cassandra.db.marshal.BytesType'),
('comment', 'foo'),
('read_repair_chance', 1.0),
('column_metadata', []),
('gc_grace_seconds', 86400),
('default_validation_class', 'org.apache.cassandra.db.marshal.BytesType'),
('min_compaction_threshold', 5),
('max_compaction_threshold', 31),
)
cfdef = ttypes.CfDef(KEYSPACE, T_CF)
for attr, val in common_attrs:
setattr(cfdef, attr, val)
yield self.client.system_add_column_family(cfdef)
ksdef = yield self.client.describe_keyspace(KEYSPACE)
cfdefs = [c for c in ksdef.cf_defs if c.name == T_CF]
self.assertEqual(len(cfdefs), 1)
cfdef2 = cfdefs[0]
for attr, val in common_attrs:
val1 = getattr(cfdef, attr)
val2 = getattr(cfdef2, attr)
self.assertEqual(val1, val2, 'attribute %s mismatch: %r != %r' % (attr, val1, val2))
if DO_SYSTEM_RENAMING:
newname = T_CF + '2'
yield self.client.system_rename_column_family(T_CF, newname)
ksdef = yield self.client.describe_keyspace(KEYSPACE)
cfdef2 = [c for c in ksdef.cf_defs if c.name == newname][0]
self.assertNotIn(T_CF, [c.name for c in ksdef.cf_defs])
cfdef.name = newname
self.assertEqual(cfdef, cfdef2)
yield self.client.system_drop_column_family(cfdef.name)
ksdef = yield self.client.describe_keyspace(KEYSPACE)
self.assertNotIn(cfdef.name, [c.name for c in ksdef.cf_defs])
@defer.inlineCallbacks
def test_describes(self):
name = yield self.client.describe_cluster_name()
self.assertIsInstance(name, str)
self.assertNotEqual(name, '')
partitioner = yield self.client.describe_partitioner()
self.assert_(partitioner.startswith('org.apache.cassandra.'),
msg='partitioner is %r' % partitioner)
snitch = yield self.client.describe_snitch()
self.assert_(snitch.startswith('org.apache.cassandra.'),
msg='snitch is %r' % snitch)
version = yield self.client.describe_version()
self.assertIsInstance(version, str)
self.assertIn('.', version)
schemavers = yield self.client.describe_schema_versions()
self.assertIsInstance(schemavers, dict)
self.assertNotEqual(schemavers, {})
ring = yield self.client.describe_ring(KEYSPACE)
self.assertIsInstance(ring, list)
self.assertNotEqual(ring, [])
for r in ring:
self.assertIsInstance(r.start_token, str)
self.assertIsInstance(r.end_token, str)
self.assertIsInstance(r.endpoints, list)
self.assertNotEqual(r.endpoints, [])
for ep in r.endpoints:
self.assertIsInstance(ep, str)
@defer.inlineCallbacks
def test_errback(self):
yield self.client.remove('poiqwe', CF)
try:
yield self.client.get('poiqwe', CF, column='foo')
except Exception, e:
pass
class CassandraClientTest(unittest.TestCase):
@defer.inlineCallbacks
def setUp(self):
self.cmanager = ManagedCassandraClientFactory(keyspace='system')
self.client = CassandraClient(self.cmanager)
for i in xrange(CONNS):
reactor.connectTCP(HOST, PORT, self.cmanager)
yield self.cmanager.deferred
self.my_keyspace = KsDef(
name=KEYSPACE,
strategy_class='org.apache.cassandra.locator.SimpleStrategy',
replication_factor=1,
cf_defs=[
CfDef(
keyspace=KEYSPACE,
name=CF,
column_type='Standard'
),
CfDef(
keyspace=KEYSPACE,
name=SCF,
column_type='Super'
),
CfDef(
keyspace=KEYSPACE,
name=IDX_CF,
column_type='Standard',
comparator_type='org.apache.cassandra.db.marshal.UTF8Type',
column_metadata=[
ColumnDef(
name='col1',
validation_class='org.apache.cassandra.db.marshal.UTF8Type',
index_type=IndexType.KEYS,
index_name='idxCol1')
],
default_validation_class='org.apache.cassandra.db.marshal.BytesType')
]
)
yield self.client.system_add_keyspace(self.my_keyspace)
yield self.client.set_keyspace(KEYSPACE)
@defer.inlineCallbacks
def tearDown(self):
yield self.client.system_drop_keyspace(self.my_keyspace.name)
self.cmanager.shutdown()
for c in reactor.getDelayedCalls():
c.cancel()
reactor.removeAll()
@defer.inlineCallbacks
def test_insert_get(self):
yield self.client.insert('test', CF, 'testval', column=COLUMN)
yield self.client.insert('test2', CF, 'testval2', column=COLUMN)
yield self.client.insert('test', SCF, 'superval', column=COLUMN, super_column=SCOLUMN)
yield self.client.insert('test2', SCF, 'superval2', column=COLUMN,
super_column=SCOLUMN)
res = yield self.client.get('test', CF, column=COLUMN)
self.assertEqual(res.column.value, 'testval')
res = yield self.client.get('test2', CF, column=COLUMN)
self.assertEqual(res.column.value, 'testval2')
res = yield self.client.get('test', SCF, column=COLUMN, super_column=SCOLUMN)
self.assertEqual(res.column.value, 'superval')
res = yield self.client.get('test2', SCF, column=COLUMN, super_column=SCOLUMN)
self.assertEqual(res.column.value, 'superval2')
@defer.inlineCallbacks
def test_batch_insert_get_slice_and_count(self):
yield self.client.batch_insert('test', CF,
{COLUMN: 'test', COLUMN2: 'test2'})
yield self.client.batch_insert('test', SCF,
{SCOLUMN: {COLUMN: 'test', COLUMN2: 'test2'}})
res = yield self.client.get_slice('test', CF, names=(COLUMN, COLUMN2))
self.assertEqual(res[0].column.value, 'test')
self.assertEqual(res[1].column.value, 'test2')
res = yield self.client.get_slice('test', SCF, names=(COLUMN, COLUMN2),
super_column=SCOLUMN)
self.assertEqual(res[0].column.value, 'test')
self.assertEqual(res[1].column.value, 'test2')
res = yield self.client.get_count('test', CF)
self.assertEqual(res, 2)
@defer.inlineCallbacks
def test_batch_mutate_and_remove(self):
yield self.client.batch_mutate({'test': {CF: {COLUMN: 'test', COLUMN2: 'test2'}, SCF: { SCOLUMN: { COLUMN: 'test', COLUMN2: 'test2'} } }, 'test2': {CF: {COLUMN: 'test', COLUMN2: 'test2'}, SCF: { SCOLUMN: { COLUMN: 'test', COLUMN2: 'test2'} } } })
res = yield self.client.get_slice('test', CF, names=(COLUMN, COLUMN2))
self.assertEqual(res[0].column.value, 'test')
self.assertEqual(res[1].column.value, 'test2')
res = yield self.client.get_slice('test2', CF, names=(COLUMN, COLUMN2))
self.assertEqual(res[0].column.value, 'test')
self.assertEqual(res[1].column.value, 'test2')
res = yield self.client.get_slice('test', SCF, names=(COLUMN, COLUMN2),
super_column=SCOLUMN)
self.assertEqual(res[0].column.value, 'test')
self.assertEqual(res[1].column.value, 'test2')
res = yield self.client.get_slice('test2', SCF, names=(COLUMN, COLUMN2),
super_column=SCOLUMN)
self.assertEqual(res[0].column.value, 'test')
self.assertEqual(res[1].column.value, 'test2')
yield self.client.batch_remove({CF: ['test', 'test2']}, names=['test', 'test2'])
yield self.client.batch_remove({SCF: ['test', 'test2']}, names=['test', 'test2'], supercolumn=SCOLUMN)
@defer.inlineCallbacks
def test_batch_mutate_with_deletion(self):
yield self.client.batch_mutate({'test': {CF: {COLUMN: 'test', COLUMN2: 'test2'}}})
res = yield self.client.get_slice('test', CF, names=(COLUMN, COLUMN2))
self.assertEqual(res[0].column.value, 'test')
self.assertEqual(res[1].column.value, 'test2')
yield self.client.batch_mutate({'test': {CF: {COLUMN: None, COLUMN2: 'test3'}}})
res = yield self.client.get_slice('test', CF, names=(COLUMN, COLUMN2))
self.assertEqual(len(res), 1)
self.assertEqual(res[0].column.value, 'test3')
@defer.inlineCallbacks
def test_multiget_slice_remove(self):
yield self.client.insert('test', CF, 'testval', column=COLUMN)
yield self.client.insert('test', CF, 'testval', column=COLUMN2)
yield self.client.insert('test2', CF, 'testval2', column=COLUMN)
res = yield self.client.multiget(['test', 'test2'], CF, column=COLUMN)
self.assertEqual(res['test'][0].column.value, 'testval')
self.assertEqual(res['test2'][0].column.value, 'testval2')
res = yield self.client.multiget_slice(['test', 'test2'], CF)
self.assertEqual(res['test'][0].column.value, 'testval')
self.assertEqual(res['test'][1].column.value, 'testval')
self.assertEqual(res['test2'][0].column.value, 'testval2')
yield self.client.remove('test', CF, column=COLUMN)
yield self.client.remove('test2', CF, column=COLUMN)
res = yield self.client.multiget(['test', 'test2'], CF, column=COLUMN)
self.assertEqual(len(res['test']), 0)
self.assertEqual(len(res['test2']), 0)
@defer.inlineCallbacks
def test_range_slices(self):
yield self.client.insert('test', CF, 'testval', column=COLUMN)
yield self.client.insert('test', CF, 'testval', column=COLUMN2)
yield self.client.insert('test2', CF, 'testval2', column=COLUMN)
ks = yield self.client.get_range_slices(CF, start='', finish='')
keys = [k.key for k in ks]
for key in ['test', 'test2']:
self.assertIn(key, keys)
@defer.inlineCallbacks
def test_indexed_slices(self):
yield self.client.insert('test1', IDX_CF, 'one', column='col1')
yield self.client.insert('test2', IDX_CF, 'two', column='col1')
yield self.client.insert('test3', IDX_CF, 'three', column='col1')
expressions = [IndexExpression('col1', IndexOperator.EQ, 'two')]
res = yield self.client.get_indexed_slices(IDX_CF, expressions, start_key='')
self.assertEquals(res[0].columns[0].column.value,'two')
def sleep(self, secs):
d = defer.Deferred()
reactor.callLater(secs, d.callback, None)
return d
@defer.inlineCallbacks
def test_ttls(self):
yield self.client.insert('test_ttls', CF, 'testval', column=COLUMN, ttl=1)
res = yield self.client.get('test_ttls', CF, column=COLUMN)
self.assertEqual(res.column.value, 'testval')
yield self.sleep(2)
yield self.assertFailure(self.client.get('test_ttls', CF, column=COLUMN), NotFoundException)
yield self.client.batch_insert('test_ttls', CF, {COLUMN:'testval'}, ttl=1)
res = yield self.client.get('test_ttls', CF, column=COLUMN)
self.assertEqual(res.column.value, 'testval')
yield self.sleep(2)
yield self.assertFailure(self.client.get('test_ttls', CF, column=COLUMN), NotFoundException)
yield self.client.batch_mutate({'test_ttls': {CF: {COLUMN: 'testval'}}}, ttl=1)
res = yield self.client.get('test_ttls', CF, column=COLUMN)
self.assertEqual(res.column.value, 'testval')
yield self.sleep(2)
yield self.assertFailure(self.client.get('test_ttls', CF, column=COLUMN), NotFoundException)
@defer.inlineCallbacks
def test_keyspace_manipulation(self):
ksdef = KsDef(name=T_KEYSPACE, strategy_class='org.apache.cassandra.locator.SimpleStrategy', replication_factor=1, cf_defs=[])
yield self.client.system_add_keyspace(ksdef)
ks2 = yield self.client.describe_keyspace(T_KEYSPACE)
self.assertEqual(ksdef, ks2)
if DO_SYSTEM_RENAMING:
newname = T_KEYSPACE + '2'
yield self.client.system_rename_keyspace(T_KEYSPACE, newname)
ks2 = yield self.client.describe_keyspace(newname)
ksdef.name = newname
self.assertEqual(ksdef, ks2)
yield self.client.system_drop_keyspace(ksdef.name)
yield self.assertFailure(self.client.describe_keyspace(T_KEYSPACE), NotFoundException)
if DO_SYSTEM_RENAMING:
yield self.assertFailure(self.client.describe_keyspace(ksdef.name), NotFoundException)
@defer.inlineCallbacks
def test_column_family_manipulation(self):
cfdef = CfDef(KEYSPACE, T_CF,
column_type='Standard',
comparator_type='org.apache.cassandra.db.marshal.BytesType',
comment='foo',
row_cache_size=0.0,
key_cache_size=200000.0,
read_repair_chance=1.0,
column_metadata=[],
gc_grace_seconds=86400,
default_validation_class='org.apache.cassandra.db.marshal.BytesType',
min_compaction_threshold=5,
max_compaction_threshold=31,
row_cache_save_period_in_seconds=0,
key_cache_save_period_in_seconds=3600,
memtable_flush_after_mins=60,
memtable_throughput_in_mb=249,
memtable_operations_in_millions=1.1671875,
)
yield self.client.system_add_column_family(cfdef)
ksdef = yield self.client.describe_keyspace(KEYSPACE)
cfdef2 = [c for c in ksdef.cf_defs if c.name == T_CF][0]
# we don't know the id ahead of time. copy the new one so the equality
# comparison won't fail
cfdef.id = cfdef2.id
self.assertEqual(cfdef, cfdef2)
if DO_SYSTEM_RENAMING:
newname = T_CF + '2'
yield self.client.system_rename_column_family(T_CF, newname)
ksdef = yield self.client.describe_keyspace(KEYSPACE)
cfdef2 = [c for c in ksdef.cf_defs if c.name == newname][0]
self.assertNotIn(T_CF, [c.name for c in ksdef.cf_defs])
cfdef.name = newname
self.assertEqual(cfdef, cfdef2)
yield self.client.system_drop_column_family(cfdef.name)
ksdef = yield self.client.describe_keyspace(KEYSPACE)
self.assertNotIn(cfdef.name, [c.name for c in ksdef.cf_defs])
@defer.inlineCallbacks
def test_describes(self):
name = yield self.client.describe_cluster_name()
self.assertIsInstance(name, str)
self.assertNotEqual(name, '')
partitioner = yield self.client.describe_partitioner()
self.assert_(partitioner.startswith('org.apache.cassandra.'),
msg='partitioner is %r' % partitioner)
snitch = yield self.client.describe_snitch()
self.assert_(snitch.startswith('org.apache.cassandra.'),
msg='snitch is %r' % snitch)
version = yield self.client.describe_version()
self.assertIsInstance(version, str)
self.assertIn('.', version)
schemavers = yield self.client.describe_schema_versions()
self.assertIsInstance(schemavers, dict)
self.assertNotEqual(schemavers, {})
ring = yield self.client.describe_ring(KEYSPACE)
self.assertIsInstance(ring, list)
self.assertNotEqual(ring, [])
for r in ring:
self.assertIsInstance(r.start_token, str)
self.assertIsInstance(r.end_token, str)
self.assertIsInstance(r.endpoints, list)
self.assertNotEqual(r.endpoints, [])
for ep in r.endpoints:
self.assertIsInstance(ep, str)
@defer.inlineCallbacks
def test_errback(self):
yield self.client.remove('poiqwe', CF)
try:
yield self.client.get('poiqwe', CF, column='foo')
except Exception, e:
pass
class CassandraClientTest(unittest.TestCase):
@defer.inlineCallbacks
def setUp(self):
self.cmanager = ManagedCassandraClientFactory(keyspace='system')
self.client = CassandraClient(self.cmanager)
for i in xrange(CONNS):
reactor.connectTCP(HOST, PORT, self.cmanager)
yield self.cmanager.deferred
remote_ver = yield self.client.describe_version()
self.version = thrift_api_ver_to_cassandra_ver(remote_ver)
self.my_keyspace = KsDef(
name=KEYSPACE,
strategy_class='org.apache.cassandra.locator.SimpleStrategy',
strategy_options={'replication_factor': '1'},
cf_defs=[
CfDef(keyspace=KEYSPACE, name=CF, column_type='Standard'),
CfDef(keyspace=KEYSPACE, name=SCF, column_type='Super'),
CfDef(
keyspace=KEYSPACE,
name=IDX_CF,
column_type='Standard',
comparator_type='org.apache.cassandra.db.marshal.UTF8Type',
column_metadata=[
ColumnDef(name='col1',
validation_class=
'org.apache.cassandra.db.marshal.UTF8Type',
index_type=IndexType.KEYS,
index_name='idxCol1')
],
default_validation_class=
'org.apache.cassandra.db.marshal.BytesType'),
])
if self.version == CASSANDRA_08_VERSION:
self.my_keyspace.cf_defs.extend([
CfDef(keyspace=KEYSPACE,
name=COUNTER_CF,
column_type='Standard',
default_validation_class=
'org.apache.cassandra.db.marshal.CounterColumnType'),
CfDef(keyspace=KEYSPACE,
name=SUPERCOUNTER_CF,
column_type='Super',
default_validation_class=
'org.apache.cassandra.db.marshal.CounterColumnType'),
])
yield self.client.system_add_keyspace(self.my_keyspace)
yield self.client.set_keyspace(KEYSPACE)
@defer.inlineCallbacks
def tearDown(self):
yield self.client.system_drop_keyspace(self.my_keyspace.name)
self.cmanager.shutdown()
for c in reactor.getDelayedCalls():
c.cancel()
reactor.removeAll()
@defer.inlineCallbacks
def test_insert_get(self):
yield self.client.insert('test', CF, 'testval', column=COLUMN)
yield self.client.insert('test2', CF, 'testval2', column=COLUMN)
yield self.client.insert('test',
SCF,
'superval',
column=COLUMN,
super_column=SCOLUMN)
yield self.client.insert('test2',
SCF,
'superval2',
column=COLUMN,
super_column=SCOLUMN)
res = yield self.client.get('test', CF, column=COLUMN)
self.assertEqual(res.column.value, 'testval')
res = yield self.client.get('test2', CF, column=COLUMN)
self.assertEqual(res.column.value, 'testval2')
res = yield self.client.get('test',
SCF,
column=COLUMN,
super_column=SCOLUMN)
self.assertEqual(res.column.value, 'superval')
res = yield self.client.get('test2',
SCF,
column=COLUMN,
super_column=SCOLUMN)
self.assertEqual(res.column.value, 'superval2')
@defer.inlineCallbacks
def test_batch_insert_get_slice_and_count(self):
yield self.client.batch_insert('test', CF, {
COLUMN: 'test',
COLUMN2: 'test2'
})
yield self.client.batch_insert(
'test', SCF, {SCOLUMN: {
COLUMN: 'test',
COLUMN2: 'test2'
}})
res = yield self.client.get_slice('test', CF, names=(COLUMN, COLUMN2))
self.assertEqual(res[0].column.value, 'test')
self.assertEqual(res[1].column.value, 'test2')
res = yield self.client.get_slice('test',
SCF,
names=(COLUMN, COLUMN2),
super_column=SCOLUMN)
self.assertEqual(res[0].column.value, 'test')
self.assertEqual(res[1].column.value, 'test2')
res = yield self.client.get_count('test', CF)
self.assertEqual(res, 2)
@defer.inlineCallbacks
def test_batch_mutate_and_remove(self):
yield self.client.batch_mutate({
'test': {
CF: {
COLUMN: 'test',
COLUMN2: 'test2'
},
SCF: {
SCOLUMN: {
COLUMN: 'test',
COLUMN2: 'test2'
}
}
},
'test2': {
CF: {
COLUMN: 'test',
COLUMN2: 'test2'
},
SCF: {
SCOLUMN: {
COLUMN: 'test',
COLUMN2: 'test2'
}
}
}
})
res = yield self.client.get_slice('test', CF, names=(COLUMN, COLUMN2))
self.assertEqual(res[0].column.value, 'test')
self.assertEqual(res[1].column.value, 'test2')
res = yield self.client.get_slice('test2', CF, names=(COLUMN, COLUMN2))
self.assertEqual(res[0].column.value, 'test')
self.assertEqual(res[1].column.value, 'test2')
res = yield self.client.get_slice('test',
SCF,
names=(COLUMN, COLUMN2),
super_column=SCOLUMN)
self.assertEqual(res[0].column.value, 'test')
self.assertEqual(res[1].column.value, 'test2')
res = yield self.client.get_slice('test2',
SCF,
names=(COLUMN, COLUMN2),
super_column=SCOLUMN)
self.assertEqual(res[0].column.value, 'test')
self.assertEqual(res[1].column.value, 'test2')
yield self.client.batch_remove({CF: ['test', 'test2']},
names=['test', 'test2'])
yield self.client.batch_remove({SCF: ['test', 'test2']},
names=['test', 'test2'],
supercolumn=SCOLUMN)
@defer.inlineCallbacks
def test_batch_mutate_with_deletion(self):
yield self.client.batch_mutate(
{'test': {
CF: {
COLUMN: 'test',
COLUMN2: 'test2'
}
}})
res = yield self.client.get_slice('test', CF, names=(COLUMN, COLUMN2))
self.assertEqual(res[0].column.value, 'test')
self.assertEqual(res[1].column.value, 'test2')
yield self.client.batch_mutate(
{'test': {
CF: {
COLUMN: None,
COLUMN2: 'test3'
}
}})
res = yield self.client.get_slice('test', CF, names=(COLUMN, COLUMN2))
self.assertEqual(len(res), 1)
self.assertEqual(res[0].column.value, 'test3')
@defer.inlineCallbacks
def test_multiget_slice_remove(self):
yield self.client.insert('test', CF, 'testval', column=COLUMN)
yield self.client.insert('test', CF, 'testval', column=COLUMN2)
yield self.client.insert('test2', CF, 'testval2', column=COLUMN)
res = yield self.client.multiget(['test', 'test2'], CF, column=COLUMN)
self.assertEqual(res['test'][0].column.value, 'testval')
self.assertEqual(res['test2'][0].column.value, 'testval2')
res = yield self.client.multiget_slice(['test', 'test2'], CF)
self.assertEqual(res['test'][0].column.value, 'testval')
self.assertEqual(res['test'][1].column.value, 'testval')
self.assertEqual(res['test2'][0].column.value, 'testval2')
yield self.client.remove('test', CF, column=COLUMN)
yield self.client.remove('test2', CF, column=COLUMN)
res = yield self.client.multiget(['test', 'test2'], CF, column=COLUMN)
self.assertEqual(len(res['test']), 0)
self.assertEqual(len(res['test2']), 0)
@defer.inlineCallbacks
def test_range_slices(self):
yield self.client.insert('test', CF, 'testval', column=COLUMN)
yield self.client.insert('test', CF, 'testval', column=COLUMN2)
yield self.client.insert('test2', CF, 'testval2', column=COLUMN)
ks = yield self.client.get_range_slices(CF, start='', finish='')
keys = [k.key for k in ks]
for key in ['test', 'test2']:
self.assertIn(key, keys)
@defer.inlineCallbacks
def test_indexed_slices(self):
yield self.client.insert('test1', IDX_CF, 'one', column='col1')
yield self.client.insert('test2', IDX_CF, 'two', column='col1')
yield self.client.insert('test3', IDX_CF, 'three', column='col1')
expressions = [IndexExpression('col1', IndexOperator.EQ, 'two')]
res = yield self.client.get_indexed_slices(IDX_CF,
expressions,
start_key='')
self.assertEquals(res[0].columns[0].column.value, 'two')
@defer.inlineCallbacks
def test_counter_add(self):
if self.version != CASSANDRA_08_VERSION:
raise unittest.SkipTest('Counters are not supported in 0.7')
# test standard column counter
yield self.client.add('test', COUNTER_CF, 1, column='col')
res = yield self.client.get('test', COUNTER_CF, column='col')
self.assertEquals(res.counter_column.value, 1)
yield self.client.add('test', COUNTER_CF, 1, column='col')
res = yield self.client.get('test', COUNTER_CF, column='col')
self.assertEquals(res.counter_column.value, 2)
# test super column counters
yield self.client.add('test',
SUPERCOUNTER_CF,
1,
column='col',
super_column='scol')
res = yield self.client.get('test',
SUPERCOUNTER_CF,
column='col',
super_column='scol')
self.assertEquals(res.counter_column.value, 1)
yield self.client.add('test',
SUPERCOUNTER_CF,
1,
column='col',
super_column='scol')
res = yield self.client.get('test',
SUPERCOUNTER_CF,
column='col',
super_column='scol')
self.assertEquals(res.counter_column.value, 2)
@defer.inlineCallbacks
def test_counter_remove(self):
if self.version != CASSANDRA_08_VERSION:
raise unittest.SkipTest('Counters are not supported in 0.7')
# test standard column counter
yield self.client.add('test', COUNTER_CF, 1, column='col')
res = yield self.client.get('test', COUNTER_CF, column='col')
self.assertEquals(res.counter_column.value, 1)
yield self.client.remove_counter('test', COUNTER_CF, column='col')
yield self.assertFailure(
self.client.get('test', COUNTER_CF, column='col'),
NotFoundException)
# test super column counters
yield self.client.add('test',
SUPERCOUNTER_CF,
1,
column='col',
super_column='scol')
res = yield self.client.get('test',
SUPERCOUNTER_CF,
column='col',
super_column='scol')
self.assertEquals(res.counter_column.value, 1)
yield self.client.remove_counter('test',
SUPERCOUNTER_CF,
column='col',
super_column='scol')
yield self.assertFailure(
self.client.get('test',
SUPERCOUNTER_CF,
column='col',
super_column='scol'), NotFoundException)
def sleep(self, secs):
d = defer.Deferred()
reactor.callLater(secs, d.callback, None)
return d
@defer.inlineCallbacks
def test_ttls(self):
yield self.client.insert('test_ttls',
CF,
'testval',
column=COLUMN,
ttl=1)
res = yield self.client.get('test_ttls', CF, column=COLUMN)
self.assertEqual(res.column.value, 'testval')
yield self.sleep(2)
yield self.assertFailure(
self.client.get('test_ttls', CF, column=COLUMN), NotFoundException)
yield self.client.batch_insert('test_ttls',
CF, {COLUMN: 'testval'},
ttl=1)
res = yield self.client.get('test_ttls', CF, column=COLUMN)
self.assertEqual(res.column.value, 'testval')
yield self.sleep(2)
yield self.assertFailure(
self.client.get('test_ttls', CF, column=COLUMN), NotFoundException)
yield self.client.batch_mutate(
{'test_ttls': {
CF: {
COLUMN: 'testval'
}
}}, ttl=1)
res = yield self.client.get('test_ttls', CF, column=COLUMN)
self.assertEqual(res.column.value, 'testval')
yield self.sleep(2)
yield self.assertFailure(
self.client.get('test_ttls', CF, column=COLUMN), NotFoundException)
def compare_keyspaces(self, ks1, ks2):
self.assertEqual(ks1.name, ks2.name)
self.assertEqual(ks1.strategy_class, ks2.strategy_class)
self.assertEqual(ks1.cf_defs, ks2.cf_defs)
def get_rf(ksdef):
rf = ksdef.replication_factor
if ksdef.strategy_options and \
'replication_factor' in ksdef.strategy_options:
rf = int(ksdef.strategy_options['replication_factor'])
return rf
def strat_opts_no_rf(ksdef):
if not ksdef.strategy_options:
return {}
opts = ksdef.strategy_options.copy()
if 'replication_factor' in ksdef.strategy_options:
del opts['replication_factor']
return opts
self.assertEqual(get_rf(ks1), get_rf(ks2))
self.assertEqual(strat_opts_no_rf(ks1), strat_opts_no_rf(ks2))
@defer.inlineCallbacks
def test_keyspace_manipulation(self):
try:
yield self.client.system_drop_keyspace(T_KEYSPACE)
except InvalidRequestException:
pass
ksdef = KsDef(
name=T_KEYSPACE,
strategy_class='org.apache.cassandra.locator.SimpleStrategy',
strategy_options={'replication_factor': '1'},
cf_defs=[])
yield self.client.system_add_keyspace(ksdef)
ks2 = yield self.client.describe_keyspace(T_KEYSPACE)
self.compare_keyspaces(ksdef, ks2)
if DO_SYSTEM_RENAMING:
newname = T_KEYSPACE + '2'
yield self.client.system_rename_keyspace(T_KEYSPACE, newname)
ks2 = yield self.client.describe_keyspace(newname)
ksdef.name = newname
self.compare_keyspaces(ksdef, ks2)
yield self.client.system_drop_keyspace(ksdef.name)
yield self.assertFailure(self.client.describe_keyspace(T_KEYSPACE),
NotFoundException)
if DO_SYSTEM_RENAMING:
yield self.assertFailure(self.client.describe_keyspace(ksdef.name),
NotFoundException)
@defer.inlineCallbacks
def test_column_family_manipulation(self):
cfdef = CfDef(
KEYSPACE,
T_CF,
column_type='Standard',
comparator_type='org.apache.cassandra.db.marshal.BytesType',
comment='foo',
row_cache_size=0.0,
key_cache_size=200000.0,
read_repair_chance=1.0,
column_metadata=[],
gc_grace_seconds=86400,
default_validation_class=
'org.apache.cassandra.db.marshal.BytesType',
key_validation_class='org.apache.cassandra.db.marshal.BytesType',
min_compaction_threshold=5,
max_compaction_threshold=31,
row_cache_save_period_in_seconds=0,
key_cache_save_period_in_seconds=3600,
memtable_flush_after_mins=60,
memtable_throughput_in_mb=249,
memtable_operations_in_millions=1.1671875,
replicate_on_write=False,
merge_shards_chance=0.10000000000000001,
row_cache_provider=None,
key_alias=None,
)
post_07_fields = [
'replicate_on_write', 'merge_shards_chance',
'key_validation_class', 'row_cache_provider', 'key_alias'
]
post_08_fields = [
'memtable_throughput_in_mb', 'memtable_flush_after_mins',
'memtable_operations_in_millions'
]
yield self.client.system_add_column_family(cfdef)
ksdef = yield self.client.describe_keyspace(KEYSPACE)
cfdef2 = [c for c in ksdef.cf_defs if c.name == T_CF][0]
for field in post_07_fields:
# Most of these are ignored in 0.7, so we can't reliably compare them
setattr(cfdef, field, None)
setattr(cfdef2, field, None)
for field in post_08_fields:
# These fields change from 0.8 to 1.0
setattr(cfdef, field, None)
setattr(cfdef2, field, None)
# we don't know the id ahead of time. copy the new one so the equality
# comparison won't fail
cfdef.id = cfdef2.id
self.assertEqual(cfdef, cfdef2)
if DO_SYSTEM_RENAMING:
newname = T_CF + '2'
yield self.client.system_rename_column_family(T_CF, newname)
ksdef = yield self.client.describe_keyspace(KEYSPACE)
cfdef2 = [c for c in ksdef.cf_defs if c.name == newname][0]
self.assertNotIn(T_CF, [c.name for c in ksdef.cf_defs])
cfdef.name = newname
self.assertEqual(cfdef, cfdef2)
yield self.client.system_drop_column_family(cfdef.name)
ksdef = yield self.client.describe_keyspace(KEYSPACE)
self.assertNotIn(cfdef.name, [c.name for c in ksdef.cf_defs])
@defer.inlineCallbacks
def test_describes(self):
name = yield self.client.describe_cluster_name()
self.assertIsInstance(name, str)
self.assertNotEqual(name, '')
partitioner = yield self.client.describe_partitioner()
self.assert_(partitioner.startswith('org.apache.cassandra.'),
msg='partitioner is %r' % partitioner)
snitch = yield self.client.describe_snitch()
self.assert_(snitch.startswith('org.apache.cassandra.'),
msg='snitch is %r' % snitch)
version = yield self.client.describe_version()
self.assertIsInstance(version, str)
self.assertIn('.', version)
schemavers = yield self.client.describe_schema_versions()
self.assertIsInstance(schemavers, dict)
self.assertNotEqual(schemavers, {})
ring = yield self.client.describe_ring(KEYSPACE)
self.assertIsInstance(ring, list)
self.assertNotEqual(ring, [])
for r in ring:
self.assertIsInstance(r.start_token, str)
self.assertIsInstance(r.end_token, str)
self.assertIsInstance(r.endpoints, list)
self.assertNotEqual(r.endpoints, [])
for ep in r.endpoints:
self.assertIsInstance(ep, str)
@defer.inlineCallbacks
def test_errback(self):
yield self.client.remove('poiqwe', CF)
try:
yield self.client.get('poiqwe', CF, column='foo')
except Exception, e:
pass
class CassandraSchemaManager(object):
"""Manages creation and destruction of cassandra schemas.
Useful for both testing and production
"""
def __init__(self, keyspace_def, error_if_existing=False):
self.keyspace_def = keyspace_def
self.error_if_existing = error_if_existing
self.created_keyspace = False
self.created_cfs = []
self.client = None
self.manager = None
self.connector = None
def connect(self, host=None, port=9160, username=None, password=None):
if not host:
host, port = get_host_port()
if username or password:
if not (username and password):
raise CassandraConfigurationError(
"Specify both username and password or neither")
else:
username, password = get_credentials()
authz = dict(username=username, password=password)
self.manager = ManagedCassandraClientFactory(credentials=authz,
check_api_version=True)
self.connector = reactor.connectTCP(host, port, self.manager)
self.client = CassandraClient(self.manager)
def disconnect(self):
if self.manager:
self.manager.shutdown()
if self.connector:
self.connector.disconnect()
@timeout(DEFAULT_CASSANDRA_TIMEOUT)
@defer.inlineCallbacks
def create(self, truncate=False):
if not self.client:
self.connect()
keyspace = self.keyspace_def
try:
existing = yield self.client.describe_keyspace(keyspace.name)
except NotFoundException:
existing = None
# keyspace already exists
if existing:
yield self.client.set_keyspace(keyspace.name)
_compare_ks_properties(existing, keyspace)
existing_cfs = dict((cf.name, cf) for cf in existing.cf_defs)
for cf in keyspace.cf_defs:
if cf.name in existing_cfs:
if truncate:
# in truncate mode we drop and readd any existing CFs.
yield self.client.system_drop_column_family(cf.name)
yield self.client.system_add_column_family(cf)
else:
_compare_cf_properties(existing_cfs[cf.name], cf)
else:
if cf.keyspace != keyspace.name:
raise CassandraSchemaError(
"CF %s has wrong keyspace name", cf.name)
self.created_cfs.append(cf.name)
yield self.client.system_add_column_family(cf)
else:
self.created_keyspace = True
yield self.client.system_add_keyspace(keyspace)
yield self.client.set_keyspace(keyspace.name)
@timeout(DEFAULT_CASSANDRA_TIMEOUT)
@defer.inlineCallbacks
def teardown(self):
if self.created_keyspace:
yield self.client.system_drop_keyspace(self.keyspace_def.name)
elif self.created_cfs:
for cf in self.created_cfs:
yield self.client.system_drop_column_family(cf)
class CassandraProvisionerStore(object):
"""
Provides high level provisioner storage operations for Cassandra
"""
# default size of paged fetches
_PAGE_SIZE = 100
LAUNCH_CF_NAME = "ProvisionerLaunches"
NODE_CF_NAME = "ProvisionerNodes"
@classmethod
def get_column_families(cls, keyspace=None, prefix=''):
"""Builds a list of column families needed by this store.
@param keyspace Name of keyspace. If None, it must be added manually.
@param prefix Optional prefix for cf names. Useful for testing.
@retval list of CfDef objects
"""
launch_cf_name = prefix + cls.LAUNCH_CF_NAME
node_cf_name = prefix + cls.NODE_CF_NAME
return [
CfDef(keyspace,
launch_cf_name,
comparator_type='org.apache.cassandra.db.marshal.UTF8Type'),
CfDef(keyspace,
node_cf_name,
comparator_type='org.apache.cassandra.db.marshal.UTF8Type')
]
def __init__(self, host, port, username, password, keyspace, prefix=''):
self._launch_column_family = prefix + self.LAUNCH_CF_NAME
self._node_column_family = prefix + self.NODE_CF_NAME
authz = {'username': username, 'password': password}
self._manager = ManagedCassandraClientFactory(credentials=authz,
check_api_version=True,
keyspace=keyspace)
self.client = CassandraClient(self._manager)
self._host = host
self._port = port
self._connector = None
def connect(self):
self._connector = reactor.connectTCP(self._host, self._port,
self._manager)
def disconnect(self):
self._manager.shutdown()
if self._connector:
self._connector.disconnect()
self._connector = None
@timeout(CASSANDRA_TIMEOUT)
def put_launch(self, launch):
"""
@brief Stores a single launch record
@param launch Launch record to store
@retval Deferred for success
"""
launch_id = launch['launch_id']
state = launch['state']
value = json.dumps(launch)
return self.client.insert(launch_id,
self._launch_column_family,
value,
column=state)
@timeout(CASSANDRA_TIMEOUT)
@defer.inlineCallbacks
def put_nodes(self, nodes):
"""
@brief Stores a set of node records
@param nodes Iterable of node records
@retval Deferred for success
"""
# could be more efficient with a batch_mutate
for node in nodes:
yield self.put_node(node)
@timeout(CASSANDRA_TIMEOUT)
def put_node(self, node):
"""
@brief Stores a node record
@param node Node record
@retval Deferred for success
"""
node_id = node['node_id']
state = node['state']
value = json.dumps(node)
return self.client.insert(node_id,
self._node_column_family,
value,
column=state)
@timeout(CASSANDRA_TIMEOUT)
def get_launch(self, launch_id, count=1):
"""
@brief Retrieves a launch record by id
@param launch_id Id of launch record to retrieve
@param count Number of launch state records to retrieve
@retval Deferred record(s), or None. A list of records if count > 1
"""
return self._get_record(launch_id, self._launch_column_family, count)
@timeout(CASSANDRA_TIMEOUT)
def get_launches(self, state=None, min_state=None, max_state=None):
"""
@brief Retrieves the latest record for all launches within a state range
@param state Only retrieve nodes in this state.
@param min_state Inclusive start bound
@param max_state Inclusive end bound
@retval Deferred list of launch records
"""
return self._get_records(self._launch_column_family,
state=state,
min_state=min_state,
max_state=max_state)
@timeout(CASSANDRA_TIMEOUT)
def get_node(self, node_id, count=1):
"""
@brief Retrieves a launch record by id
@param node_id Id of node record to retrieve
@param count Number of node state records to retrieve
@retval Deferred record(s), or None. A list of records if count > 1
"""
return self._get_record(node_id, self._node_column_family, count)
@timeout(CASSANDRA_TIMEOUT)
def get_nodes(self, state=None, min_state=None, max_state=None):
"""
@brief Retrieves all launch record within a state range
@param state Only retrieve nodes in this state.
@param min_state Inclusive start bound.
@param max_state Inclusive end bound
@retval Deferred list of launch records
"""
return self._get_records(self._node_column_family,
state=state,
min_state=min_state,
max_state=max_state)
@defer.inlineCallbacks
def _get_record(self, key, column_family, count):
slice = yield self.client.get_slice(key,
column_family,
reverse=True,
count=count)
# we're probably only interested in the last record, in sorted order.
# This is the latest state the object has recorded.
records = [json.loads(column.column.value) for column in slice]
if count == 1:
if records:
ret = records[0]
else:
ret = None
else:
ret = records
defer.returnValue(ret)
@defer.inlineCallbacks
def _get_records(self,
column_family,
state=None,
min_state=None,
max_state=None,
reverse=True):
# overrides range arguments
if state:
min_state = max_state = state
start = ''
end = min_state or ''
if not reverse:
start, end = end, start
# this is tricky. We are only concerned with the latest state record
# (by sort order not necessarily time). So when we look for records
# within a state range, we effectively must pull down the latest state
# for each record, and filter them locally. This is slightly improved
# when a first_state (or last when reverse=False) is specified as the
# server can skip any records not >= that state.
records = []
done = False
start_key = ''
iterations = 0
while not done:
slices = yield self.client.get_range_slices(column_family,
column_start=start,
column_finish=end,
reverse=reverse,
column_count=1,
start=start_key,
count=self._PAGE_SIZE)
skipped_one = False
for slice in slices:
if not skipped_one and iterations:
# if this not the first batch, skip the first element as it
# will be a dupe.
skipped_one = True
continue
if not slice.columns:
# rows without matching columns will still be returned
continue
record = json.loads(slice.columns[0].column.value)
if not max_state or record['state'] <= max_state:
if not min_state or record['state'] >= min_state:
records.append(record)
# page through results. by default only 100 are returned at a time
if len(slices) == self._PAGE_SIZE:
start_key = slices[-1].key
else:
done = True
iterations += 1
defer.returnValue(records)
def on_deactivate(self, *args, **kwargs):
self._manager.shutdown()
log.info('on_deactivate: Lose Connection TCP')
def on_terminate(self, *args, **kwargs):
self._manager.shutdown()
log.info('on_terminate: Lose Connection TCP')
# from the data structure
res = yield client.batch_insert(key='test', column_family=CF, mapping={colname: 'bar'})
print "batch_insert", res
res = yield client.batch_insert(key='test', column_family=SCF, mapping={'foo': {colname: 'bar'}})
print "batch_insert", res
# with ttypes, you pass a list as you would for raw thrift
# this way you can set custom timestamps
cols = [Column(colname, 'bar', 1234), Column('bar', 'baz', 54321)]
res = yield client.batch_insert(key='test', column_family=CF, mapping=cols)
print "batch_insert", res
cols = [SuperColumn(name=colname, columns=cols)]
# of course you don't have to use kwargs if the order is correct
res = yield client.batch_insert('test', SCF, cols)
print "batch_insert", res
if __name__ == '__main__':
from twisted.internet import reactor
from twisted.python import log
import sys
log.startLogging(sys.stdout)
f = ManagedCassandraClientFactory(KEYSPACE)
c = CassandraClient(f)
dostuff(c)
reactor.connectTCP(HOST, PORT, f)
reactor.run()
def setUp(self):
self.cmanager = ManagedCassandraClientFactory(keyspace='system')
self.client = CassandraClient(self.cmanager)
for i in xrange(CONNS):
reactor.connectTCP(HOST, PORT, self.cmanager)
yield self.cmanager.deferred
remote_ver = yield self.client.describe_version()
self.version = tuple(map(int, remote_ver.split('.')))
self.my_keyspace = ttypes.KsDef(
name=KEYSPACE,
strategy_class='org.apache.cassandra.locator.SimpleStrategy',
strategy_options={},
cf_defs=[
ttypes.CfDef(
keyspace=KEYSPACE,
name=CF,
column_type='Standard'
),
ttypes.CfDef(
keyspace=KEYSPACE,
name=SCF,
column_type='Super'
),
ttypes.CfDef(
keyspace=KEYSPACE,
name=IDX_CF,
column_type='Standard',
comparator_type='org.apache.cassandra.db.marshal.UTF8Type',
column_metadata=[
ttypes.ColumnDef(
name='col1',
validation_class='org.apache.cassandra.db.marshal.UTF8Type',
index_type=ttypes.IndexType.KEYS,
index_name='idxCol1')
],
default_validation_class='org.apache.cassandra.db.marshal.BytesType'
),
]
)
if self.version <= KS_RF_ATTRIBUTE:
self.my_keyspace.replication_factor = 1
else:
self.my_keyspace.strategy_options['replication_factor'] = '1'
if self.version >= COUNTERS_SUPPORTED_API:
self.my_keyspace.cf_defs.extend([
ttypes.CfDef(
keyspace=KEYSPACE,
name=COUNTER_CF,
column_type='Standard',
default_validation_class='org.apache.cassandra.db.marshal.CounterColumnType'
),
ttypes.CfDef(
keyspace=KEYSPACE,
name=SUPERCOUNTER_CF,
column_type='Super',
default_validation_class='org.apache.cassandra.db.marshal.CounterColumnType'
),
])
yield self.client.system_add_keyspace(self.my_keyspace)
yield self.client.set_keyspace(KEYSPACE)
class CassandraClientTest(unittest.TestCase):
def setUp(self):
self.cmanager = ManagedCassandraClientFactory()
self.client = CassandraClient(self.cmanager, KEYSPACE)
for i in xrange(CONNS):
reactor.connectTCP(HOST, PORT, self.cmanager)
return self.cmanager.deferred
@defer.inlineCallbacks
def tearDown(self):
yield self.client.remove('test', CF)
yield self.client.remove('test2', CF)
yield self.client.remove('test', SCF)
yield self.client.remove('test2', SCF)
self.cmanager.shutdown()
for c in reactor.getDelayedCalls():
c.cancel()
reactor.removeAll()
@defer.inlineCallbacks
def test_insert_get(self):
yield self.client.insert('test', ColumnPath(CF, None, COLUMN), 'testval')
yield self.client.insert('test2', CF, 'testval2', column=COLUMN)
yield self.client.insert('test', ColumnPath(SCF, SCOLUMN, COLUMN), 'superval')
yield self.client.insert('test2', SCF, 'superval2', column=COLUMN,
super_column=SCOLUMN)
res = yield self.client.get('test', CF, column=COLUMN)
self.assert_(res.column.value == 'testval')
res = yield self.client.get('test2', CF, column=COLUMN)
self.assert_(res.column.value == 'testval2')
res = yield self.client.get('test', SCF, column=COLUMN, super_column=SCOLUMN)
self.assert_(res.column.value == 'superval')
res = yield self.client.get('test2', SCF, column=COLUMN, super_column=SCOLUMN)
self.assert_(res.column.value == 'superval2')
@defer.inlineCallbacks
def test_batch_insert_get_slice_and_count(self):
yield self.client.batch_insert('test', CF,
{COLUMN: 'test', COLUMN2: 'test2'})
yield self.client.batch_insert('test', SCF,
{SCOLUMN: {COLUMN: 'test', COLUMN2: 'test2'}})
res = yield self.client.get_slice('test', CF, names=(COLUMN, COLUMN2))
self.assert_(res[0].column.value == 'test')
self.assert_(res[1].column.value == 'test2')
res = yield self.client.get_slice('test', SCF, names=(COLUMN, COLUMN2),
super_column=SCOLUMN)
self.assert_(res[0].column.value == 'test')
self.assert_(res[1].column.value == 'test2')
res = yield self.client.get_count('test', CF)
self.assert_(res == 2)
@defer.inlineCallbacks
def test_multiget_slice_remove(self):
yield self.client.insert('test', CF, 'testval', column=COLUMN)
yield self.client.insert('test', CF, 'testval', column=COLUMN2)
yield self.client.insert('test2', CF, 'testval2', column=COLUMN)
res = yield self.client.multiget(['test', 'test2'], CF, column=COLUMN)
self.assert_(res['test'].column.value == 'testval')
self.assert_(res['test2'].column.value == 'testval2')
res = yield self.client.multiget_slice(['test', 'test2'], CF)
self.assert_(res['test'][0].column.value == 'testval')
self.assert_(res['test'][1].column.value == 'testval')
self.assert_(res['test2'][0].column.value == 'testval2')
yield self.client.remove('test', CF, column=COLUMN)
yield self.client.remove('test2', CF, column=COLUMN)
res = yield self.client.multiget(['test', 'test2'], CF, column=COLUMN)
self.assert_(res['test'].column == None)
self.assert_(res['test2'].column == None)
class CassandraControllerStore(TCPConnection):
"""Cassandra persistence for EPU controller state
All EPU controllers within a system share the same column families.
The "known" CFs hold a list of known instance IDs and sensor IDs for each
EPU controller. The value is irrelevant and inserts are of course
idempotent. These are used for figuring out all information about a
controller without walking the entire instances and sensors CFs.
ControllerKnownInstances = {
Controller1 = {
instance_id_1,
instance_id_2
},
Controller2 = {
instance_id_1,
instance_id_2
}
}
ControllerKnownSensors = {
Controller1 = {
sensor_1,
sensor_2
},
Controller2 = {
sensor_1,
sensor_2
}
}
All records for an instance are held in a single row. The column keys
are TimeUUIDs. Because instance records have more complicated ordering
than time, it is necessary to ensure a record is actually new before
inserting. This is safe since in the current architecture there is only
one writer for controller.
One limitation is that out-of-order records cannot be inserted in the
store. So if the messaging layer provides the STARTED state record for
an instance *after* the RUNNING record, it cannot be stored and must
be dropped. The correct state will be preserved, but not all of history
will be.
ControllerInstances = { #comparator = TimeUUIDType
Controller1Instance1 = {
TimeUUID1 : 'the actual record',
TimeUUID2 : 'the actual record',
TimeUUID3 : 'the actual record',
},
Controller2Instance1 = {
TimeUUID1 : 'the actual record',
TimeUUID2 : 'the actual record',
TimeUUID3 : 'the actual record',
}
}
Sensor records are stored similarly. Instead of a TimeUUID, they use
longs as keys which are likely to be a timestamp. Again, the controller
must check the timestamp and not treat the most recently arrived value
as the latest. However it can still write older values as they will be
correctly inserted into history.
ControllerSensors = { #comparator = LongType
Controller1Sensor1 = {
timestamp1 : 'sensor message',
timestamp2 : 'sensor message'
}
}
The controller starts up with a dictionary of config values that are
passed to the decision engine. These can be changed by calling the
reconfigure operation. Reconfigured values are stored here in
Cassandra and on reboot/recovery, they are folded into the original
config before it is passed to the decision engine.
Each controller gets a row. Each column represents a single key/value
pair where the key is a string and the value is a JSON-encoded object.
ControllerEngineConfig = {
Controller1 = {
key1 : 'value1',
key2 : 'value2'
},
Controller2 = {
key1 : 'value1',
key2 : 'value2'
}
}
"""
CONFIG_CF_NAME = "ControllerEngineConfig"
INSTANCE_CF_NAME = "ControllerInstances"
INSTANCE_ID_CF_NAME = "ControllerKnownInstances"
SENSOR_CF_NAME = "ControllerSensors"
SENSOR_ID_CF_NAME = "ControllerKnownSensors"
_PAGE_SIZE = 100
@classmethod
def get_column_families(cls, keyspace=None, prefix=''):
"""Builds a list of column families needed by this store.
@param keyspace Name of keyspace. If None, it must be added manually.
@param prefix Optional prefix for cf names. Useful for testing.
@retval list of CfDef objects
"""
instance_cf=prefix+cls.INSTANCE_CF_NAME
instance_id_cf=prefix+cls.INSTANCE_ID_CF_NAME
sensor_cf=prefix+cls.SENSOR_CF_NAME
sensor_id_cf=prefix+cls.SENSOR_ID_CF_NAME
config_cf = prefix+cls.CONFIG_CF_NAME
return [CfDef(keyspace, instance_cf,
comparator_type='org.apache.cassandra.db.marshal.TimeUUIDType'),
CfDef(keyspace, instance_id_cf,
comparator_type='org.apache.cassandra.db.marshal.UTF8Type'),
CfDef(keyspace, sensor_cf,
comparator_type='org.apache.cassandra.db.marshal.LongType'),
CfDef(keyspace, sensor_id_cf,
comparator_type='org.apache.cassandra.db.marshal.UTF8Type'),
CfDef(keyspace, config_cf,
comparator_type='org.apache.cassandra.db.marshal.UTF8Type'),
]
def __init__(self, controller_name, host, port, username, password,
keyspace, instance_factory, sensor_item_factory,
prefix=''):
self.controller_name = str(controller_name)
# keep a set of known instances and sensors so we can save on
# unnecessary inserts to the controller instance/sensor lists
self.seen_instances = set()
self.seen_sensors = set()
self.instance_factory = instance_factory
self.sensor_item_factory = sensor_item_factory
authorization_dictionary = {'username': username, 'password': password}
self.manager = ManagedCassandraClientFactory(
credentials=authorization_dictionary,
check_api_version=True, keyspace=keyspace)
TCPConnection.__init__(self, host, port, self.manager)
self.client = CassandraClient(self.manager)
self.instance_cf = prefix + self.INSTANCE_CF_NAME
self.instance_id_cf = prefix + self.INSTANCE_ID_CF_NAME
self.sensor_cf = prefix + self.SENSOR_CF_NAME
self.sensor_id_cf = prefix + self.SENSOR_ID_CF_NAME
self.config_cf = prefix + self.CONFIG_CF_NAME
@timeout(CASSANDRA_TIMEOUT)
@defer.inlineCallbacks
def check_schema(self):
ks = yield self.client.describe_keyspace(self.manager.keyspace)
cfs = dict((cf.name,cf) for cf in ks.cf_defs)
missing = [cf for cf in (self.instance_cf, self.instance_id_cf,
self.sensor_cf, self.sensor_id_cf,
self.config_cf)
if cf in cfs]
if missing:
error = "EPU Controller is missing Cassandra column families: %s"
raise Exception(error % ", ".join(missing))
@timeout(CASSANDRA_TIMEOUT)
@defer.inlineCallbacks
def add_instance(self, instance):
"""Adds a new instance object to persistence
@param instance Instance to add
@retval Deferred
"""
instance_id = str(instance.instance_id)
if instance_id not in self.seen_instances:
yield self.client.insert(self.controller_name, self.instance_id_cf,
"", column=instance_id)
self.seen_instances.add(instance_id)
key = self.controller_name + instance_id
value = json.dumps(dict(instance.iteritems()))
col = uuid.uuid1().bytes
yield self.client.insert(key, self.instance_cf, value, column=col)
@timeout(CASSANDRA_TIMEOUT)
def get_instance_ids(self):
"""Retrieves a list of known instances
@retval Deferred of list of instance IDs
"""
return self._get_ids(self.instance_id_cf)
@timeout(CASSANDRA_TIMEOUT)
@defer.inlineCallbacks
def get_instance(self, instance_id):
"""Retrieves the latest instance object for the specified id
@param instance_id ID of instance to retrieve
@retval Deferred of Instance object or None
"""
key = self.controller_name + str(instance_id)
slice = yield self.client.get_slice(key, self.instance_cf,
reverse=True, count=1)
if slice:
d = json.loads(slice[0].column.value)
ret = self.instance_factory(**d)
else:
ret = None
defer.returnValue(ret)
@timeout(CASSANDRA_TIMEOUT)
@defer.inlineCallbacks
def add_sensor(self, sensor):
"""Adds a new sensor object to persistence
@param sensor Sensor to add
@retval Deferred
"""
sensor_id = str(sensor.sensor_id)
if sensor_id not in self.seen_sensors:
yield self.client.insert(self.controller_name, self.sensor_id_cf,
"", column=sensor_id)
self.seen_sensors.add(sensor_id)
key = self.controller_name + sensor_id
value = json.dumps(sensor.value)
col = struct.pack('!Q', int(sensor.time))
yield self.client.insert(key, self.sensor_cf, value, column=col)
@timeout(CASSANDRA_TIMEOUT)
def get_sensor_ids(self):
"""Retrieves a list of known sensors
@retval Deferred of list of sensor IDs
"""
return self._get_ids(self.sensor_id_cf)
@timeout(CASSANDRA_TIMEOUT)
@defer.inlineCallbacks
def get_sensor(self, sensor_id):
"""Retrieve the latest sensor item for the specified sensor
@param sensor_id ID of the sensor item to retrieve
@retval Deferred of SensorItem object or None
"""
key = self.controller_name + str(sensor_id)
slice = yield self.client.get_slice(key, self.sensor_cf,
reverse=True, count=1)
if slice:
col = slice[0].column
timestamp = struct.unpack("!Q", col.name)[0]
val = json.loads(col.value)
ret = self.sensor_item_factory(sensor_id, long(timestamp), val)
else:
ret = None
defer.returnValue(ret)
@timeout(CASSANDRA_TIMEOUT)
@defer.inlineCallbacks
def get_config(self, keys=None):
"""Retrieve the engine config dictionary.
@param keys optional list of keys to retrieve
@retval Deferred of config dictionary object
"""
key = self.controller_name
slice = yield self.client.get_slice(key, self.config_cf, names=keys)
cfg = {}
if slice:
for col in slice:
key = col.column.name
val = col.column.value
cfg[key] = json.loads(val)
defer.returnValue(cfg)
@timeout(CASSANDRA_TIMEOUT)
def add_config(self, conf):
"""Store a dictionary of new engine conf values.
These are folded into the existing configuration map. So for example
if you first store {'a' : 1, 'b' : 1} and then store {'b' : 2},
the result from get_config() will be {'a' : 1, 'b' : 2}.
@param conf dictionary mapping strings to JSON-serializable objects
@retval Deferred
"""
d = dict((k, json.dumps(v)) for k,v in conf.iteritems())
return self.client.batch_insert(self.controller_name, self.config_cf, d)
@defer.inlineCallbacks
def _get_ids(self, cf):
"""Retrieves IDs from either instance or sensor column families
"""
# using a set because it isn't totally clear if the start parameter
# to get_slice() is always inclusive or exclusive. Seeing mixed
# messages on mailing list, so also not convinced if this isn't
# something that has changed, or might.
found_ids = set()
start = ""
done = False
while not done:
slice = yield self.client.get_slice(self.controller_name, cf,
count=self._PAGE_SIZE,
start=start)
if slice:
found_ids.update(col.column.name for col in slice)
if len(slice) == self._PAGE_SIZE:
start = slice[-1].column.name
else:
done = True
else:
done = True
defer.returnValue(list(found_ids))
def on_deactivate(self, *args, **kwargs):
self.manager.shutdown()
def on_terminate(self, *args, **kwargs):
self.manager.shutdown()
def setUp(self):
self.cmanager = ManagedCassandraClientFactory()
self.client = CassandraClient(self.cmanager, KEYSPACE)
for i in xrange(CONNS):
reactor.connectTCP(HOST, PORT, self.cmanager)
return self.cmanager.deferred
class CassandraClientTest(unittest.TestCase):
def setUp(self):
self.cmanager = ManagedCassandraClientFactory()
self.client = CassandraClient(self.cmanager, KEYSPACE)
for i in xrange(CONNS):
reactor.connectTCP(HOST, PORT, self.cmanager)
return self.cmanager.deferred
@defer.inlineCallbacks
def tearDown(self):
yield self.client.remove('test', CF)
yield self.client.remove('test2', CF)
yield self.client.remove('test', SCF)
yield self.client.remove('test2', SCF)
self.cmanager.shutdown()
for c in reactor.getDelayedCalls():
c.cancel()
reactor.removeAll()
@defer.inlineCallbacks
def test_insert_get(self):
yield self.client.insert('test', ColumnPath(CF, None, COLUMN), 'testval')
yield self.client.insert('test2', CF, 'testval2', column=COLUMN)
yield self.client.insert('test', ColumnPath(SCF, SCOLUMN, COLUMN), 'superval')
yield self.client.insert('test2', SCF, 'superval2', column=COLUMN,
super_column=SCOLUMN)
res = yield self.client.get('test', CF, column=COLUMN)
self.assert_(res.column.value == 'testval')
res = yield self.client.get('test2', CF, column=COLUMN)
self.assert_(res.column.value == 'testval2')
res = yield self.client.get('test', SCF, column=COLUMN, super_column=SCOLUMN)
self.assert_(res.column.value == 'superval')
res = yield self.client.get('test2', SCF, column=COLUMN, super_column=SCOLUMN)
self.assert_(res.column.value == 'superval2')
@defer.inlineCallbacks
def test_batch_insert_get_slice_and_count(self):
yield self.client.batch_insert('test', CF,
{COLUMN: 'test', COLUMN2: 'test2'})
yield self.client.batch_insert('test', SCF,
{SCOLUMN: {COLUMN: 'test', COLUMN2: 'test2'}})
res = yield self.client.get_slice('test', CF, names=(COLUMN, COLUMN2))
self.assert_(res[0].column.value == 'test')
self.assert_(res[1].column.value == 'test2')
res = yield self.client.get_slice('test', SCF, names=(COLUMN, COLUMN2),
super_column=SCOLUMN)
self.assert_(res[0].column.value == 'test')
self.assert_(res[1].column.value == 'test2')
res = yield self.client.get_count('test', CF)
self.assert_(res == 2)
@defer.inlineCallbacks
def test_batch_mutate_and_remove(self):
yield self.client.batch_mutate({'test': {CF: {COLUMN: 'test', COLUMN2: 'test2'}, SCF: { SCOLUMN: { COLUMN: 'test', COLUMN2: 'test2'} } }, 'test2': {CF: {COLUMN: 'test', COLUMN2: 'test2'}, SCF: { SCOLUMN: { COLUMN: 'test', COLUMN2: 'test2'} } } })
res = yield self.client.get_slice('test', CF, names=(COLUMN, COLUMN2))
self.assert_(res[0].column.value == 'test')
self.assert_(res[1].column.value == 'test2')
res = yield self.client.get_slice('test2', CF, names=(COLUMN, COLUMN2))
self.assert_(res[0].column.value == 'test')
self.assert_(res[1].column.value == 'test2')
res = yield self.client.get_slice('test', SCF, names=(COLUMN, COLUMN2),
super_column=SCOLUMN)
self.assert_(res[0].column.value == 'test')
self.assert_(res[1].column.value == 'test2')
res = yield self.client.get_slice('test2', SCF, names=(COLUMN, COLUMN2),
super_column=SCOLUMN)
self.assert_(res[0].column.value == 'test')
self.assert_(res[1].column.value == 'test2')
yield self.client.batch_remove({CF: ['test', 'test2']}, names=['test', 'test2'])
yield self.client.batch_remove({SCF: ['test', 'test2']}, names=['test', 'test2'], supercolumn=SCOLUMN)
@defer.inlineCallbacks
def test_batch_mutate_with_deletion(self):
yield self.client.batch_mutate({'test': {CF: {COLUMN: 'test', COLUMN2: 'test2'}}})
res = yield self.client.get_slice('test', CF, names=(COLUMN, COLUMN2))
self.assert_(res[0].column.value == 'test')
self.assert_(res[1].column.value == 'test2')
yield self.client.batch_mutate({'test': {CF: {COLUMN: None, COLUMN2: 'test3'}}})
res = yield self.client.get_slice('test', CF, names=(COLUMN, COLUMN2))
self.assert_(len(res) == 1)
self.assert_(res[0].column.value == 'test3')
@defer.inlineCallbacks
def test_multiget_slice_remove(self):
yield self.client.insert('test', CF, 'testval', column=COLUMN)
yield self.client.insert('test', CF, 'testval', column=COLUMN2)
yield self.client.insert('test2', CF, 'testval2', column=COLUMN)
res = yield self.client.multiget(['test', 'test2'], CF, column=COLUMN)
self.assert_(res['test'].column.value == 'testval')
self.assert_(res['test2'].column.value == 'testval2')
res = yield self.client.multiget_slice(['test', 'test2'], CF)
self.assert_(res['test'][0].column.value == 'testval')
self.assert_(res['test'][1].column.value == 'testval')
self.assert_(res['test2'][0].column.value == 'testval2')
yield self.client.remove('test', CF, column=COLUMN)
yield self.client.remove('test2', CF, column=COLUMN)
res = yield self.client.multiget(['test', 'test2'], CF, column=COLUMN)
self.assert_(res['test'].column == None)
self.assert_(res['test2'].column == None)
@defer.inlineCallbacks
def test_range_slices(self):
yield self.client.insert('test', CF, 'testval', column=COLUMN)
yield self.client.insert('test', CF, 'testval', column=COLUMN2)
yield self.client.insert('test2', CF, 'testval2', column=COLUMN)
ks = yield self.client.get_range_slices(CF, start='', finish='')
keys = [k.key for k in ks]
for key in ['test', 'test2']:
self.assert_(key in keys)
@defer.inlineCallbacks
def test_errback(self):
yield self.client.remove('poiqwe', CF)
try:
yield self.client.get('poiqwe', CF, column='foo')
except Exception, e:
pass
