def setup_databases(self, **kwargs):
settings.KEYSPACE = settings.KEYSPACE_TESTS
print "Patched value of KEYSPACE to '{0}'".format(settings.KEYSPACE)
if "dont_drop_daedalus_tests" not in os.environ:
sys_mgr = SystemManager()
try:
sys_mgr.drop_keyspace(settings.KEYSPACE)
print "Keyspace {0} droped OK".format(settings.KEYSPACE)
except:
pass
finally:
sys_mgr.close()
sys_mgr = SystemManager()
# keyspace 'settings.KEYSPACE' shouldn't exists
if settings.KEYSPACE in sys_mgr.list_keyspaces():
print "ERROR: keyspace {0} EXISTS after been droped".format(
settings.KEYSPACE)
print " - Keyspaces:", sys_mgr.list_keyspaces()
assert False, "ERROR"
sys_mgr.close()
storage.get_service().create_keyspace_and_cfs()
return super(CassandraDjangoTestSuiteRunner,
self).setup_databases(**kwargs)
def _cassandra_ensure_keyspace(self, server_list, keyspace_name,
cf_info_list):
# Retry till cassandra is up
server_idx = 0
num_dbnodes = len(self._server_list)
connected = False
while not connected:
try:
cass_server = self._server_list[server_idx]
sys_mgr = SystemManager(cass_server)
connected = True
except Exception as e:
# TODO do only for
# thrift.transport.TTransport.TTransportException
server_idx = (server_idx + 1) % num_dbnodes
time.sleep(3)
if self._reset_config:
try:
sys_mgr.drop_keyspace(keyspace_name)
except pycassa.cassandra.ttypes.InvalidRequestException as e:
# TODO verify only EEXISTS
self._logger("Warning! " + str(e), level=SandeshLevel.SYS_WARN)
try:
sys_mgr.create_keyspace(keyspace_name, SIMPLE_STRATEGY,
{'replication_factor': str(num_dbnodes)})
except pycassa.cassandra.ttypes.InvalidRequestException as e:
# TODO verify only EEXISTS
self._logger("Warning! " + str(e), level=SandeshLevel.SYS_WARN)
gc_grace_sec = 0
if num_dbnodes > 1:
gc_grace_sec = 60
for cf_info in cf_info_list:
try:
(cf_name, comparator_type) = cf_info
if comparator_type:
sys_mgr.create_column_family(
keyspace_name,
cf_name,
comparator_type=comparator_type,
gc_grace_seconds=gc_grace_sec,
default_validation_class='UTF8Type')
else:
sys_mgr.create_column_family(
keyspace_name,
cf_name,
gc_grace_seconds=gc_grace_sec,
default_validation_class='UTF8Type')
except pycassa.cassandra.ttypes.InvalidRequestException as e:
# TODO verify only EEXISTS
self._logger("Warning! " + str(e), level=SandeshLevel.SYS_WARN)
sys_mgr.alter_column_family(
keyspace_name,
cf_name,
gc_grace_seconds=gc_grace_sec,
default_validation_class='UTF8Type')
def delete_namespace(self):
sm = SystemManager(self._chosen_server)
try:
sm.drop_keyspace(self._app_namespace)
except pycassa.InvalidRequestException, exc:
if exc.why.startswith('Cannot drop non existing keyspace'):
pass
else:
raise exc
def _cassandra_ensure_keyspace(self, server_list,
keyspace_name, cf_info_list):
# Retry till cassandra is up
server_idx = 0
num_dbnodes = len(self._server_list)
connected = False
while not connected:
try:
cass_server = self._server_list[server_idx]
sys_mgr = SystemManager(cass_server)
connected = True
except Exception as e:
# TODO do only for
# thrift.transport.TTransport.TTransportException
server_idx = (server_idx + 1) % num_dbnodes
time.sleep(3)
if self._reset_config:
try:
sys_mgr.drop_keyspace(keyspace_name)
except pycassa.cassandra.ttypes.InvalidRequestException as e:
# TODO verify only EEXISTS
self._logger("Warning! " + str(e), level=SandeshLevel.SYS_WARN)
try:
sys_mgr.create_keyspace(keyspace_name, SIMPLE_STRATEGY,
{'replication_factor': str(num_dbnodes)})
except pycassa.cassandra.ttypes.InvalidRequestException as e:
# TODO verify only EEXISTS
self._logger("Warning! " + str(e), level=SandeshLevel.SYS_WARN)
gc_grace_sec = 0
if num_dbnodes > 1:
gc_grace_sec = 60
for cf_info in cf_info_list:
try:
(cf_name, comparator_type) = cf_info
if comparator_type:
sys_mgr.create_column_family(
keyspace_name, cf_name,
comparator_type=comparator_type,
gc_grace_seconds=gc_grace_sec,
default_validation_class='UTF8Type')
else:
sys_mgr.create_column_family(keyspace_name, cf_name,
gc_grace_seconds=gc_grace_sec,
default_validation_class='UTF8Type')
except pycassa.cassandra.ttypes.InvalidRequestException as e:
# TODO verify only EEXISTS
self._logger("Warning! " + str(e), level=SandeshLevel.SYS_WARN)
sys_mgr.alter_column_family(keyspace_name, cf_name,
gc_grace_seconds=gc_grace_sec,
default_validation_class='UTF8Type')
def test_big_batched_writes():
## this is an m1.xlarge doing nothing but supporting this test
server = 'localhost:9160'
keyspace = 'testkeyspace_' + getpass.getuser().replace('-', '_')
family = 'testcf'
sm = SystemManager(server)
try:
sm.drop_keyspace(keyspace)
except pycassa.InvalidRequestException:
pass
sm.create_keyspace(keyspace, SIMPLE_STRATEGY, {'replication_factor': '1'})
sm.create_column_family(keyspace, family, super=False,
key_validation_class = LEXICAL_UUID_TYPE,
default_validation_class = LEXICAL_UUID_TYPE,
column_name_class = ASCII_TYPE)
sm.alter_column(keyspace, family, 'test', ASCII_TYPE)
sm.close()
pool = ConnectionPool(keyspace, [server], max_retries=10, pool_timeout=0, pool_size=10, timeout=120)
pool.fill()
pool.add_listener( Listener() )
## assert that we are using framed transport
conn = pool._q.get()
assert isinstance(conn.transport, thrift.transport.TTransport.TFramedTransport)
pool._q.put(conn)
try:
for num_rows in range(14, 20):
## write some data to cassandra using increasing data sizes
one_mb = ' ' * 2**20
rows = []
for i in xrange(num_rows):
key = uuid.uuid4()
rows.append((key, dict(test=one_mb)))
testcf = pycassa.ColumnFamily(pool, family)
with testcf.batch() as batch:
for (key, data_dict) in rows:
data_size = len(data_dict.values()[0])
logger.critical('adding %r with %.6f MB' % (key, float(data_size)/2**20))
batch.insert(key, data_dict)
logger.critical('%d rows written' % num_rows)
finally:
sm = SystemManager(server)
try:
sm.drop_keyspace(keyspace)
except pycassa.InvalidRequestException:
pass
sm.close()
logger.critical('clearing test keyspace: %r' % keyspace)
class CassandraDbSysManager:
def __init__(self):
try:
logger.info("Connection to " + DB_IP + ":" + DB_PORT + " ...")
self.sysDB = SystemManager(DB_IP + ":" + DB_PORT)
logger.info("Connection to the systemManager established.")
except Exception as exep:
logger.warning("Could not connect to the DB '" + DB_IP + ":" + DB_PORT + "'. | " + str(exep))
def create_keyspace(self, KeyspaceName, replica_factor):
try:
logger.info(
"Creating the Keyspace '" + KeyspaceName + "' with replica_factor '" + str(replica_factor) + "'")
self.sysDB.create_keyspace(KeyspaceName, replica_factor)
logger.info("Keyspace created.")
except Exception as exep:
logger.warning("Could not create the keyspace '" + KeyspaceName + "'. | " + str(exep))
def drop_keyspace(self, KeyspaceName):
try:
logger.info("Dropping the Keyspace '" + KeyspaceName + "'")
self.sysDB.drop_keyspace(KeyspaceName)
logger.info("Keyspace dropped.")
except Exception as exep:
logger.warning("Could not drop the keyspace '" + KeyspaceName + "'. | " + str(exep))
def create_column_family(self, KeyspaceName, columnFamily):
try:
logger.info("Creating the Column family '" + columnFamily + "' into the keyspace '" + KeyspaceName + "'")
self.sysDB.create_column_family(KeyspaceName, columnFamily, super=False,
comparator_type=pycassa.system_manager.ASCII_TYPE)
logger.info("Column Family created.")
except Exception as exep:
logger.warning(
"Could not create the Column Family '" + columnFamily + "' into the keyspace '" + KeyspaceName + "'. | " + str(
exep))
def drop_column_family(self, KeyspaceName, columnFamily):
try:
logger.info("Dropping the Column Family '" + columnFamily + "' from the Keyspace '" + KeyspaceName + "'")
self.sysDB.drop_column_family(KeyspaceName, columnFamily)
logger.info("Column Family dropped.")
except Exception as exep:
logger.warning("Could not drop the column family '" + columnFamily + "'. | " + str(exep))
def setup_package():
sys = SystemManager()
if TEST_KS in sys.list_keyspaces():
sys.drop_keyspace(TEST_KS)
try:
sys.create_keyspace(TEST_KS, 'SimpleStrategy', {'replication_factor': '1'})
sys.create_column_family(TEST_KS, 'Standard1')
sys.create_column_family(TEST_KS, 'Super1', super=True)
sys.create_column_family(TEST_KS, 'Indexed1')
sys.create_index(TEST_KS, 'Indexed1', 'birthdate', 'LongType')
sys.create_column_family(TEST_KS, 'Counter1',
default_validation_class='CounterColumnType')
sys.create_column_family(TEST_KS, 'SuperCounter1', super=True,
default_validation_class='CounterColumnType')
except Exception, e:
print e
try:
sys.drop_keyspace(TEST_KS)
except:
pass
raise e
def setup_databases(self, **kwargs):
settings.KEYSPACE = settings.KEYSPACE_TESTS
print "Patched value of KEYSPACE to '{0}'".format(settings.KEYSPACE)
if "dont_drop_daedalus_tests" not in os.environ:
sys_mgr = SystemManager()
try:
sys_mgr.drop_keyspace(settings.KEYSPACE)
print "Keyspace {0} droped OK".format(settings.KEYSPACE)
except:
pass
finally:
sys_mgr.close()
sys_mgr = SystemManager()
# keyspace 'settings.KEYSPACE' shouldn't exists
if settings.KEYSPACE in sys_mgr.list_keyspaces():
print "ERROR: keyspace {0} EXISTS after been droped".format(settings.KEYSPACE)
print " - Keyspaces:", sys_mgr.list_keyspaces()
assert False, "ERROR"
sys_mgr.close()
storage.get_service().create_keyspace_and_cfs()
return super(CassandraDjangoTestSuiteRunner, self).setup_databases(**kwargs)
class ComedySysManagerConnection(object):
"""
This is a also a client connection of Cassandra, difference is it is only for System-Manager level
"""
def __init__(self, server='localhost:9160', credentials=None, framed_transport=True,
timeout=_DEFAULT_TIMEOUT):
self.conn = SystemManager(server,credentials,framed_transport,timeout)
def create_keyspace(self, name, replication_strategy='SimpleStrategy', strategy_options=None, durable_writes=True, **ks_kwargs):
"""Create a database"""
self.conn.create_keyspace(name, replication_strategy, strategy_options, durable_writes)
create_database = create_keyspace
def drop_keyspace(self, keyspace):
"""Drop a database"""
self.conn.drop_keyspace(keyspace)
drop_database = drop_keyspace
def create_column_family(self, keyspace, name, column_validation_classes=None, **cf_kwargs):
"""Create a table"""
self.conn.create_column_family(keyspace, name, column_validation_classes=None, **cf_kwargs)
create_table = create_column_family
def drop_column_family(self, keyspace, column_family):
"""Drop a table"""
self.conn.drop_column_family(keyspace, column_family)
drop_table = drop_column_family
def close(self):
pass
def test(host='localhost', keyspace_provided=None):
from pycassa.types import BytesType, CompositeType
from pycassa.system_manager import SystemManager, SIMPLE_STRATEGY
# Create a fake keyspace, if not provided
if not keyspace_provided:
import random
keyspace = '%s_%s' % ('json', random.randrange(1000000, 100000000))
else:
keyspace = keyspace_provided
# Connect to cluster and create keyspace
system_manager = SystemManager(host)
system_manager.create_keyspace(keyspace, SIMPLE_STRATEGY,
{'replication_factor': '1'})
try:
# Create CF with many CompositeFields
comparator = CompositeType(BytesType(), BytesType(), BytesType(),
BytesType(), BytesType(), BytesType(),
BytesType(), BytesType(), BytesType(),
BytesType(), BytesType(), BytesType(),
BytesType(), BytesType(), BytesType(),
BytesType(), BytesType(), BytesType(),
BytesType(), BytesType(), BytesType(),
BytesType(), BytesType(), BytesType(),
BytesType(), BytesType(), BytesType(),
BytesType(), BytesType(), BytesType())
system_manager.create_column_family(keyspace,
'json',
comparator_type=comparator)
# Connect to the KS/CF and save samples
db = CassandraJsonMapper.db(keyspace, 'json')
tests = 0
# ----------------------------------------------------------------------
# Test a complicated structure
sample_1 = {
'key1': {
'a': 1,
2: 'b',
'c': {
'd': 3,
'e': {
'f': True
},
'g': [
['h', 'i', 'j', 4, 5],
['k', 'l', 'm', 'n', 'o'],
],
'p': [{
'id': 6,
'q': 'r'
}, {
'id': 7,
's': 't'
}],
'u': [],
'v': None
}
}
}
db.save(sample_1)
if db.get('key1') != sample_1['key1']:
raise AssertionError(
'What was saved is not being equally returned.')
tests += 1
# ----------------------------------------------------------------------
# Test improper format missing values
sample_2 = {'key1': 1}
try:
db.save(sample_2)
raise AssertionError('sample_2 should have thrown a KeyError.')
except KeyError:
pass
tests += 1
# ----------------------------------------------------------------------
# Test saving multiple keys
sample_3 = {'key2': {2: 2}, 'key3': {3: 3}, 'key4': {4: 4}}
db.save(sample_3)
if db.get('key2') != {2:2} or db.get('key3') != {3:3} or \
db.get('key4') != {4:4}:
raise AssertionError('Not all keys in a json_payload were saved.')
tests += 1
# ----------------------------------------------------------------------
# Test delete
db.delete('key1')
if db.get('key1'):
raise AssertionError('Not all keys in were delted.')
tests += 1
# ----------------------------------------------------------------------
# Test deletes
db.delete(['key2', 'key3', 'key4'])
if db.get('key2') or db.get('key3') or db.get('key4'):
raise AssertionError('Not all keys in were delted.')
tests += 1
# ----------------------------------------------------------------------
# Test reading from fake keys
if db.get('fake_key') != {}:
raise AssertionError('A get(fake_key) should return {}.')
tests += 1
# ----------------------------------------------------------------------
# Test multi-threaded save
db.mt_save(sample_1)
db.mt_finish()
if db.get('key1') != sample_1['key1']:
raise AssertionError(
'What was saved is not being equally returned.')
tests += 1
# ----------------------------------------------------------------------
# print json_format(db.get('key1'))
print '{0}/{0} tests passed!'.format(tests)
except:
raise
finally:
# For debugging purposes
# raw_input('Press ENTER to continue...')
if not keyspace_provided:
# Delete the temporary KS's
system_manager.drop_keyspace(keyspace)
system_manager.close()
class DistributedCassandraLockTestCase(unittest.TestCase):
def setUp(self):
self.sysman = SystemManager()
self.sysman.create_keyspace(
TEST_KS, SIMPLE_STRATEGY,
dict(replication_factor='1')
)
self.sysman.create_column_family(TEST_KS, TEST_CF)
self.pool = ConnectionPool(TEST_KS)
self.cf = ColumnFamily(self.pool, TEST_CF)
def tearDown(self):
self.sysman.drop_keyspace(TEST_KS)
del self.sysman
def test_ttl(self):
l = CassandraDistributedRowLock(
self.pool, TEST_CF, "test_ttl",
ttl=2.0,
consistency_level=ConsistencyLevel.ONE,
timeout=1.0
)
try:
l.acquire()
self.assertEqual(1, len(l.read_lock_columns()))
time.sleep(3)
self.assertEqual(0, len(l.read_lock_columns()))
except:
raise
finally:
l.release()
self.assertEqual(0, len(l.read_lock_columns()))
def test_stale_lock(self):
l1 = CassandraDistributedRowLock(
self.pool, TEST_CF, "test_stale_lock",
ttl=TEST_TTL, consistency_level=ConsistencyLevel.ONE,
timeout=1.0
)
l2 = CassandraDistributedRowLock(
self.pool, TEST_CF, "test_stale_lock",
ttl=TEST_TTL, consistency_level=ConsistencyLevel.ONE,
timeout=9.0
)
try:
l1.acquire()
time.sleep(5)
try:
l2.acquire()
except:
raise
finally:
l2.release()
except:
raise
finally:
l1.release()
def test_stale_lock_with_fail(self):
l1 = CassandraDistributedRowLock(
self.pool, TEST_CF, "test_stale_lock",
ttl=TEST_TTL, consistency_level=ConsistencyLevel.ONE,
timeout=1.0
)
l2 = CassandraDistributedRowLock(
self.pool, TEST_CF, "test_stale_lock",
ttl=TEST_TTL, consistency_level=ConsistencyLevel.ONE,
timeout=9.0, fail_on_stale_lock=True
)
try:
l1.acquire()
time.sleep(2)
try:
l2.acquire()
assert False, "derp"
except StaleLockException:
print 'things are working smoothly'
except:
raise
finally:
l2.release()
except:
raise
finally:
l2.release()
from pycassa.system_manager import SystemManager
from pycassa.system_manager import SIMPLE_STRATEGY
from pycassa.system_manager import UTF8_TYPE
from pycassa.system_manager import LONG_TYPE
from storage.cassandra_settings import HOSTS
from storage.cassandra_settings import KEYSPACE_NAME
sys = SystemManager(HOSTS[0])
if KEYSPACE_NAME in sys.list_keyspaces():
sys.drop_keyspace(KEYSPACE_NAME)
sys.create_keyspace(KEYSPACE_NAME, SIMPLE_STRATEGY, {'replication_factor': '1'})
sys.create_column_family(KEYSPACE_NAME, 'LoveActivity', comparator_type=UTF8_TYPE)
sys.create_column_family(KEYSPACE_NAME, 'Feed', comparator_type=UTF8_TYPE)
sys.create_column_family(KEYSPACE_NAME, 'AggregatedFeed', comparator_type=LONG_TYPE, super=True)
def test(host='localhost', keyspace_provided=None):
from pycassa.types import BytesType, CompositeType
from pycassa.system_manager import SystemManager, SIMPLE_STRATEGY
# Create a fake keyspace, if not provided
if not keyspace_provided:
import random
keyspace = '%s_%s' % ('json', random.randrange(1000000, 100000000))
else:
keyspace = keyspace_provided
# Connect to cluster and create keyspace
system_manager = SystemManager(host)
system_manager.create_keyspace(keyspace, SIMPLE_STRATEGY,
{'replication_factor': '1'})
try:
# Create CF with many CompositeFields
comparator = CompositeType(BytesType(), BytesType(), BytesType(),
BytesType(), BytesType(), BytesType(),
BytesType(), BytesType(), BytesType(),
BytesType(), BytesType(), BytesType(),
BytesType(), BytesType(), BytesType(),
BytesType(), BytesType(), BytesType(),
BytesType(), BytesType(), BytesType(),
BytesType(), BytesType(), BytesType(),
BytesType(), BytesType(), BytesType(),
BytesType(), BytesType(), BytesType())
system_manager.create_column_family(keyspace, 'json',
comparator_type=comparator)
# Connect to the KS/CF and save samples
db = CassandraJsonMapper.db(keyspace, 'json')
tests = 0
# ----------------------------------------------------------------------
# Test a complicated structure
sample_1 = {
'key1': {
'a': 1,
2: 'b',
'c': {
'd': 3,
'e': {
'f': True
},
'g': [
[
'h',
'i',
'j',
4,
5
], [
'k',
'l',
'm',
'n',
'o'
],
],
'p': [
{
'id': 6,
'q': 'r'
}, {
'id': 7,
's': 't'
}
],
'u': [],
'v': None
}
}
}
db.save(sample_1)
if db.get('key1') != sample_1['key1']:
raise AssertionError('What was saved is not being equally returned.')
tests += 1
# ----------------------------------------------------------------------
# Test improper format missing values
sample_2 = {
'key1': 1
}
try:
db.save(sample_2)
raise AssertionError('sample_2 should have thrown a KeyError.')
except KeyError:
pass
tests += 1
# ----------------------------------------------------------------------
# Test saving multiple keys
sample_3 = {
'key2': {2:2},
'key3': {3:3},
'key4': {4:4}
}
db.save(sample_3)
if db.get('key2') != {2:2} or db.get('key3') != {3:3} or \
db.get('key4') != {4:4}:
raise AssertionError('Not all keys in a json_payload were saved.')
tests += 1
# ----------------------------------------------------------------------
# Test delete
db.delete('key1')
if db.get('key1'):
raise AssertionError('Not all keys in were delted.')
tests += 1
# ----------------------------------------------------------------------
# Test deletes
db.delete(['key2', 'key3', 'key4'])
if db.get('key2') or db.get('key3') or db.get('key4'):
raise AssertionError('Not all keys in were delted.')
tests += 1
# ----------------------------------------------------------------------
# Test reading from fake keys
if db.get('fake_key') != {}:
raise AssertionError('A get(fake_key) should return {}.')
tests += 1
# ----------------------------------------------------------------------
# Test multi-threaded save
db.mt_save(sample_1)
db.mt_finish()
if db.get('key1') != sample_1['key1']:
raise AssertionError('What was saved is not being equally returned.')
tests += 1
# ----------------------------------------------------------------------
# print json_format(db.get('key1'))
print '{0}/{0} tests passed!'.format(tests)
except:
raise
finally:
# For debugging purposes
# raw_input('Press ENTER to continue...')
if not keyspace_provided:
# Delete the temporary KS's
system_manager.drop_keyspace(keyspace)
system_manager.close()
"""
Drop the Cassandra database.
"""
import pycassa
from pycassa.system_manager import SystemManager
mgr = SystemManager()
mgr.drop_keyspace('drought')
mgr.close()
def delete_namespace(self):
sm = SystemManager(random.choice(self.server_list))
sm.drop_keyspace(self.namespace)
sm.close()
def _delete_namespace(config):
sm = SystemManager(config['storage_addresses'][0])
sm.drop_keyspace(config['namespace'])
sm.close()
def delete_namespace(self):
sm = SystemManager(random.choice(self.server_list))
sm.drop_keyspace(self.namespace)
sm.close()
def teardown_package():
sys = SystemManager()
if TEST_KS in sys.list_keyspaces():
sys.drop_keyspace(TEST_KS)
sys.close()
from pycassa import ConnectionPool
from pycassa.columnfamily import ColumnFamily
from pycassa.system_manager import SystemManager
from pycassa.system_manager import SIMPLE_STRATEGY
from pycassa.system_manager import UTF8_TYPE
from pycassa.system_manager import ASCII_TYPE
import time
############################## Create Keyspace ################################
server_list = ['cassandra1:9160', 'cassandra2:9160', 'cassandra3:9160']
sys = SystemManager(server_list[0])
sys.list_keyspaces()
if 'entries' in sys.list_keyspaces():
sys.drop_keyspace('entries')
sys.create_keyspace('entries', SIMPLE_STRATEGY, {'replication_factor': '1'})
############################## Connection Pooling ############################
#pool = ConnectionPool('entries', server_list=server_list, pool_size=20)
pool = ConnectionPool('entries', server_list=server_list)
############################## Create Column Family ###########################
sys.create_column_family('entries', 'Author', comparator_type=UTF8_TYPE)
author_cf = ColumnFamily(pool, 'Author')
################################ INSERT #######################################
# Insert a row with a Column
author_cf.insert('sacharya', {'first_name': 'Sudarshan'})
# Insert a row with multiple columns
author_cf.insert('sacharya1', {'first_name': 'Sudarshan', 'last_name':
def _clean_db(server):
dbsys = SystemManager(server)
dbsys.drop_keyspace(KEY_SPACE)
dbsys.close()
def _delete_namespace(config):
sm = SystemManager(config['storage_addresses'][0])
sm.drop_keyspace(config['namespace'])
sm.close()
