def CreateCF(connStr, keySpace, cfs, columesTypeList = None):
sysmgt = SystemManager(connStr)
list = sysmgt.list_keyspaces()
if keySpace in list:
#print('pyagent is existed')
pass
else:
sysmgt.create_keyspace(keySpace, SIMPLE_STRATEGY, {'replication_factor': '3'})
cfDict = sysmgt.get_keyspace_column_families(keySpace)
allCF = cfDict.keys()
for i in range(len(cfs)):
columnFamily = cfs[i]
#print(columnFamily)
if columesTypeList != None:
columes = columesTypeList[i]
else:
columes = None
if columnFamily in allCF:
#print(columnFamily + ' is existed')
pass
else:
#print(columnFamily + ' is creating')
sysmgt.create_column_family(keySpace, columnFamily, super=False,
comparator_type=UTF8_TYPE,
key_validation_class=ASCII_TYPE,
default_validation_class=UTF8_TYPE,
column_validation_classes=columes,
gc_grace_seconds=1000)
sysmgt.close()
def get_connection():
"""
Creates a connection to Cassandra.
Returs:
pool
"""
cassandra_host = os.environ.get('CASSANDRA_HOST', 'localhost')
sys_mgr = SystemManager()
try:
sys_mgr.describe_ring(KEYSPACE)
except:
sys_mgr.create_keyspace(KEYSPACE, SIMPLE_STRATEGY, {'replication_factor': '1'})
pool = ConnectionPool(KEYSPACE, server_list=[cassandra_host])
for cf_name in [CF_LOGS, CF_LOGS_BY_APP, CF_LOGS_BY_HOST, CF_LOGS_BY_SEVERITY]:
try:
cf = ColumnFamily(pool, cf_name)
except:
sys_mgr.create_column_family(KEYSPACE, cf_name, comparator_type=TimeUUIDType())
cf = ColumnFamily(pool, cf_name)
cf.get_count(str(uuid.uuid4()))
sys_mgr.close()
return pool
def initializeTableLayout(keyspace, server_list, replicationStrategy,
strategyOptions, localDCName, credentials):
sys_manager = SystemManager(server_list[0], credentials=credentials)
# Make sure the the keyspace exists
if keyspace not in sys_manager.list_keyspaces():
sys_manager.create_keyspace(keyspace, replicationStrategy,
strategyOptions)
cf_defs = sys_manager.get_keyspace_column_families(keyspace)
# Create UTF8 CF's
for tablename in ["global_nodes", "metadata"]:
if tablename not in cf_defs.keys():
createUTF8ColumnFamily(sys_manager, keyspace, tablename)
if localDCName:
dcNodes = "dc_%s_nodes" % (localDCName,)
if dcNodes not in cf_defs.keys():
createUTF8ColumnFamily(sys_manager, keyspace, dcNodes)
else:
# TODO Log we do not have the DC name
pass
if "node_slices" not in cf_defs.keys():
sys_manager.create_column_family(
keyspace,
"node_slices",
super=False,
comparator_type=pycassa_types.LongType(),
key_validation_class=pycassa_types.UTF8Type(),
default_validation_class=pycassa_types.LongType()
)
def create_cfs(self):
"""
Creates the Cassandra Column Families (if not exist)
"""
sys_mgr = None
pool = None
try:
sys_mgr = SystemManager()
pool = ConnectionPool(settings.KEYSPACE,
server_list=settings.CASSANDRA_HOSTS)
for cf_name in [
CF_LOGS, CF_LOGS_BY_APP, CF_LOGS_BY_HOST,
CF_LOGS_BY_SEVERITY
]:
try:
cf = ColumnFamily(pool, cf_name)
except:
logger.info("create_cfs(): Creating column family %s",
cf_name)
sys_mgr.create_column_family(
settings.KEYSPACE,
cf_name,
comparator_type=TimeUUIDType())
cf = ColumnFamily(pool, cf_name)
cf.get_count(str(uuid.uuid4()))
finally:
if pool:
pool.dispose()
if sys_mgr:
sys_mgr.close()
def __init__(self, engine, **kw):
super(CassandraStore, self).__init__(engine, **kw)
spliturl = urlsplit(engine)
_, keyspace, column_family = spliturl.path.split('/')
try:
self._store = pycassa.ColumnFamily(
pycassa.ConnectionPool(keyspace, [spliturl.hostname]),
column_family,
)
except pycassa.InvalidRequestException:
from pycassa.system_manager import SystemManager # @UnresolvedImport @IgnorePep8
system_manager = SystemManager(spliturl[1])
system_manager.create_keyspace(
keyspace,
pycassa.system_manager.SIMPLE_STRATEGY,
dict(replication_factor=native(kw.get('replication', 1))),
)
system_manager.create_column_family(keyspace, column_family)
self._store = pycassa.ColumnFamily(
pycassa.ConnectionPool(keyspace, [spliturl.netloc]),
column_family,
)
except pycassa.NotFoundException:
from pycassa.system_manager import SystemManager # @UnresolvedImport @IgnorePep8
system_manager = SystemManager(spliturl[1])
system_manager.create_column_family(keyspace, column_family)
self._store = pycassa.ColumnFamily(
pycassa.ConnectionPool(keyspace, [spliturl.netloc]),
column_family,
)
def ensure_cassandra_cf(self):
s = SystemManager()
if self.keyspace not in s.list_keyspaces():
s.create_keyspace(self.keyspace, SIMPLE_STRATEGY, {'replication_factor': '1'})
if self.cassandra_columns_family not in s.get_keyspace_column_families(self.keyspace):
s.create_column_family(self.keyspace, self.cassandra_columns_family)
self.columnfamily = ColumnFamily(self.cassandra_session, self.cassandra_columns_family)
def _cassandra_ensure_keyspace(self, server_list, keyspace_name,
cf_info_list):
# Retry till cassandra is up
server_idx = 0
num_dbnodes = len(self._server_list)
connected = False
while not connected:
try:
cass_server = self._server_list[server_idx]
sys_mgr = SystemManager(cass_server)
connected = True
except Exception as e:
# TODO do only for
# thrift.transport.TTransport.TTransportException
server_idx = (server_idx + 1) % num_dbnodes
time.sleep(3)
if self._reset_config:
try:
sys_mgr.drop_keyspace(keyspace_name)
except pycassa.cassandra.ttypes.InvalidRequestException as e:
# TODO verify only EEXISTS
self._logger("Warning! " + str(e), level=SandeshLevel.SYS_WARN)
try:
sys_mgr.create_keyspace(keyspace_name, SIMPLE_STRATEGY,
{'replication_factor': str(num_dbnodes)})
except pycassa.cassandra.ttypes.InvalidRequestException as e:
# TODO verify only EEXISTS
self._logger("Warning! " + str(e), level=SandeshLevel.SYS_WARN)
gc_grace_sec = 0
if num_dbnodes > 1:
gc_grace_sec = 60
for cf_info in cf_info_list:
try:
(cf_name, comparator_type) = cf_info
if comparator_type:
sys_mgr.create_column_family(
keyspace_name,
cf_name,
comparator_type=comparator_type,
gc_grace_seconds=gc_grace_sec,
default_validation_class='UTF8Type')
else:
sys_mgr.create_column_family(
keyspace_name,
cf_name,
gc_grace_seconds=gc_grace_sec,
default_validation_class='UTF8Type')
except pycassa.cassandra.ttypes.InvalidRequestException as e:
# TODO verify only EEXISTS
self._logger("Warning! " + str(e), level=SandeshLevel.SYS_WARN)
sys_mgr.alter_column_family(
keyspace_name,
cf_name,
gc_grace_seconds=gc_grace_sec,
default_validation_class='UTF8Type')
def setup_class(cls):
sys = SystemManager()
sys.create_column_family(
TEST_KS,
'CustomComposite1',
comparator_type=CompositeType(
IntegerType(),
UTF8Type()))
def test_single_component_composite(self):
sys = SystemManager()
sys.create_column_family(TEST_KS, 'SingleComposite',
comparator_type=CompositeType(IntegerType()))
cf = ColumnFamily(pool, 'SingleComposite')
cf.insert('key', {(123456,): 'val'})
assert_equal(cf.get('key'), {(123456,): 'val'})
def setUp(self):
from shove import Shove
from pycassa.system_manager import SystemManager
system_manager = SystemManager('localhost:9160')
try:
system_manager.create_column_family('Foo', 'shove')
except:
pass
self.store = Shove('cassandra://localhost:9160/Foo/shove')
def setUp(self):
from shove import Shove
from pycassa.system_manager import SystemManager
system_manager = SystemManager('localhost:9160')
try:
system_manager.create_column_family('Foo', 'shove')
except:
pass
self.store = Shove('cassandra://localhost:9160/Foo/shove')
def setUp(self):
from shove import Shove
from pycassa.system_manager import SystemManager # @UnresolvedImport @IgnorePep8
system_manager = SystemManager('localhost:9160')
try:
system_manager.create_column_family('Murk', 'shove')
except:
pass
self.store = Shove('cassandra://localhost:9160/Murk/shove')
def setUp(self):
from shove import Shove
from pycassa.system_manager import SystemManager # @UnresolvedImport @IgnorePep8
system_manager = SystemManager('localhost:9160')
try:
system_manager.create_column_family('Murk', 'shove')
except:
pass
self.store = Shove('cassandra://localhost:9160/Murk/shove')
def setup_class(cls):
sys = SystemManager()
sys.create_column_family(TEST_KS, 'StaticComposite',
comparator_type=CompositeType(LongType(),
IntegerType(),
TimeUUIDType(reversed=True),
LexicalUUIDType(reversed=False),
AsciiType(),
UTF8Type(),
BytesType()))
def ensure_cassandra_cf(self):
s = SystemManager()
if self.keyspace not in s.list_keyspaces():
s.create_keyspace(self.keyspace, SIMPLE_STRATEGY,
{'replication_factor': '1'})
if self.cassandra_columns_family not in s.get_keyspace_column_families(
self.keyspace):
s.create_column_family(self.keyspace,
self.cassandra_columns_family)
self.columnfamily = ColumnFamily(self.cassandra_session,
self.cassandra_columns_family)
def _cassandra_ensure_keyspace(self, server_list,
keyspace_name, cf_info_list):
# Retry till cassandra is up
server_idx = 0
num_dbnodes = len(self._server_list)
connected = False
while not connected:
try:
cass_server = self._server_list[server_idx]
sys_mgr = SystemManager(cass_server)
connected = True
except Exception as e:
# TODO do only for
# thrift.transport.TTransport.TTransportException
server_idx = (server_idx + 1) % num_dbnodes
time.sleep(3)
if self._reset_config:
try:
sys_mgr.drop_keyspace(keyspace_name)
except pycassa.cassandra.ttypes.InvalidRequestException as e:
# TODO verify only EEXISTS
self._logger("Warning! " + str(e), level=SandeshLevel.SYS_WARN)
try:
sys_mgr.create_keyspace(keyspace_name, SIMPLE_STRATEGY,
{'replication_factor': str(num_dbnodes)})
except pycassa.cassandra.ttypes.InvalidRequestException as e:
# TODO verify only EEXISTS
self._logger("Warning! " + str(e), level=SandeshLevel.SYS_WARN)
gc_grace_sec = 0
if num_dbnodes > 1:
gc_grace_sec = 60
for cf_info in cf_info_list:
try:
(cf_name, comparator_type) = cf_info
if comparator_type:
sys_mgr.create_column_family(
keyspace_name, cf_name,
comparator_type=comparator_type,
gc_grace_seconds=gc_grace_sec,
default_validation_class='UTF8Type')
else:
sys_mgr.create_column_family(keyspace_name, cf_name,
gc_grace_seconds=gc_grace_sec,
default_validation_class='UTF8Type')
except pycassa.cassandra.ttypes.InvalidRequestException as e:
# TODO verify only EEXISTS
self._logger("Warning! " + str(e), level=SandeshLevel.SYS_WARN)
sys_mgr.alter_column_family(keyspace_name, cf_name,
gc_grace_seconds=gc_grace_sec,
default_validation_class='UTF8Type')
def test_uuid_composites(self):
sys = SystemManager()
sys.create_column_family(TEST_KS, 'UUIDComposite',
comparator_type=CompositeType(IntegerType(reversed=True), TimeUUIDType()),
key_validation_class=TimeUUIDType(),
default_validation_class=UTF8Type())
key, u1, u2 = uuid.uuid1(), uuid.uuid1(), uuid.uuid1()
cf = ColumnFamily(pool, 'UUIDComposite')
cf.insert(key, {(123123, u1): 'foo'})
cf.insert(key, {(123123, u1): 'foo', (-1, u2): 'bar', (-123123123, u1): 'baz'})
assert_equal(cf.get(key), {(123123, u1): 'foo', (-1, u2): 'bar', (-123123123, u1): 'baz'})
def test_big_batched_writes():
## this is an m1.xlarge doing nothing but supporting this test
server = 'localhost:9160'
keyspace = 'testkeyspace_' + getpass.getuser().replace('-', '_')
family = 'testcf'
sm = SystemManager(server)
try:
sm.drop_keyspace(keyspace)
except pycassa.InvalidRequestException:
pass
sm.create_keyspace(keyspace, SIMPLE_STRATEGY, {'replication_factor': '1'})
sm.create_column_family(keyspace, family, super=False,
key_validation_class = LEXICAL_UUID_TYPE,
default_validation_class = LEXICAL_UUID_TYPE,
column_name_class = ASCII_TYPE)
sm.alter_column(keyspace, family, 'test', ASCII_TYPE)
sm.close()
pool = ConnectionPool(keyspace, [server], max_retries=10, pool_timeout=0, pool_size=10, timeout=120)
pool.fill()
pool.add_listener( Listener() )
## assert that we are using framed transport
conn = pool._q.get()
assert isinstance(conn.transport, thrift.transport.TTransport.TFramedTransport)
pool._q.put(conn)
try:
for num_rows in range(14, 20):
## write some data to cassandra using increasing data sizes
one_mb = ' ' * 2**20
rows = []
for i in xrange(num_rows):
key = uuid.uuid4()
rows.append((key, dict(test=one_mb)))
testcf = pycassa.ColumnFamily(pool, family)
with testcf.batch() as batch:
for (key, data_dict) in rows:
data_size = len(data_dict.values()[0])
logger.critical('adding %r with %.6f MB' % (key, float(data_size)/2**20))
batch.insert(key, data_dict)
logger.critical('%d rows written' % num_rows)
finally:
sm = SystemManager(server)
try:
sm.drop_keyspace(keyspace)
except pycassa.InvalidRequestException:
pass
sm.close()
logger.critical('clearing test keyspace: %r' % keyspace)
def handle(self, **options):
"""
:param options:
"""
sys = SystemManager(server=CASSANDRA_HOSTS[0])
sys.create_column_family(CASSANDRA_SESSIONS_KEYSPACE, CASSANDRA_SESSIONS_COLUMN_FAMILY,
key_validation_class=types.UTF8Type(),
column_validation_classes={
'session_data': types.UTF8Type()
})
sys.close()
def _create_cf(self, cf_name):
sys_mng = SystemManager('localhost:9160')
validators = {'name': UTF8_TYPE, 'content': UTF8_TYPE, 'post_time': UTF8_TYPE}
sys_mng.create_column_family(
'bbs',
cf_name,
super=False,
comparator_type=UTF8_TYPE,
key_validation_class=UTF8_TYPE,
column_validation_classes=validators
)
sys_mng.close()
def __init__(self, engine, **kw):
super(CassandraStore, self).__init__(engine, **kw)
spliturl = urlparse.urlsplit(engine)
_, keyspace, column_family = spliturl[2].split('/')
try:
self._pool = pycassa.connect(keyspace, [spliturl[1]])
self._store = pycassa.ColumnFamily(self._pool, column_family)
except pycassa.InvalidRequestException:
from pycassa.system_manager import SystemManager
system_manager = SystemManager(spliturl[1])
system_manager.create_keyspace(keyspace, kw.get('replication', 1))
system_manager.create_column_family(keyspace, column_family)
self._pool = pycassa.connect(keyspace, [spliturl[1]])
self._store = pycassa.ColumnFamily(self._pool, column_family)
def test_add_remove_counter(self):
sys = SystemManager()
sys.create_column_family(TEST_KS, 'KeyLongCounter', key_validation_class=LongType(),
default_validation_class='CounterColumnType')
sys.close()
cf_long = ColumnFamily(pool, 'KeyLongCounter')
key = 1111111111111111L
cf_long.add(key, 'col')
assert_equal(cf_long.get(key), {'col': 1})
cf_long.remove_counter(key, 'col')
time.sleep(0.1)
assert_raises(NotFoundException, cf_long.get, key)
def handle_noargs(self, **options):
sys = SystemManager(server=settings.CASSANDRA_SERVERS[0])
REPLICATION_STRATEGY = getattr(pycassa.system_manager, settings.CASSANDRA_REPLICATION_STRATEGY)
existing_cfs = sys.get_keyspace_column_families(settings.CASSANDRA_KEYSPACE).keys()
if 'APIConsumers' not in existing_cfs:
print 'Creating missing column family: APIConsumers'
sys.create_column_family(settings.CASSANDRA_KEYSPACE, 'APIConsumers', comparator_type=UTF8_TYPE)
sys.alter_column(settings.CASSANDRA_KEYSPACE, 'APIConsumers', 'consumer_key', UTF8_TYPE)
sys.alter_column(settings.CASSANDRA_KEYSPACE, 'APIConsumers', 'consumer_secret', UTF8_TYPE)
sys.alter_column(settings.CASSANDRA_KEYSPACE, 'APIConsumers', 'username', UTF8_TYPE)
print 'All done!'
def _init_db(server):
dbsys = SystemManager(server)
dbsys.create_keyspace(KEY_SPACE, SIMPLE_STRATEGY,
{"replication_factor": "1"})
column_family_type = types.CompositeType(
types.UTF8Type(),
types.UTF8Type(),
)
dbsys.create_column_family(
KEY_SPACE,
COLUMN_FAMILY,
comparator_type=column_family_type,
key_validation_class=types.UTF8Type(),
default_validation_class=types.UTF8Type(),
)
def cassandra_reset():
from feedly import settings
hostname = settings.FEEDLY_CASSANDRA_HOSTS[0]
keyspace = 'test_feedly'
sys = SystemManager(hostname)
# sys.drop_keyspace(keyspace)
if keyspace not in sys.list_keyspaces():
sys.create_keyspace(keyspace, SIMPLE_STRATEGY,
{'replication_factor': '1'})
sys.create_column_family(keyspace,
'activity',
comparator_type=UTF8_TYPE)
sys.create_column_family(keyspace,
'timeline',
comparator_type=IntegerType(reversed=True))
def setup_package():
sys = SystemManager()
if TEST_KS in sys.list_keyspaces():
sys.drop_keyspace(TEST_KS)
try:
sys.create_keyspace(TEST_KS, 'SimpleStrategy', {'replication_factor': '1'})
sys.create_column_family(TEST_KS, 'Standard1')
sys.create_column_family(TEST_KS, 'Super1', super=True)
sys.create_column_family(TEST_KS, 'Indexed1')
sys.create_index(TEST_KS, 'Indexed1', 'birthdate', 'LongType')
sys.create_column_family(TEST_KS, 'Counter1',
default_validation_class='CounterColumnType')
sys.create_column_family(TEST_KS, 'SuperCounter1', super=True,
default_validation_class='CounterColumnType')
except Exception, e:
print e
try:
sys.drop_keyspace(TEST_KS)
except:
pass
raise e
def __init__(self, engine, **kw):
super(CassandraStore, self).__init__(engine, **kw)
spliturl = urlsplit(engine)
_, keyspace, column_family = spliturl.path.split('/')
try:
self._store = pycassa.ColumnFamily(
pycassa.ConnectionPool(keyspace, [spliturl.hostname]),
column_family,
)
except pycassa.InvalidRequestException:
from pycassa.system_manager import SystemManager # @UnresolvedImport @IgnorePep8
system_manager = SystemManager(spliturl[1])
system_manager.create_keyspace(
keyspace,
pycassa.system_manager.SIMPLE_STRATEGY,
dict(replication_factor=native(kw.get('replication', 1))),
)
system_manager.create_column_family(keyspace, column_family)
self._store = pycassa.ColumnFamily(
pycassa.ConnectionPool(keyspace, [spliturl.netloc]),
column_family,
)
def _create_column_family(self,
family,
bytes_columns=[],
key_validation_class=TIME_UUID_TYPE):
'''
Creates a column family of the name 'family' and sets any of
the names in the bytes_column list to have the BYTES_TYPE.
key_validation_class defaults to TIME_UUID_TYPE and could also
be ASCII_TYPE for md5 hash keys, like we use for 'inbound'
'''
sm = SystemManager(random.choice(self.server_list))
# sys.create_column_family(self.namespace, family, super=False)
sm.create_column_family(self.namespace,
family,
super=False,
key_validation_class=key_validation_class,
default_validation_class=TIME_UUID_TYPE,
column_name_class=ASCII_TYPE)
for column in bytes_columns:
sm.alter_column(self.namespace, family, column, BYTES_TYPE)
sm.close()
def test(host='localhost', keyspace_provided=None):
from pycassa.types import BytesType, CompositeType
from pycassa.system_manager import SystemManager, SIMPLE_STRATEGY
# Create a fake keyspace, if not provided
if not keyspace_provided:
import random
keyspace = '%s_%s' % ('json', random.randrange(1000000, 100000000))
else:
keyspace = keyspace_provided
# Connect to cluster and create keyspace
system_manager = SystemManager(host)
system_manager.create_keyspace(keyspace, SIMPLE_STRATEGY,
{'replication_factor': '1'})
try:
# Create CF with many CompositeFields
comparator = CompositeType(BytesType(), BytesType(), BytesType(),
BytesType(), BytesType(), BytesType(),
BytesType(), BytesType(), BytesType(),
BytesType(), BytesType(), BytesType(),
BytesType(), BytesType(), BytesType(),
BytesType(), BytesType(), BytesType(),
BytesType(), BytesType(), BytesType(),
BytesType(), BytesType(), BytesType(),
BytesType(), BytesType(), BytesType(),
BytesType(), BytesType(), BytesType())
system_manager.create_column_family(keyspace,
'json',
comparator_type=comparator)
# Connect to the KS/CF and save samples
db = CassandraJsonMapper.db(keyspace, 'json')
tests = 0
# ----------------------------------------------------------------------
# Test a complicated structure
sample_1 = {
'key1': {
'a': 1,
2: 'b',
'c': {
'd': 3,
'e': {
'f': True
},
'g': [
['h', 'i', 'j', 4, 5],
['k', 'l', 'm', 'n', 'o'],
],
'p': [{
'id': 6,
'q': 'r'
}, {
'id': 7,
's': 't'
}],
'u': [],
'v': None
}
}
}
db.save(sample_1)
if db.get('key1') != sample_1['key1']:
raise AssertionError(
'What was saved is not being equally returned.')
tests += 1
# ----------------------------------------------------------------------
# Test improper format missing values
sample_2 = {'key1': 1}
try:
db.save(sample_2)
raise AssertionError('sample_2 should have thrown a KeyError.')
except KeyError:
pass
tests += 1
# ----------------------------------------------------------------------
# Test saving multiple keys
sample_3 = {'key2': {2: 2}, 'key3': {3: 3}, 'key4': {4: 4}}
db.save(sample_3)
if db.get('key2') != {2:2} or db.get('key3') != {3:3} or \
db.get('key4') != {4:4}:
raise AssertionError('Not all keys in a json_payload were saved.')
tests += 1
# ----------------------------------------------------------------------
# Test delete
db.delete('key1')
if db.get('key1'):
raise AssertionError('Not all keys in were delted.')
tests += 1
# ----------------------------------------------------------------------
# Test deletes
db.delete(['key2', 'key3', 'key4'])
if db.get('key2') or db.get('key3') or db.get('key4'):
raise AssertionError('Not all keys in were delted.')
tests += 1
# ----------------------------------------------------------------------
# Test reading from fake keys
if db.get('fake_key') != {}:
raise AssertionError('A get(fake_key) should return {}.')
tests += 1
# ----------------------------------------------------------------------
# Test multi-threaded save
db.mt_save(sample_1)
db.mt_finish()
if db.get('key1') != sample_1['key1']:
raise AssertionError(
'What was saved is not being equally returned.')
tests += 1
# ----------------------------------------------------------------------
# print json_format(db.get('key1'))
print '{0}/{0} tests passed!'.format(tests)
except:
raise
finally:
# For debugging purposes
# raw_input('Press ENTER to continue...')
if not keyspace_provided:
# Delete the temporary KS's
system_manager.drop_keyspace(keyspace)
system_manager.close()
class DistributedCassandraLockTestCase(unittest.TestCase):
def setUp(self):
self.sysman = SystemManager()
self.sysman.create_keyspace(
TEST_KS, SIMPLE_STRATEGY,
dict(replication_factor='1')
)
self.sysman.create_column_family(TEST_KS, TEST_CF)
self.pool = ConnectionPool(TEST_KS)
self.cf = ColumnFamily(self.pool, TEST_CF)
def tearDown(self):
self.sysman.drop_keyspace(TEST_KS)
del self.sysman
def test_ttl(self):
l = CassandraDistributedRowLock(
self.pool, TEST_CF, "test_ttl",
ttl=2.0,
consistency_level=ConsistencyLevel.ONE,
timeout=1.0
)
try:
l.acquire()
self.assertEqual(1, len(l.read_lock_columns()))
time.sleep(3)
self.assertEqual(0, len(l.read_lock_columns()))
except:
raise
finally:
l.release()
self.assertEqual(0, len(l.read_lock_columns()))
def test_stale_lock(self):
l1 = CassandraDistributedRowLock(
self.pool, TEST_CF, "test_stale_lock",
ttl=TEST_TTL, consistency_level=ConsistencyLevel.ONE,
timeout=1.0
)
l2 = CassandraDistributedRowLock(
self.pool, TEST_CF, "test_stale_lock",
ttl=TEST_TTL, consistency_level=ConsistencyLevel.ONE,
timeout=9.0
)
try:
l1.acquire()
time.sleep(5)
try:
l2.acquire()
except:
raise
finally:
l2.release()
except:
raise
finally:
l1.release()
def test_stale_lock_with_fail(self):
l1 = CassandraDistributedRowLock(
self.pool, TEST_CF, "test_stale_lock",
ttl=TEST_TTL, consistency_level=ConsistencyLevel.ONE,
timeout=1.0
)
l2 = CassandraDistributedRowLock(
self.pool, TEST_CF, "test_stale_lock",
ttl=TEST_TTL, consistency_level=ConsistencyLevel.ONE,
timeout=9.0, fail_on_stale_lock=True
)
try:
l1.acquire()
time.sleep(2)
try:
l2.acquire()
assert False, "derp"
except StaleLockException:
print 'things are working smoothly'
except:
raise
finally:
l2.release()
except:
raise
finally:
l2.release()
def create_cfs(self):
"""
Creates the Cassandra Column Families (if not exist)
"""
sys_mgr = None
pool = None
try:
sys_mgr = SystemManager()
pool = ConnectionPool(settings.KEYSPACE,
server_list=settings.CASSANDRA_HOSTS)
try:
cf = ColumnFamily(pool, CF_LOGS)
except:
logger.info("create_cfs(): Creating column family %s", CF_LOGS)
#========================================
# Column key -> CompositeType
#========================================
# 1. UUID + Timestamp
# 2. Host / Origin
# 3. Application
# 4. Severiry
comparator = CompositeType(TimeUUIDType(), UTF8Type(),
UTF8Type(), UTF8Type())
sys_mgr.create_column_family(settings.KEYSPACE,
CF_LOGS,
comparator_type=comparator)
cf = ColumnFamily(pool, CF_LOGS)
# cf.get_count(str(uuid.uuid4()))
try:
cf = ColumnFamily(pool, CF_METADATA)
except:
logger.info("create_cfs(): Creating column family %s",
CF_METADATA)
sys_mgr.create_column_family(settings.KEYSPACE,
CF_METADATA,
comparator_type=UTF8Type())
cf = ColumnFamily(pool, CF_METADATA)
cf.get_count(str(uuid.uuid4()))
try:
cf = ColumnFamily(pool, CF_TIMESTAMP_BITMAP)
except:
logger.info("create_cfs(): Creating column family %s",
CF_TIMESTAMP_BITMAP)
sys_mgr.create_column_family(settings.KEYSPACE,
CF_TIMESTAMP_BITMAP,
comparator_type=IntegerType())
cf = ColumnFamily(pool, CF_TIMESTAMP_BITMAP)
try:
cf = ColumnFamily(pool, CF_MULTI_MESSAGELOGS)
except:
logger.info("create_cfs(): Creating column family %s",
CF_MULTI_MESSAGELOGS)
sys_mgr.create_column_family(settings.KEYSPACE,
CF_MULTI_MESSAGELOGS,
comparator_type=UTF8Type())
cf = ColumnFamily(pool, CF_MULTI_MESSAGELOGS)
sys_mgr.create_index(settings.KEYSPACE,
CF_MULTI_MESSAGELOGS,
'meta:host',
UTF8_TYPE,
index_name='multimsg_host_index')
sys_mgr.create_index(settings.KEYSPACE,
CF_MULTI_MESSAGELOGS,
'meta:application',
UTF8_TYPE,
index_name='multimsg_application_index')
sys_mgr.create_index(settings.KEYSPACE,
CF_MULTI_MESSAGELOGS,
'meta:status',
UTF8_TYPE,
index_name='multimsg_finish_status_index')
finally:
if pool:
pool.dispose()
if sys_mgr:
sys_mgr.close()