def CreateCF(connStr, keySpace, cfs, columesTypeList = None):
sysmgt = SystemManager(connStr)
list = sysmgt.list_keyspaces()
if keySpace in list:
#print('pyagent is existed')
pass
else:
sysmgt.create_keyspace(keySpace, SIMPLE_STRATEGY, {'replication_factor': '3'})
cfDict = sysmgt.get_keyspace_column_families(keySpace)
allCF = cfDict.keys()
for i in range(len(cfs)):
columnFamily = cfs[i]
#print(columnFamily)
if columesTypeList != None:
columes = columesTypeList[i]
else:
columes = None
if columnFamily in allCF:
#print(columnFamily + ' is existed')
pass
else:
#print(columnFamily + ' is creating')
sysmgt.create_column_family(keySpace, columnFamily, super=False,
comparator_type=UTF8_TYPE,
key_validation_class=ASCII_TYPE,
default_validation_class=UTF8_TYPE,
column_validation_classes=columes,
gc_grace_seconds=1000)
sysmgt.close()
class Cluster(object):
def __init__(self, name):
self.keyspaces = OrderedDict()
self.name = name
self._sm_client = None
cmcache.append('clusters', self)
def setclient(self, servers_list=[connection.DEFAULT_SERVER]):
for server in servers_list:
try:
self._sm_client = SystemManager(server)
return
except:
pass
if self._sm_client is None:
raise NoServerAvailable('Cannot create system_manager connections on server list %s' % str(servers_list))
def registerKeyspace(self, name, keyspc):
self.keyspaces[name] = keyspc
if self._sm_client is None:
self.setclient()
# try to actually register this keyspace to the system via
if not name in self._sm_client.list_keyspaces():
self._sm_client.create_keyspace(name, keyspec.strategy_class)
def __str__(self):
return self.name
def get_connection():
"""
Creates a connection to Cassandra.
Returs:
pool
"""
cassandra_host = os.environ.get('CASSANDRA_HOST', 'localhost')
sys_mgr = SystemManager()
try:
sys_mgr.describe_ring(KEYSPACE)
except:
sys_mgr.create_keyspace(KEYSPACE, SIMPLE_STRATEGY, {'replication_factor': '1'})
pool = ConnectionPool(KEYSPACE, server_list=[cassandra_host])
for cf_name in [CF_LOGS, CF_LOGS_BY_APP, CF_LOGS_BY_HOST, CF_LOGS_BY_SEVERITY]:
try:
cf = ColumnFamily(pool, cf_name)
except:
sys_mgr.create_column_family(KEYSPACE, cf_name, comparator_type=TimeUUIDType())
cf = ColumnFamily(pool, cf_name)
cf.get_count(str(uuid.uuid4()))
sys_mgr.close()
return pool
def __init__(self, engine, **kw):
super(CassandraStore, self).__init__(engine, **kw)
spliturl = urlsplit(engine)
_, keyspace, column_family = spliturl.path.split('/')
try:
self._store = pycassa.ColumnFamily(
pycassa.ConnectionPool(keyspace, [spliturl.hostname]),
column_family,
)
except pycassa.InvalidRequestException:
from pycassa.system_manager import SystemManager # @UnresolvedImport @IgnorePep8
system_manager = SystemManager(spliturl[1])
system_manager.create_keyspace(
keyspace,
pycassa.system_manager.SIMPLE_STRATEGY,
dict(replication_factor=native(kw.get('replication', 1))),
)
system_manager.create_column_family(keyspace, column_family)
self._store = pycassa.ColumnFamily(
pycassa.ConnectionPool(keyspace, [spliturl.netloc]),
column_family,
)
except pycassa.NotFoundException:
from pycassa.system_manager import SystemManager # @UnresolvedImport @IgnorePep8
system_manager = SystemManager(spliturl[1])
system_manager.create_column_family(keyspace, column_family)
self._store = pycassa.ColumnFamily(
pycassa.ConnectionPool(keyspace, [spliturl.netloc]),
column_family,
)
def initializeTableLayout(keyspace, server_list, replicationStrategy,
strategyOptions, localDCName, credentials):
sys_manager = SystemManager(server_list[0], credentials=credentials)
# Make sure the the keyspace exists
if keyspace not in sys_manager.list_keyspaces():
sys_manager.create_keyspace(keyspace, replicationStrategy,
strategyOptions)
cf_defs = sys_manager.get_keyspace_column_families(keyspace)
# Create UTF8 CF's
for tablename in ["global_nodes", "metadata"]:
if tablename not in cf_defs.keys():
createUTF8ColumnFamily(sys_manager, keyspace, tablename)
if localDCName:
dcNodes = "dc_%s_nodes" % (localDCName,)
if dcNodes not in cf_defs.keys():
createUTF8ColumnFamily(sys_manager, keyspace, dcNodes)
else:
# TODO Log we do not have the DC name
pass
if "node_slices" not in cf_defs.keys():
sys_manager.create_column_family(
keyspace,
"node_slices",
super=False,
comparator_type=pycassa_types.LongType(),
key_validation_class=pycassa_types.UTF8Type(),
default_validation_class=pycassa_types.LongType()
)
def ensure_cassandra_cf(self):
s = SystemManager()
if self.keyspace not in s.list_keyspaces():
s.create_keyspace(self.keyspace, SIMPLE_STRATEGY, {'replication_factor': '1'})
if self.cassandra_columns_family not in s.get_keyspace_column_families(self.keyspace):
s.create_column_family(self.keyspace, self.cassandra_columns_family)
self.columnfamily = ColumnFamily(self.cassandra_session, self.cassandra_columns_family)
def _cassandra_ensure_keyspace(self, server_list, keyspace_name,
cf_info_list):
# Retry till cassandra is up
server_idx = 0
num_dbnodes = len(self._server_list)
connected = False
while not connected:
try:
cass_server = self._server_list[server_idx]
sys_mgr = SystemManager(cass_server)
connected = True
except Exception as e:
# TODO do only for
# thrift.transport.TTransport.TTransportException
server_idx = (server_idx + 1) % num_dbnodes
time.sleep(3)
if self._reset_config:
try:
sys_mgr.drop_keyspace(keyspace_name)
except pycassa.cassandra.ttypes.InvalidRequestException as e:
# TODO verify only EEXISTS
self._logger("Warning! " + str(e), level=SandeshLevel.SYS_WARN)
try:
sys_mgr.create_keyspace(keyspace_name, SIMPLE_STRATEGY,
{'replication_factor': str(num_dbnodes)})
except pycassa.cassandra.ttypes.InvalidRequestException as e:
# TODO verify only EEXISTS
self._logger("Warning! " + str(e), level=SandeshLevel.SYS_WARN)
gc_grace_sec = 0
if num_dbnodes > 1:
gc_grace_sec = 60
for cf_info in cf_info_list:
try:
(cf_name, comparator_type) = cf_info
if comparator_type:
sys_mgr.create_column_family(
keyspace_name,
cf_name,
comparator_type=comparator_type,
gc_grace_seconds=gc_grace_sec,
default_validation_class='UTF8Type')
else:
sys_mgr.create_column_family(
keyspace_name,
cf_name,
gc_grace_seconds=gc_grace_sec,
default_validation_class='UTF8Type')
except pycassa.cassandra.ttypes.InvalidRequestException as e:
# TODO verify only EEXISTS
self._logger("Warning! " + str(e), level=SandeshLevel.SYS_WARN)
sys_mgr.alter_column_family(
keyspace_name,
cf_name,
gc_grace_seconds=gc_grace_sec,
default_validation_class='UTF8Type')
def _cassandra_ensure_keyspace(self, server_list,
keyspace_name, cf_info_list):
# Retry till cassandra is up
server_idx = 0
num_dbnodes = len(self._server_list)
connected = False
while not connected:
try:
cass_server = self._server_list[server_idx]
sys_mgr = SystemManager(cass_server)
connected = True
except Exception as e:
# TODO do only for
# thrift.transport.TTransport.TTransportException
server_idx = (server_idx + 1) % num_dbnodes
time.sleep(3)
if self._reset_config:
try:
sys_mgr.drop_keyspace(keyspace_name)
except pycassa.cassandra.ttypes.InvalidRequestException as e:
# TODO verify only EEXISTS
self._logger("Warning! " + str(e), level=SandeshLevel.SYS_WARN)
try:
sys_mgr.create_keyspace(keyspace_name, SIMPLE_STRATEGY,
{'replication_factor': str(num_dbnodes)})
except pycassa.cassandra.ttypes.InvalidRequestException as e:
# TODO verify only EEXISTS
self._logger("Warning! " + str(e), level=SandeshLevel.SYS_WARN)
gc_grace_sec = 0
if num_dbnodes > 1:
gc_grace_sec = 60
for cf_info in cf_info_list:
try:
(cf_name, comparator_type) = cf_info
if comparator_type:
sys_mgr.create_column_family(
keyspace_name, cf_name,
comparator_type=comparator_type,
gc_grace_seconds=gc_grace_sec,
default_validation_class='UTF8Type')
else:
sys_mgr.create_column_family(keyspace_name, cf_name,
gc_grace_seconds=gc_grace_sec,
default_validation_class='UTF8Type')
except pycassa.cassandra.ttypes.InvalidRequestException as e:
# TODO verify only EEXISTS
self._logger("Warning! " + str(e), level=SandeshLevel.SYS_WARN)
sys_mgr.alter_column_family(keyspace_name, cf_name,
gc_grace_seconds=gc_grace_sec,
default_validation_class='UTF8Type')
def ensure_cassandra_cf(self):
s = SystemManager()
if self.keyspace not in s.list_keyspaces():
s.create_keyspace(self.keyspace, SIMPLE_STRATEGY,
{'replication_factor': '1'})
if self.cassandra_columns_family not in s.get_keyspace_column_families(
self.keyspace):
s.create_column_family(self.keyspace,
self.cassandra_columns_family)
self.columnfamily = ColumnFamily(self.cassandra_session,
self.cassandra_columns_family)
def test_big_batched_writes():
## this is an m1.xlarge doing nothing but supporting this test
server = 'localhost:9160'
keyspace = 'testkeyspace_' + getpass.getuser().replace('-', '_')
family = 'testcf'
sm = SystemManager(server)
try:
sm.drop_keyspace(keyspace)
except pycassa.InvalidRequestException:
pass
sm.create_keyspace(keyspace, SIMPLE_STRATEGY, {'replication_factor': '1'})
sm.create_column_family(keyspace, family, super=False,
key_validation_class = LEXICAL_UUID_TYPE,
default_validation_class = LEXICAL_UUID_TYPE,
column_name_class = ASCII_TYPE)
sm.alter_column(keyspace, family, 'test', ASCII_TYPE)
sm.close()
pool = ConnectionPool(keyspace, [server], max_retries=10, pool_timeout=0, pool_size=10, timeout=120)
pool.fill()
pool.add_listener( Listener() )
## assert that we are using framed transport
conn = pool._q.get()
assert isinstance(conn.transport, thrift.transport.TTransport.TFramedTransport)
pool._q.put(conn)
try:
for num_rows in range(14, 20):
## write some data to cassandra using increasing data sizes
one_mb = ' ' * 2**20
rows = []
for i in xrange(num_rows):
key = uuid.uuid4()
rows.append((key, dict(test=one_mb)))
testcf = pycassa.ColumnFamily(pool, family)
with testcf.batch() as batch:
for (key, data_dict) in rows:
data_size = len(data_dict.values()[0])
logger.critical('adding %r with %.6f MB' % (key, float(data_size)/2**20))
batch.insert(key, data_dict)
logger.critical('%d rows written' % num_rows)
finally:
sm = SystemManager(server)
try:
sm.drop_keyspace(keyspace)
except pycassa.InvalidRequestException:
pass
sm.close()
logger.critical('clearing test keyspace: %r' % keyspace)
def __init__(self, engine, **kw):
super(CassandraStore, self).__init__(engine, **kw)
spliturl = urlparse.urlsplit(engine)
_, keyspace, column_family = spliturl[2].split('/')
try:
self._pool = pycassa.connect(keyspace, [spliturl[1]])
self._store = pycassa.ColumnFamily(self._pool, column_family)
except pycassa.InvalidRequestException:
from pycassa.system_manager import SystemManager
system_manager = SystemManager(spliturl[1])
system_manager.create_keyspace(keyspace, kw.get('replication', 1))
system_manager.create_column_family(keyspace, column_family)
self._pool = pycassa.connect(keyspace, [spliturl[1]])
self._store = pycassa.ColumnFamily(self._pool, column_family)
def create_keyspace(self):
"""
Creates the Cassandra Keyspace (if not exist)
"""
sys_mgr = None
try:
sys_mgr = SystemManager()
try:
sys_mgr.describe_ring(settings.KEYSPACE)
except:
logger.info("create_keyspace(): Creating keyspace %s", settings.KEYSPACE)
sys_mgr.create_keyspace(settings.KEYSPACE, SIMPLE_STRATEGY, {'replication_factor': '1'})
finally:
if sys_mgr:
sys_mgr.close()
def __init__(self, engine, **kw):
super(CassandraStore, self).__init__(engine, **kw)
spliturl = urlparse.urlsplit(engine)
_, keyspace, column_family = spliturl[2].split('/')
try:
self._pool = pycassa.connect(keyspace, [spliturl[1]])
self._store = pycassa.ColumnFamily(self._pool, column_family)
except pycassa.InvalidRequestException:
from pycassa.system_manager import SystemManager
system_manager = SystemManager(spliturl[1])
system_manager.create_keyspace(
keyspace, kw.get('replication', 1)
)
system_manager.create_column_family(keyspace, column_family)
self._pool = pycassa.connect(keyspace, [spliturl[1]])
self._store = pycassa.ColumnFamily(self._pool, column_family)
def cassandra_reset():
from feedly import settings
hostname = settings.FEEDLY_CASSANDRA_HOSTS[0]
keyspace = "test_feedly"
sys = SystemManager(hostname)
# sys.drop_keyspace(keyspace)
if keyspace not in sys.list_keyspaces():
sys.create_keyspace(keyspace, SIMPLE_STRATEGY, {"replication_factor": "1"})
sys.create_column_family(keyspace, "activity", comparator_type=UTF8_TYPE)
sys.create_column_family(keyspace, "timeline", comparator_type=IntegerType(reversed=True))
def create_keyspace(self):
"""
Creates the Cassandra Keyspace (if not exist)
"""
sys_mgr = None
try:
sys_mgr = SystemManager()
try:
sys_mgr.describe_ring(settings.KEYSPACE)
except:
logger.info("create_keyspace(): Creating keyspace %s",
settings.KEYSPACE)
sys_mgr.create_keyspace(settings.KEYSPACE, SIMPLE_STRATEGY,
{'replication_factor': '1'})
finally:
if sys_mgr:
sys_mgr.close()
class CassandraDbSysManager:
def __init__(self):
try:
logger.info("Connection to " + DB_IP + ":" + DB_PORT + " ...")
self.sysDB = SystemManager(DB_IP + ":" + DB_PORT)
logger.info("Connection to the systemManager established.")
except Exception as exep:
logger.warning("Could not connect to the DB '" + DB_IP + ":" + DB_PORT + "'. | " + str(exep))
def create_keyspace(self, KeyspaceName, replica_factor):
try:
logger.info(
"Creating the Keyspace '" + KeyspaceName + "' with replica_factor '" + str(replica_factor) + "'")
self.sysDB.create_keyspace(KeyspaceName, replica_factor)
logger.info("Keyspace created.")
except Exception as exep:
logger.warning("Could not create the keyspace '" + KeyspaceName + "'. | " + str(exep))
def drop_keyspace(self, KeyspaceName):
try:
logger.info("Dropping the Keyspace '" + KeyspaceName + "'")
self.sysDB.drop_keyspace(KeyspaceName)
logger.info("Keyspace dropped.")
except Exception as exep:
logger.warning("Could not drop the keyspace '" + KeyspaceName + "'. | " + str(exep))
def create_column_family(self, KeyspaceName, columnFamily):
try:
logger.info("Creating the Column family '" + columnFamily + "' into the keyspace '" + KeyspaceName + "'")
self.sysDB.create_column_family(KeyspaceName, columnFamily, super=False,
comparator_type=pycassa.system_manager.ASCII_TYPE)
logger.info("Column Family created.")
except Exception as exep:
logger.warning(
"Could not create the Column Family '" + columnFamily + "' into the keyspace '" + KeyspaceName + "'. | " + str(
exep))
def drop_column_family(self, KeyspaceName, columnFamily):
try:
logger.info("Dropping the Column Family '" + columnFamily + "' from the Keyspace '" + KeyspaceName + "'")
self.sysDB.drop_column_family(KeyspaceName, columnFamily)
logger.info("Column Family dropped.")
except Exception as exep:
logger.warning("Could not drop the column family '" + columnFamily + "'. | " + str(exep))
def _init_db(server):
dbsys = SystemManager(server)
dbsys.create_keyspace(KEY_SPACE, SIMPLE_STRATEGY,
{"replication_factor": "1"})
column_family_type = types.CompositeType(
types.UTF8Type(),
types.UTF8Type(),
)
dbsys.create_column_family(
KEY_SPACE,
COLUMN_FAMILY,
comparator_type=column_family_type,
key_validation_class=types.UTF8Type(),
default_validation_class=types.UTF8Type(),
)
def cassandra_reset():
from feedly import settings
hostname = settings.FEEDLY_CASSANDRA_HOSTS[0]
keyspace = 'test_feedly'
sys = SystemManager(hostname)
# sys.drop_keyspace(keyspace)
if keyspace not in sys.list_keyspaces():
sys.create_keyspace(keyspace, SIMPLE_STRATEGY,
{'replication_factor': '1'})
sys.create_column_family(keyspace,
'activity',
comparator_type=UTF8_TYPE)
sys.create_column_family(keyspace,
'timeline',
comparator_type=IntegerType(reversed=True))
def setup_package():
sys = SystemManager()
if TEST_KS in sys.list_keyspaces():
sys.drop_keyspace(TEST_KS)
try:
sys.create_keyspace(TEST_KS, 'SimpleStrategy', {'replication_factor': '1'})
sys.create_column_family(TEST_KS, 'Standard1')
sys.create_column_family(TEST_KS, 'Super1', super=True)
sys.create_column_family(TEST_KS, 'Indexed1')
sys.create_index(TEST_KS, 'Indexed1', 'birthdate', 'LongType')
sys.create_column_family(TEST_KS, 'Counter1',
default_validation_class='CounterColumnType')
sys.create_column_family(TEST_KS, 'SuperCounter1', super=True,
default_validation_class='CounterColumnType')
except Exception, e:
print e
try:
sys.drop_keyspace(TEST_KS)
except:
pass
raise e
def setup_namespace(self, table_names):
if self.pool:
self.pool.dispose()
del self.pool
self.pool = None
start_connect_time = time.time()
super(CStorage, self).setup_namespace(table_names)
sm = SystemManager(self._chosen_server)
try:
sm.create_keyspace(
self._app_namespace, SIMPLE_STRATEGY,
{
'replication_factor': str(self._config.get('replication_factor', '1'))
},
)
except pycassa.InvalidRequestException, exc:
if exc.why.startswith('Keyspace names must be case-insensitively unique'):
pass
else:
raise exc
def __init__(self, engine, **kw):
super(CassandraStore, self).__init__(engine, **kw)
spliturl = urlsplit(engine)
_, keyspace, column_family = spliturl.path.split('/')
try:
self._store = pycassa.ColumnFamily(
pycassa.ConnectionPool(keyspace, [spliturl.hostname]),
column_family,
)
except pycassa.InvalidRequestException:
from pycassa.system_manager import SystemManager # @UnresolvedImport @IgnorePep8
system_manager = SystemManager(spliturl[1])
system_manager.create_keyspace(
keyspace,
pycassa.system_manager.SIMPLE_STRATEGY,
dict(replication_factor=native(kw.get('replication', 1))),
)
system_manager.create_column_family(keyspace, column_family)
self._store = pycassa.ColumnFamily(
pycassa.ConnectionPool(keyspace, [spliturl.netloc]),
column_family,
)
class ComedySysManagerConnection(object):
"""
This is a also a client connection of Cassandra, difference is it is only for System-Manager level
"""
def __init__(self, server='localhost:9160', credentials=None, framed_transport=True,
timeout=_DEFAULT_TIMEOUT):
self.conn = SystemManager(server,credentials,framed_transport,timeout)
def create_keyspace(self, name, replication_strategy='SimpleStrategy', strategy_options=None, durable_writes=True, **ks_kwargs):
"""Create a database"""
self.conn.create_keyspace(name, replication_strategy, strategy_options, durable_writes)
create_database = create_keyspace
def drop_keyspace(self, keyspace):
"""Drop a database"""
self.conn.drop_keyspace(keyspace)
drop_database = drop_keyspace
def create_column_family(self, keyspace, name, column_validation_classes=None, **cf_kwargs):
"""Create a table"""
self.conn.create_column_family(keyspace, name, column_validation_classes=None, **cf_kwargs)
create_table = create_column_family
def drop_column_family(self, keyspace, column_family):
"""Drop a table"""
self.conn.drop_column_family(keyspace, column_family)
drop_table = drop_column_family
def close(self):
pass
def test(host='localhost', keyspace_provided=None):
from pycassa.types import BytesType, CompositeType
from pycassa.system_manager import SystemManager, SIMPLE_STRATEGY
# Create a fake keyspace, if not provided
if not keyspace_provided:
import random
keyspace = '%s_%s' % ('json', random.randrange(1000000, 100000000))
else:
keyspace = keyspace_provided
# Connect to cluster and create keyspace
system_manager = SystemManager(host)
system_manager.create_keyspace(keyspace, SIMPLE_STRATEGY,
{'replication_factor': '1'})
try:
# Create CF with many CompositeFields
comparator = CompositeType(BytesType(), BytesType(), BytesType(),
BytesType(), BytesType(), BytesType(),
BytesType(), BytesType(), BytesType(),
BytesType(), BytesType(), BytesType(),
BytesType(), BytesType(), BytesType(),
BytesType(), BytesType(), BytesType(),
BytesType(), BytesType(), BytesType(),
BytesType(), BytesType(), BytesType(),
BytesType(), BytesType(), BytesType(),
BytesType(), BytesType(), BytesType())
system_manager.create_column_family(keyspace, 'json',
comparator_type=comparator)
# Connect to the KS/CF and save samples
db = CassandraJsonMapper.db(keyspace, 'json')
tests = 0
# ----------------------------------------------------------------------
# Test a complicated structure
sample_1 = {
'key1': {
'a': 1,
2: 'b',
'c': {
'd': 3,
'e': {
'f': True
},
'g': [
[
'h',
'i',
'j',
4,
5
], [
'k',
'l',
'm',
'n',
'o'
],
],
'p': [
{
'id': 6,
'q': 'r'
}, {
'id': 7,
's': 't'
}
],
'u': [],
'v': None
}
}
}
db.save(sample_1)
if db.get('key1') != sample_1['key1']:
raise AssertionError('What was saved is not being equally returned.')
tests += 1
# ----------------------------------------------------------------------
# Test improper format missing values
sample_2 = {
'key1': 1
}
try:
db.save(sample_2)
raise AssertionError('sample_2 should have thrown a KeyError.')
except KeyError:
pass
tests += 1
# ----------------------------------------------------------------------
# Test saving multiple keys
sample_3 = {
'key2': {2:2},
'key3': {3:3},
'key4': {4:4}
}
db.save(sample_3)
if db.get('key2') != {2:2} or db.get('key3') != {3:3} or \
db.get('key4') != {4:4}:
raise AssertionError('Not all keys in a json_payload were saved.')
tests += 1
# ----------------------------------------------------------------------
# Test delete
db.delete('key1')
if db.get('key1'):
raise AssertionError('Not all keys in were delted.')
tests += 1
# ----------------------------------------------------------------------
# Test deletes
db.delete(['key2', 'key3', 'key4'])
if db.get('key2') or db.get('key3') or db.get('key4'):
raise AssertionError('Not all keys in were delted.')
tests += 1
# ----------------------------------------------------------------------
# Test reading from fake keys
if db.get('fake_key') != {}:
raise AssertionError('A get(fake_key) should return {}.')
tests += 1
# ----------------------------------------------------------------------
# Test multi-threaded save
db.mt_save(sample_1)
db.mt_finish()
if db.get('key1') != sample_1['key1']:
raise AssertionError('What was saved is not being equally returned.')
tests += 1
# ----------------------------------------------------------------------
# print json_format(db.get('key1'))
print '{0}/{0} tests passed!'.format(tests)
except:
raise
finally:
# For debugging purposes
# raw_input('Press ENTER to continue...')
if not keyspace_provided:
# Delete the temporary KS's
system_manager.drop_keyspace(keyspace)
system_manager.close()
def test_composite_column_names(client):
'''
examine the unique nature of cassandras "wide sorted rows";
Basically, C* is good at storing large OrderedDict objects and
less good at the simpler kind of key=value storage that we were
doing earlier.
'''
config = client._config
namespace = client._app_namespace
chosen_server = client._chosen_server
sm = SystemManager(chosen_server)
sm.create_keyspace(namespace, SIMPLE_STRATEGY, {'replication_factor': '1'})
family = 'test'
sm.create_column_family(
namespace, family, super=False,
key_validation_class = ASCII_TYPE,
default_validation_class = BYTES_TYPE,
comparator_type=CompositeType(UUIDType(), UUIDType()),
#column_name_class = LEXICAL_UUID_TYPE
)
logger.info( sm.describe_schema_versions() )
pool = ConnectionPool(namespace, config['storage_addresses'],
max_retries=1000, pool_timeout=10, pool_size=2, timeout=120)
cf = pycassa.ColumnFamily(pool, family)
u1, u2, u3, u4 = uuid.uuid1(), uuid.uuid1(), uuid.uuid1(), uuid.uuid1()
## insert four
cf.insert('inbound', {(u1, u2): b''})
cf.insert('inbound', {(u1, u3): b''})
cf.insert('inbound', {(u1, u4): b''})
cf.insert('inbound', {(u1, u1): b'45'})
## insert three with duplicates
cf.insert('inbound', {(u3, u3): b'42'})
cf.insert('inbound', {(u3, u3): b'43'})
cf.insert('inbound', {(u3, u2): b''})
cf.insert('inbound', {(u3, u4): b''})
## insert two
cf.insert('inbound', {(u2, u3): b''})
cf.insert('inbound', {(u2, u4): b''})
## verify start/finish parameters work as expected
assert 4 == len(cf.get('inbound', column_start=(u1,), column_finish=(u1,)))
assert 2 == len(cf.get('inbound', column_start=(u2,), column_finish=(u2,)))
assert 3 == len(cf.get('inbound', column_start=(u3,), column_finish=(u3,)))
## check delete of a single column
cf.remove('inbound', columns=[(u2, u3)])
assert 1 == len(cf.get('inbound', column_start=(u2,), column_finish=(u2,)))
## check that the last inserted value appears at the expected
## location in the OrderedDict
assert '43' == cf.get('inbound', column_start=(u3,), column_finish=(u3,)).items()[1][1]
rec1 = cf.get('inbound', column_start=(u3,u3), column_finish=(u3,u3)).items()
assert len(rec1) == 1
start = uuid.UUID(int=0)
finish = uuid.UUID(int=2**128-1)
assert start < u1 < u2 < u3 < u4 < finish
assert 8 == len(cf.get('inbound', column_start=(start,), column_finish=(finish,)).items())
## test range searching
start = uuid.UUID(int=u3.int - 1)
finish = uuid.UUID(int=u3.int + 1)
assert start.int < u3.int < finish.int
rec2 = cf.get('inbound', column_start=(start,), column_finish=(finish,)).items()
assert rec2[1][0] == rec1[0][0]
assert rec2[1][1] == rec1[0][1] == '43'
cf.insert('inbound', {(u3,u3): b''.join(map(lambda u: u.bytes, (u1,u2,u3,u4)))})
data = cf.get('inbound', column_start=(u3,u3), column_finish=(u3,u3)).items()[0][1]
assert [u1,u2,u3,u4] == map(lambda b: uuid.UUID(bytes=b), grouper(data, 16))
sm.close()
def test_composite_column_names_second_level_range_query_with_decomposited_keys(client):
'''
check that we can execute range queries on the second part of a
CompositeType column name after we unpack the composite key into a
long string of concatenated hex forms of the UUIDs
'''
config = client._config
namespace = client._app_namespace
chosen_server = client._chosen_server
sm = SystemManager(chosen_server)
sm.create_keyspace(namespace, SIMPLE_STRATEGY, {'replication_factor': '1'})
family = 'test'
sm.create_column_family(
namespace, family, super=False,
key_validation_class = ASCII_TYPE,
default_validation_class = BYTES_TYPE,
comparator_type=UTF8Type(),
)
pool = ConnectionPool(namespace, config['storage_addresses'],
max_retries=1000, pool_timeout=10, pool_size=2, timeout=120)
cf = pycassa.ColumnFamily(pool, family)
u1, u2, u3, u4 = uuid.uuid1(), uuid.uuid1(), uuid.uuid1(), uuid.uuid1()
cf.insert('inbound', {join_uuids(u1, u2): b''})
cf.insert('inbound', {join_uuids(u1, u3): b''})
cf.insert('inbound', {join_uuids(u1, u4): b''})
## test range searching
start = uuid.UUID(int=u3.int - 1)
finish = uuid.UUID(int=u3.int + 1)
assert start.int < u3.int < finish.int
rec3 = cf.get('inbound',
column_start =join_uuids(u1, start),
column_finish=join_uuids(u1, finish)).items()
assert len(rec3) == 1
assert split_uuids(rec3[0][0])[1] == u3
#### This assert above passes!
#### This next part fails :-/
## now insert many rows -- enough that some should fall in each
## subrange below
for i in xrange(1000):
cf.insert('inbound', {join_uuids(u1, uuid.uuid4()): b''})
## do four ranges, and expect more than zero in each
step_size = 2**(128 - 2)
for i in range(2**2, 0, -1):
start = uuid.UUID(int=(i-1) * step_size)
finish = uuid.UUID(int=min(i * step_size, 2**128 - 1))
recs = cf.get('inbound',
column_start =join_uuids(u1, start),
column_finish=join_uuids(u1, finish)).items()
for key, val in recs:
key = split_uuids(key)
assert val == b''
assert key[0] == u1
assert key[1] < finish
assert start < key[1] ## this passes!!
assert len(recs) > 0
logger.info( '%r for %r %r' % (len(recs), start, finish))
sm.close()
class DistributedCassandraLockTestCase(unittest.TestCase):
def setUp(self):
self.sysman = SystemManager()
self.sysman.create_keyspace(
TEST_KS, SIMPLE_STRATEGY,
dict(replication_factor='1')
)
self.sysman.create_column_family(TEST_KS, TEST_CF)
self.pool = ConnectionPool(TEST_KS)
self.cf = ColumnFamily(self.pool, TEST_CF)
def tearDown(self):
self.sysman.drop_keyspace(TEST_KS)
del self.sysman
def test_ttl(self):
l = CassandraDistributedRowLock(
self.pool, TEST_CF, "test_ttl",
ttl=2.0,
consistency_level=ConsistencyLevel.ONE,
timeout=1.0
)
try:
l.acquire()
self.assertEqual(1, len(l.read_lock_columns()))
time.sleep(3)
self.assertEqual(0, len(l.read_lock_columns()))
except:
raise
finally:
l.release()
self.assertEqual(0, len(l.read_lock_columns()))
def test_stale_lock(self):
l1 = CassandraDistributedRowLock(
self.pool, TEST_CF, "test_stale_lock",
ttl=TEST_TTL, consistency_level=ConsistencyLevel.ONE,
timeout=1.0
)
l2 = CassandraDistributedRowLock(
self.pool, TEST_CF, "test_stale_lock",
ttl=TEST_TTL, consistency_level=ConsistencyLevel.ONE,
timeout=9.0
)
try:
l1.acquire()
time.sleep(5)
try:
l2.acquire()
except:
raise
finally:
l2.release()
except:
raise
finally:
l1.release()
def test_stale_lock_with_fail(self):
l1 = CassandraDistributedRowLock(
self.pool, TEST_CF, "test_stale_lock",
ttl=TEST_TTL, consistency_level=ConsistencyLevel.ONE,
timeout=1.0
)
l2 = CassandraDistributedRowLock(
self.pool, TEST_CF, "test_stale_lock",
ttl=TEST_TTL, consistency_level=ConsistencyLevel.ONE,
timeout=9.0, fail_on_stale_lock=True
)
try:
l1.acquire()
time.sleep(2)
try:
l2.acquire()
assert False, "derp"
except StaleLockException:
print 'things are working smoothly'
except:
raise
finally:
l2.release()
except:
raise
finally:
l2.release()
def test(host='localhost', keyspace_provided=None):
from pycassa.types import BytesType, CompositeType
from pycassa.system_manager import SystemManager, SIMPLE_STRATEGY
# Create a fake keyspace, if not provided
if not keyspace_provided:
import random
keyspace = '%s_%s' % ('json', random.randrange(1000000, 100000000))
else:
keyspace = keyspace_provided
# Connect to cluster and create keyspace
system_manager = SystemManager(host)
system_manager.create_keyspace(keyspace, SIMPLE_STRATEGY,
{'replication_factor': '1'})
try:
# Create CF with many CompositeFields
comparator = CompositeType(BytesType(), BytesType(), BytesType(),
BytesType(), BytesType(), BytesType(),
BytesType(), BytesType(), BytesType(),
BytesType(), BytesType(), BytesType(),
BytesType(), BytesType(), BytesType(),
BytesType(), BytesType(), BytesType(),
BytesType(), BytesType(), BytesType(),
BytesType(), BytesType(), BytesType(),
BytesType(), BytesType(), BytesType(),
BytesType(), BytesType(), BytesType())
system_manager.create_column_family(keyspace,
'json',
comparator_type=comparator)
# Connect to the KS/CF and save samples
db = CassandraJsonMapper.db(keyspace, 'json')
tests = 0
# ----------------------------------------------------------------------
# Test a complicated structure
sample_1 = {
'key1': {
'a': 1,
2: 'b',
'c': {
'd': 3,
'e': {
'f': True
},
'g': [
['h', 'i', 'j', 4, 5],
['k', 'l', 'm', 'n', 'o'],
],
'p': [{
'id': 6,
'q': 'r'
}, {
'id': 7,
's': 't'
}],
'u': [],
'v': None
}
}
}
db.save(sample_1)
if db.get('key1') != sample_1['key1']:
raise AssertionError(
'What was saved is not being equally returned.')
tests += 1
# ----------------------------------------------------------------------
# Test improper format missing values
sample_2 = {'key1': 1}
try:
db.save(sample_2)
raise AssertionError('sample_2 should have thrown a KeyError.')
except KeyError:
pass
tests += 1
# ----------------------------------------------------------------------
# Test saving multiple keys
sample_3 = {'key2': {2: 2}, 'key3': {3: 3}, 'key4': {4: 4}}
db.save(sample_3)
if db.get('key2') != {2:2} or db.get('key3') != {3:3} or \
db.get('key4') != {4:4}:
raise AssertionError('Not all keys in a json_payload were saved.')
tests += 1
# ----------------------------------------------------------------------
# Test delete
db.delete('key1')
if db.get('key1'):
raise AssertionError('Not all keys in were delted.')
tests += 1
# ----------------------------------------------------------------------
# Test deletes
db.delete(['key2', 'key3', 'key4'])
if db.get('key2') or db.get('key3') or db.get('key4'):
raise AssertionError('Not all keys in were delted.')
tests += 1
# ----------------------------------------------------------------------
# Test reading from fake keys
if db.get('fake_key') != {}:
raise AssertionError('A get(fake_key) should return {}.')
tests += 1
# ----------------------------------------------------------------------
# Test multi-threaded save
db.mt_save(sample_1)
db.mt_finish()
if db.get('key1') != sample_1['key1']:
raise AssertionError(
'What was saved is not being equally returned.')
tests += 1
# ----------------------------------------------------------------------
# print json_format(db.get('key1'))
print '{0}/{0} tests passed!'.format(tests)
except:
raise
finally:
# For debugging purposes
# raw_input('Press ENTER to continue...')
if not keyspace_provided:
# Delete the temporary KS's
system_manager.drop_keyspace(keyspace)
system_manager.close()
from pycassa.system_manager import SystemManager
from pycassa.system_manager import SIMPLE_STRATEGY
from pycassa.system_manager import UTF8_TYPE
from pycassa.system_manager import LONG_TYPE
from storage.cassandra_settings import HOSTS
from storage.cassandra_settings import KEYSPACE_NAME
sys = SystemManager(HOSTS[0])
if KEYSPACE_NAME in sys.list_keyspaces():
sys.drop_keyspace(KEYSPACE_NAME)
sys.create_keyspace(KEYSPACE_NAME, SIMPLE_STRATEGY, {'replication_factor': '1'})
sys.create_column_family(KEYSPACE_NAME, 'LoveActivity', comparator_type=UTF8_TYPE)
sys.create_column_family(KEYSPACE_NAME, 'Feed', comparator_type=UTF8_TYPE)
sys.create_column_family(KEYSPACE_NAME, 'AggregatedFeed', comparator_type=LONG_TYPE, super=True)
from pycassa.system_manager import SystemManager, SIMPLE_STRATEGY
from pycassa import InvalidRequestException
mgr = SystemManager()
try:
mgr.create_keyspace("craystack", SIMPLE_STRATEGY, {"replication_factor": "1"})
print "Created keyspace."
except InvalidRequestException:
print "Keyspace already exists!"
try:
mgr.create_column_family("craystack", "objects")
print "Created column family."
except InvalidRequestException:
print "Column family already exists!"
def _create_namespace(self, namespace):
sm = SystemManager(random.choice(self.server_list))
sm.create_keyspace(namespace, SIMPLE_STRATEGY,
{'replication_factor': '1'})
sm.close()
from pycassa import ConnectionPool
from pycassa.columnfamily import ColumnFamily
from pycassa.system_manager import SystemManager
from pycassa.system_manager import SIMPLE_STRATEGY
from pycassa.system_manager import UTF8_TYPE
from pycassa.system_manager import ASCII_TYPE
import time
############################## Create Keyspace ################################
server_list = ['cassandra1:9160', 'cassandra2:9160', 'cassandra3:9160']
sys = SystemManager(server_list[0])
sys.list_keyspaces()
if 'entries' in sys.list_keyspaces():
sys.drop_keyspace('entries')
sys.create_keyspace('entries', SIMPLE_STRATEGY, {'replication_factor': '1'})
############################## Connection Pooling ############################
#pool = ConnectionPool('entries', server_list=server_list, pool_size=20)
pool = ConnectionPool('entries', server_list=server_list)
############################## Create Column Family ###########################
sys.create_column_family('entries', 'Author', comparator_type=UTF8_TYPE)
author_cf = ColumnFamily(pool, 'Author')
################################ INSERT #######################################
# Insert a row with a Column
author_cf.insert('sacharya', {'first_name': 'Sudarshan'})
# Insert a row with multiple columns
author_cf.insert('sacharya1', {'first_name': 'Sudarshan', 'last_name':
'Acharya'})
def _create_namespace(self, namespace):
sm = SystemManager(random.choice(self.server_list))
sm.create_keyspace(namespace, SIMPLE_STRATEGY, {'replication_factor': '1'})
sm.close()
"""
Create the Cassandra database.
"""
import pycassa
from pycassa.system_manager import SystemManager
mgr = SystemManager()
mgr.create_keyspace('drought', strategy_options={'replication_factor': '1'})
mgr.create_column_family('drought', 'cmip5')
mgr.create_column_family('drought', 'zipcodes')
mgr.alter_column('drought', 'zipcodes', 'ZIPCODE', pycassa.types.IntegerType())
mgr.create_index('drought', 'zipcodes', 'ZIPCODE', pycassa.types.IntegerType())
mgr.alter_column('drought', 'zipcodes', 'CENTER_LATITUDE', pycassa.types.FloatType())
mgr.alter_column('drought', 'zipcodes', 'CENTER_LONGITUDE', pycassa.types.FloatType())
mgr.create_column_family('drought', 'counties')
mgr.close()
