class Cluster(object):
def __init__(self, name):
self.keyspaces = OrderedDict()
self.name = name
self._sm_client = None
cmcache.append('clusters', self)
def setclient(self, servers_list=[connection.DEFAULT_SERVER]):
for server in servers_list:
try:
self._sm_client = SystemManager(server)
return
except:
pass
if self._sm_client is None:
raise NoServerAvailable('Cannot create system_manager connections on server list %s' % str(servers_list))
def registerKeyspace(self, name, keyspc):
self.keyspaces[name] = keyspc
if self._sm_client is None:
self.setclient()
# try to actually register this keyspace to the system via
if not name in self._sm_client.list_keyspaces():
self._sm_client.create_keyspace(name, keyspec.strategy_class)
def __str__(self):
return self.name
def tearDown(self):
if self.store._store is not None:
self.store.clear()
self.store.close()
from pycassa.system_manager import SystemManager # @UnresolvedImport @IgnorePep8
system_manager = SystemManager('localhost:9160')
system_manager.drop_column_family('Murk', 'shove')
def tearDown(self):
if self.store._store is not None:
self.store.clear()
self.store.close()
from pycassa.system_manager import SystemManager # @UnresolvedImport @IgnorePep8
system_manager = SystemManager('localhost:9160')
system_manager.drop_column_family('Murk', 'shove')
def db_export(self):
db_contents = {'cassandra': {}, 'zookeeper': {}}
cassandra_contents = db_contents['cassandra']
for ks_name in (set(KEYSPACES) - set(self._args.omit_keyspaces or [])):
if self._api_args.cluster_id:
full_ks_name = '%s_%s' % (self._api_args.cluster_id, ks_name)
else:
full_ks_name = ks_name
cassandra_contents[ks_name] = {}
socket_factory = pycassa.connection.default_socket_factory
if self._api_args.cassandra_use_ssl:
socket_factory = pycassa.connection.make_ssl_socket_factory(
self._api_args.cassandra_ca_certs, validate=False)
pool = pycassa.ConnectionPool(full_ks_name,
self._api_args.cassandra_server_list,
pool_timeout=120,
max_retries=-1,
timeout=5,
socket_factory=socket_factory)
creds = None
if (self._api_args.cassandra_user
and self._api_args.cassandra_password):
creds = {
'username': self._api_args.cassandra_user,
'password': self._api_args.cassandra_password
}
sys_mgr = SystemManager(self._api_args.cassandra_server_list[0],
credentials=creds)
for cf_name in sys_mgr.get_keyspace_column_families(full_ks_name):
cassandra_contents[ks_name][cf_name] = {}
cf = pycassa.ColumnFamily(pool, cf_name)
for r, c in cf.get_range(column_count=10000000,
include_timestamp=True):
cassandra_contents[ks_name][cf_name][r] = c
def get_nodes(path):
if not zk.get_children(path):
return [(path, zk.get(path))]
nodes = []
for child in zk.get_children(path):
nodes.extend(get_nodes('%s%s/' % (path, child)))
return nodes
zk = kazoo.client.KazooClient(self._api_args.zk_server_ip)
zk.start()
nodes = get_nodes(self._api_args.cluster_id + '/')
zk.stop()
db_contents['zookeeper'] = json.dumps(nodes)
f = open(self._args.export_to, 'w')
try:
f.write(json.dumps(db_contents))
finally:
f.close()
def setup_class(cls):
sys = SystemManager()
sys.create_column_family(
TEST_KS,
'CustomComposite1',
comparator_type=CompositeType(
IntegerType(),
UTF8Type()))
def test_single_component_composite(self):
sys = SystemManager()
sys.create_column_family(TEST_KS, 'SingleComposite',
comparator_type=CompositeType(IntegerType()))
cf = ColumnFamily(pool, 'SingleComposite')
cf.insert('key', {(123456,): 'val'})
assert_equal(cf.get('key'), {(123456,): 'val'})
def setUp(self):
from shove import Shove
from pycassa.system_manager import SystemManager # @UnresolvedImport @IgnorePep8
system_manager = SystemManager('localhost:9160')
try:
system_manager.create_column_family('Murk', 'shove')
except:
pass
self.store = Shove('cassandra://localhost:9160/Murk/shove')
def setUp(self):
from shove import Shove
from pycassa.system_manager import SystemManager
system_manager = SystemManager('localhost:9160')
try:
system_manager.create_column_family('Foo', 'shove')
except:
pass
self.store = Shove('cassandra://localhost:9160/Foo/shove')
def setUp(self):
from shove import Shove
from pycassa.system_manager import SystemManager
system_manager = SystemManager('localhost:9160')
try:
system_manager.create_column_family('Foo', 'shove')
except:
pass
self.store = Shove('cassandra://localhost:9160/Foo/shove')
def setUp(self):
from shove import Shove
from pycassa.system_manager import SystemManager # @UnresolvedImport @IgnorePep8
system_manager = SystemManager('localhost:9160')
try:
system_manager.create_column_family('Murk', 'shove')
except:
pass
self.store = Shove('cassandra://localhost:9160/Murk/shove')
def delete_namespace(self):
sm = SystemManager(self._chosen_server)
try:
sm.drop_keyspace(self._app_namespace)
except pycassa.InvalidRequestException, exc:
if exc.why.startswith('Cannot drop non existing keyspace'):
pass
else:
raise exc
def db_export(self):
db_contents = {'cassandra': {},
'zookeeper': {}}
cassandra_contents = db_contents['cassandra']
for ks_name in (set(KEYSPACES) -
set(self._args.omit_keyspaces or [])):
if self._api_args.cluster_id:
full_ks_name = '%s_%s' %(self._api_args.cluster_id, ks_name)
else:
full_ks_name = ks_name
cassandra_contents[ks_name] = {}
socket_factory = pycassa.connection.default_socket_factory
if self._api_args.cassandra_use_ssl:
socket_factory = pycassa.connection.make_ssl_socket_factory(
self._api_args.cassandra_ca_certs, validate=False)
pool = pycassa.ConnectionPool(
full_ks_name, self._api_args.cassandra_server_list,
pool_timeout=120, max_retries=-1, timeout=5,
socket_factory=socket_factory)
creds = None
if (self._api_args.cassandra_user and
self._api_args.cassandra_password):
creds = {'username': self._api_args.cassandra_user,
'password': self._api_args.cassandra_password}
sys_mgr = SystemManager(self._api_args.cassandra_server_list[0],
credentials=creds)
for cf_name in sys_mgr.get_keyspace_column_families(full_ks_name):
cassandra_contents[ks_name][cf_name] = {}
cf = pycassa.ColumnFamily(pool, cf_name)
for r,c in cf.get_range(column_count=10000000, include_timestamp=True):
cassandra_contents[ks_name][cf_name][r] = c
def get_nodes(path):
if not zk.get_children(path):
return [(path, zk.get(path))]
nodes = []
for child in zk.get_children(path):
nodes.extend(get_nodes('%s%s/' %(path, child)))
return nodes
zk = kazoo.client.KazooClient(self._api_args.zk_server_ip)
zk.start()
nodes = get_nodes(self._api_args.cluster_id+'/')
zk.stop()
db_contents['zookeeper'] = json.dumps(nodes)
f = open(self._args.export_to, 'w')
try:
f.write(json.dumps(db_contents))
finally:
f.close()
def setup_class(cls):
sys = SystemManager()
sys.create_column_family(TEST_KS, 'StaticComposite',
comparator_type=CompositeType(LongType(),
IntegerType(),
TimeUUIDType(reversed=True),
LexicalUUIDType(reversed=False),
AsciiType(),
UTF8Type(),
BytesType()))
def CreateCF(connStr, keySpace, cfs, columesTypeList = None):
sysmgt = SystemManager(connStr)
list = sysmgt.list_keyspaces()
if keySpace in list:
#print('pyagent is existed')
pass
else:
sysmgt.create_keyspace(keySpace, SIMPLE_STRATEGY, {'replication_factor': '3'})
cfDict = sysmgt.get_keyspace_column_families(keySpace)
allCF = cfDict.keys()
for i in range(len(cfs)):
columnFamily = cfs[i]
#print(columnFamily)
if columesTypeList != None:
columes = columesTypeList[i]
else:
columes = None
if columnFamily in allCF:
#print(columnFamily + ' is existed')
pass
else:
#print(columnFamily + ' is creating')
sysmgt.create_column_family(keySpace, columnFamily, super=False,
comparator_type=UTF8_TYPE,
key_validation_class=ASCII_TYPE,
default_validation_class=UTF8_TYPE,
column_validation_classes=columes,
gc_grace_seconds=1000)
sysmgt.close()
def get_connection():
"""
Creates a connection to Cassandra.
Returs:
pool
"""
cassandra_host = os.environ.get('CASSANDRA_HOST', 'localhost')
sys_mgr = SystemManager()
try:
sys_mgr.describe_ring(KEYSPACE)
except:
sys_mgr.create_keyspace(KEYSPACE, SIMPLE_STRATEGY, {'replication_factor': '1'})
pool = ConnectionPool(KEYSPACE, server_list=[cassandra_host])
for cf_name in [CF_LOGS, CF_LOGS_BY_APP, CF_LOGS_BY_HOST, CF_LOGS_BY_SEVERITY]:
try:
cf = ColumnFamily(pool, cf_name)
except:
sys_mgr.create_column_family(KEYSPACE, cf_name, comparator_type=TimeUUIDType())
cf = ColumnFamily(pool, cf_name)
cf.get_count(str(uuid.uuid4()))
sys_mgr.close()
return pool
def main(argv):
usage = "Usage: dbdiscover.py"
args, kwargs = parse(argv)
host = kwargs.get('host', settings.DEFAULT_CASSANDRA_HOST)
port = kwargs.get('port', settings.DEFAULT_CASSANDRA_PORT)
try:
system = SystemManager("%s:%s" % (host, port))
ksprops = system.get_keyspace_properties("system").keys()
# header = ["keyspace"] + sorted(ksprops)
header = ["keyspace", "column_family"]
writer = csv.writer(output)
writer.writerow(header)
for keyspace in system.list_keyspaces():
ksinfo = system.get_keyspace_column_families(
keyspace, use_dict_for_col_metadata=True)
attrs = [
"id",
"column_type",
"comment",
"comparator_type",
"default_validation_class",
"gc_grace_seconds",
"key_alias",
"key_cache_save_period_in_seconds",
"key_cache_size",
"key_validation_class",
"max_compaction_threshold",
"memtable_operations_in_millions",
"memtable_throughput_in_mb",
"memtable_flush_after_mins",
"merge_shards_chance",
"min_compaction_threshold",
"read_repair_chance",
"replicate_on_write",
"row_cache_provider",
"row_cache_save_period_in_seconds",
"row_cache_size",
"subcomparator_type",
]
for cfname, cfdef in ksinfo.iteritems():
ks = ("keyspace=" + keyspace)
cf = ("column_family=" + cfname)
row = (ks, cf)
writer.writerow(row)
return
except:
error(output, excinfo(), 2)
def setUp(self):
self.sysman = SystemManager()
self.sysman.create_keyspace(
TEST_KS, SIMPLE_STRATEGY,
dict(replication_factor='1')
)
self.sysman.create_column_family(TEST_KS, TEST_CF)
self.pool = ConnectionPool(TEST_KS)
self.cf = ColumnFamily(self.pool, TEST_CF)
def main(argv):
usage = "Usage: dbdiscover.py"
args, kwargs = parse(argv)
host = kwargs.get('host', settings.DEFAULT_CASSANDRA_HOST)
port = kwargs.get('port', settings.DEFAULT_CASSANDRA_PORT)
try:
system = SystemManager("%s:%s" % (host, port))
ksprops = system.get_keyspace_properties("system").keys()
# header = ["keyspace"] + sorted(ksprops)
header = ["keyspace", "column_family"]
writer = csv.writer(output)
writer.writerow(header)
for keyspace in system.list_keyspaces():
ksinfo = system.get_keyspace_column_families(keyspace, use_dict_for_col_metadata=True)
attrs = [
"id",
"column_type",
"comment",
"comparator_type",
"default_validation_class",
"gc_grace_seconds",
"key_alias",
"key_cache_save_period_in_seconds",
"key_cache_size",
"key_validation_class",
"max_compaction_threshold",
"memtable_operations_in_millions",
"memtable_throughput_in_mb",
"memtable_flush_after_mins",
"merge_shards_chance",
"min_compaction_threshold",
"read_repair_chance",
"replicate_on_write",
"row_cache_provider",
"row_cache_save_period_in_seconds",
"row_cache_size",
"subcomparator_type",
]
for cfname, cfdef in ksinfo.iteritems():
ks = ("keyspace="+ keyspace)
cf = ("column_family="+ cfname)
row =(ks, cf)
writer.writerow(row)
return
except:
error(output, excinfo(), 2)
def handle(self, **options):
"""
:param options:
"""
sys = SystemManager(server=CASSANDRA_HOSTS[0])
sys.create_column_family(CASSANDRA_SESSIONS_KEYSPACE, CASSANDRA_SESSIONS_COLUMN_FAMILY,
key_validation_class=types.UTF8Type(),
column_validation_classes={
'session_data': types.UTF8Type()
})
sys.close()
def create():
keyspace = configuration.cassandra_keyspace
mgr = SystemManager()
try:
workaround_1779(mgr.create_column_family, keyspace, 'Indexes',
comparator_type=UTF8_TYPE)
workaround_1779(mgr.create_column_family, keyspace, 'Stacktrace',
comparator_type=UTF8_TYPE)
workaround_1779(mgr.create_column_family, keyspace, 'AwaitingRetrace',
comparator_type=UTF8_TYPE)
finally:
mgr.close()
def _create_cf(self, cf_name):
sys_mng = SystemManager('localhost:9160')
validators = {'name': UTF8_TYPE, 'content': UTF8_TYPE, 'post_time': UTF8_TYPE}
sys_mng.create_column_family(
'bbs',
cf_name,
super=False,
comparator_type=UTF8_TYPE,
key_validation_class=UTF8_TYPE,
column_validation_classes=validators
)
sys_mng.close()
def test_uuid_composites(self):
sys = SystemManager()
sys.create_column_family(TEST_KS, 'UUIDComposite',
comparator_type=CompositeType(IntegerType(reversed=True), TimeUUIDType()),
key_validation_class=TimeUUIDType(),
default_validation_class=UTF8Type())
key, u1, u2 = uuid.uuid1(), uuid.uuid1(), uuid.uuid1()
cf = ColumnFamily(pool, 'UUIDComposite')
cf.insert(key, {(123123, u1): 'foo'})
cf.insert(key, {(123123, u1): 'foo', (-1, u2): 'bar', (-123123123, u1): 'baz'})
assert_equal(cf.get(key), {(123123, u1): 'foo', (-1, u2): 'bar', (-123123123, u1): 'baz'})
def do_connect(self, server):
try:
self.sm = SystemManager(server)
except TTransportException:
return self.perror('Can not connect to %s' % server)
self.server = server
self.prompt = '%s> ' % server
self.keyspace = None
self.pool = None
print 'Successfully connected to %s' % server
def wait_for_consistency(self, sm=None):
if sm is None:
sm = SystemManager(self._chosen_server)
start_consistency_delay = time.time()
consistency_delay = 0
while len(sm.describe_schema_versions()) > 1 and \
consistency_delay < self.max_consistency_delay:
consistency_delay = time.time() - start_consistency_delay
if consistency_delay > 20:
logger.warn('waited %.1f seconds for cluster-wide consistency %r' % (
consistency_delay, sm.describe_schema_versions()))
time.sleep(0.2)
logger.info('number of schemas in cluster: %d' % len(sm.describe_schema_versions()))
def initializeTableLayout(keyspace, server_list, replicationStrategy,
strategyOptions, localDCName, credentials):
sys_manager = SystemManager(server_list[0], credentials=credentials)
# Make sure the the keyspace exists
if keyspace not in sys_manager.list_keyspaces():
sys_manager.create_keyspace(keyspace, replicationStrategy,
strategyOptions)
cf_defs = sys_manager.get_keyspace_column_families(keyspace)
# Create UTF8 CF's
for tablename in ["global_nodes", "metadata"]:
if tablename not in cf_defs.keys():
createUTF8ColumnFamily(sys_manager, keyspace, tablename)
if localDCName:
dcNodes = "dc_%s_nodes" % (localDCName,)
if dcNodes not in cf_defs.keys():
createUTF8ColumnFamily(sys_manager, keyspace, dcNodes)
else:
# TODO Log we do not have the DC name
pass
if "node_slices" not in cf_defs.keys():
sys_manager.create_column_family(
keyspace,
"node_slices",
super=False,
comparator_type=pycassa_types.LongType(),
key_validation_class=pycassa_types.UTF8Type(),
default_validation_class=pycassa_types.LongType()
)
def test_get_indexed_slices(self):
sys = SystemManager()
for cf, keys in self.type_groups:
sys.create_index(TEST_KS, cf.column_family, 'birthdate', LongType())
cf = ColumnFamily(pool, cf.column_family)
for key in keys:
cf.insert(key, {'birthdate': 1})
expr = create_index_expression('birthdate', 1)
clause = create_index_clause([expr])
rows = list(cf.get_indexed_slices(clause))
assert_equal(len(rows), len(keys))
for k, c in rows:
assert_true(k in keys)
assert_equal(c, {'birthdate': 1})
def test_add_remove_counter(self):
sys = SystemManager()
sys.create_column_family(TEST_KS, 'KeyLongCounter', key_validation_class=LongType(),
default_validation_class='CounterColumnType')
sys.close()
cf_long = ColumnFamily(pool, 'KeyLongCounter')
key = 1111111111111111L
cf_long.add(key, 'col')
assert_equal(cf_long.get(key), {'col': 1})
cf_long.remove_counter(key, 'col')
time.sleep(0.1)
assert_raises(NotFoundException, cf_long.get, key)
def handle_noargs(self, **options):
sys = SystemManager(server=settings.CASSANDRA_SERVERS[0])
REPLICATION_STRATEGY = getattr(pycassa.system_manager, settings.CASSANDRA_REPLICATION_STRATEGY)
existing_cfs = sys.get_keyspace_column_families(settings.CASSANDRA_KEYSPACE).keys()
if 'APIConsumers' not in existing_cfs:
print 'Creating missing column family: APIConsumers'
sys.create_column_family(settings.CASSANDRA_KEYSPACE, 'APIConsumers', comparator_type=UTF8_TYPE)
sys.alter_column(settings.CASSANDRA_KEYSPACE, 'APIConsumers', 'consumer_key', UTF8_TYPE)
sys.alter_column(settings.CASSANDRA_KEYSPACE, 'APIConsumers', 'consumer_secret', UTF8_TYPE)
sys.alter_column(settings.CASSANDRA_KEYSPACE, 'APIConsumers', 'username', UTF8_TYPE)
print 'All done!'
def __init__(self, engine, **kw):
super(CassandraStore, self).__init__(engine, **kw)
spliturl = urlsplit(engine)
_, keyspace, column_family = spliturl.path.split('/')
try:
self._store = pycassa.ColumnFamily(
pycassa.ConnectionPool(keyspace, [spliturl.hostname]),
column_family,
)
except pycassa.InvalidRequestException:
from pycassa.system_manager import SystemManager # @UnresolvedImport @IgnorePep8
system_manager = SystemManager(spliturl[1])
system_manager.create_keyspace(
keyspace,
pycassa.system_manager.SIMPLE_STRATEGY,
dict(replication_factor=native(kw.get('replication', 1))),
)
system_manager.create_column_family(keyspace, column_family)
self._store = pycassa.ColumnFamily(
pycassa.ConnectionPool(keyspace, [spliturl.netloc]),
column_family,
)
except pycassa.NotFoundException:
from pycassa.system_manager import SystemManager # @UnresolvedImport @IgnorePep8
system_manager = SystemManager(spliturl[1])
system_manager.create_column_family(keyspace, column_family)
self._store = pycassa.ColumnFamily(
pycassa.ConnectionPool(keyspace, [spliturl.netloc]),
column_family,
)
def __init__(self, engine, **kw):
super(CassandraStore, self).__init__(engine, **kw)
spliturl = urlparse.urlsplit(engine)
_, keyspace, column_family = spliturl[2].split('/')
try:
self._pool = pycassa.connect(keyspace, [spliturl[1]])
self._store = pycassa.ColumnFamily(self._pool, column_family)
except pycassa.InvalidRequestException:
from pycassa.system_manager import SystemManager
system_manager = SystemManager(spliturl[1])
system_manager.create_keyspace(
keyspace, kw.get('replication', 1)
)
system_manager.create_column_family(keyspace, column_family)
self._pool = pycassa.connect(keyspace, [spliturl[1]])
self._store = pycassa.ColumnFamily(self._pool, column_family)
def main(argv):
usage = "Usage: dbschema [options] [{resource}]"
args, kwargs = parse(argv)
host = kwargs.get('host', settings.DEFAULT_CASSANDRA_HOST)
port = kwargs.get('port', settings.DEFAULT_CASSANDRA_PORT)
try:
system = SystemManager("%s:%s" % (host, port))
# FOR TESTING ONLY
# print str(argv)
if len(argv) == 0:
list_keyspaces(system)
return
if len(argv) == 1:
ksname = argv[0]
list_keyspace(system, ksname)
if len(argv) == 2:
ksname = argv[0]
cfname = argv[1]
if cfname != 'ANY':
list_column_family(system, ksname, cfname)
else:
list_keyspace(system, ksname)
except:
error(output, excinfo(), 2)
def __init__(self):
try:
logger.info("Connection to " + DB_IP + ":" + DB_PORT + " ...")
self.sysDB = SystemManager(DB_IP + ":" + DB_PORT)
logger.info("Connection to the systemManager established.")
except Exception as exep:
logger.warning("Could not connect to the DB '" + DB_IP + ":" + DB_PORT + "'. | " + str(exep))
def _sys_mgr():
"""
Return a pycassa system manager connection object
"""
thrift_port = str(__salt__["config.option"]("cassandra.THRIFT_PORT"))
host = __salt__["config.option"]("cassandra.host")
return SystemManager("{}:{}".format(host, thrift_port))
def _sys_mgr():
'''
Return a pycassa system manager connection object
'''
thrift_port = str(__salt__['config.option']('cassandra.THRIFT_PORT'))
host = __salt__['config.option']('cassandra.host')
return SystemManager('{0}:{1}'.format(host, thrift_port))
def ensure_cassandra_cf(self):
s = SystemManager()
if self.keyspace not in s.list_keyspaces():
s.create_keyspace(self.keyspace, SIMPLE_STRATEGY, {'replication_factor': '1'})
if self.cassandra_columns_family not in s.get_keyspace_column_families(self.keyspace):
s.create_column_family(self.keyspace, self.cassandra_columns_family)
self.columnfamily = ColumnFamily(self.cassandra_session, self.cassandra_columns_family)
def _create_tables(self, table_names, sm=None):
if sm is None:
sm = SystemManager(self._chosen_server)
for family, num_uuids in table_names.items():
#CompositeType(*[UUIDType() for i in range(num_uuids)])
comparator = AsciiType()
try:
sm.create_column_family(
self._app_namespace, family, super=False,
key_validation_class = ASCII_TYPE,
default_validation_class = BYTES_TYPE,
comparator_type=comparator,
)
except pycassa.InvalidRequestException, exc:
if exc.why.startswith('Cannot add already existing column family'):
pass
else:
raise exc
def setclient(self, servers_list=[connection.DEFAULT_SERVER]):
for server in servers_list:
try:
self._sm_client = SystemManager(server)
return
except:
pass
if self._sm_client is None:
raise NoServerAvailable('Cannot create system_manager connections on server list %s' % str(servers_list))
def setup_module():
global pool, cf, scf, counter_cf, super_counter_cf, sysman
credentials = {'username': '******', 'password': '******'}
pool = ConnectionPool(keyspace='PycassaTestKeyspace', credentials=credentials)
cf = ColumnFamily(pool, 'Standard1')
scf = ColumnFamily(pool, 'Super1')
sysman = SystemManager()
counter_cf = ColumnFamily(pool, 'Counter1')
super_counter_cf = ColumnFamily(pool, 'SuperCounter1')
def _cassandra_ensure_keyspace(self, server_list, keyspace_name,
cf_info_list):
# Retry till cassandra is up
server_idx = 0
num_dbnodes = len(self._server_list)
connected = False
while not connected:
try:
cass_server = self._server_list[server_idx]
sys_mgr = SystemManager(cass_server)
connected = True
except Exception as e:
# TODO do only for
# thrift.transport.TTransport.TTransportException
server_idx = (server_idx + 1) % num_dbnodes
time.sleep(3)
if self._reset_config:
try:
sys_mgr.drop_keyspace(keyspace_name)
except pycassa.cassandra.ttypes.InvalidRequestException as e:
# TODO verify only EEXISTS
self._logger("Warning! " + str(e), level=SandeshLevel.SYS_WARN)
try:
sys_mgr.create_keyspace(keyspace_name, SIMPLE_STRATEGY,
{'replication_factor': str(num_dbnodes)})
except pycassa.cassandra.ttypes.InvalidRequestException as e:
# TODO verify only EEXISTS
self._logger("Warning! " + str(e), level=SandeshLevel.SYS_WARN)
for cf_info in cf_info_list:
try:
(cf_name, comparator_type) = cf_info
if comparator_type:
sys_mgr.create_column_family(
keyspace_name,
cf_name,
comparator_type=comparator_type)
else:
sys_mgr.create_column_family(keyspace_name, cf_name)
except pycassa.cassandra.ttypes.InvalidRequestException as e:
# TODO verify only EEXISTS
self._logger("Warning! " + str(e), level=SandeshLevel.SYS_WARN)
def ensure_cassandra_cf(self):
s = SystemManager()
if self.keyspace not in s.list_keyspaces():
s.create_keyspace(self.keyspace, SIMPLE_STRATEGY,
{'replication_factor': '1'})
if self.cassandra_columns_family not in s.get_keyspace_column_families(
self.keyspace):
s.create_column_family(self.keyspace,
self.cassandra_columns_family)
self.columnfamily = ColumnFamily(self.cassandra_session,
self.cassandra_columns_family)
def detect_nodes(seeds, keyspace):
from feedly import settings
if not settings.FEEDLY_DISCOVER_CASSANDRA_NODES:
logger.warning('cassandra nodes discovery is off')
return seeds
nodes = frozenset(seeds)
logging.info('retrieve nodes from seeds %r' % seeds)
for seed in seeds:
try:
sys_manager = SystemManager(seed)
except TTransportException:
logging.warning('%s is not a seed or is not reachable' % seed)
continue
ring_description = sys_manager.describe_ring(keyspace)
for ring_range in ring_description:
endpoint_details = ring_range.endpoint_details[0]
hostname = endpoint_details.host
port = getattr(endpoint_details, 'port', 9160)
nodes = nodes.union({'%s:%s' % (hostname, port), }, nodes)
break
return nodes
def create_cfs(self):
"""
Creates the Cassandra Column Families (if not exist)
"""
sys_mgr = None
pool = None
try:
sys_mgr = SystemManager()
pool = ConnectionPool(settings.KEYSPACE,
server_list=settings.CASSANDRA_HOSTS)
for cf_name in [
CF_LOGS, CF_LOGS_BY_APP, CF_LOGS_BY_HOST,
CF_LOGS_BY_SEVERITY
]:
try:
cf = ColumnFamily(pool, cf_name)
except:
logger.info("create_cfs(): Creating column family %s",
cf_name)
sys_mgr.create_column_family(
settings.KEYSPACE,
cf_name,
comparator_type=TimeUUIDType())
cf = ColumnFamily(pool, cf_name)
cf.get_count(str(uuid.uuid4()))
finally:
if pool:
pool.dispose()
if sys_mgr:
sys_mgr.close()
def detect_nodes(seeds, keyspace):
from feedly import settings
if not settings.FEEDLY_DISCOVER_CASSANDRA_NODES:
logger.warning('cassandra nodes discovery is off')
return seeds
nodes = frozenset(seeds)
logging.info('retrieve nodes from seeds %r' % seeds)
for seed in seeds:
try:
sys_manager = SystemManager(seed)
except TTransportException:
logging.warning('%s is not a seed or is not reachable' % seed)
continue
ring_description = sys_manager.describe_ring(keyspace)
for ring_range in ring_description:
endpoint_details = ring_range.endpoint_details[0]
hostname = endpoint_details.host
port = getattr(endpoint_details, 'port', 9160)
nodes = nodes.union({
'%s:%s' % (hostname, port),
}, nodes)
break
return nodes
def cassandra_reset():
from feedly import settings
hostname = settings.FEEDLY_CASSANDRA_HOSTS[0]
keyspace = 'test_feedly'
sys = SystemManager(hostname)
# sys.drop_keyspace(keyspace)
if keyspace not in sys.list_keyspaces():
sys.create_keyspace(keyspace, SIMPLE_STRATEGY,
{'replication_factor': '1'})
sys.create_column_family(keyspace,
'activity',
comparator_type=UTF8_TYPE)
sys.create_column_family(keyspace,
'timeline',
comparator_type=IntegerType(reversed=True))
def createTSColumnFamily(servers, keyspace, tableName, credentials=None):
"""Create a tsXX Column Family using one of the servers in the ``servers``
list and the ``keysapce`` and ``tableName``.
"""
for server in servers:
try:
manager = SystemManager(server, credentials=credentials)
createUTF8ColumnFamily(manager, keyspace, tableName, ts_table=True)
return None
except (Exception) as e:
# TODO: log when we know how to log
lastError = e
raise RuntimeError("Failed to create CF %s.%s using the server list %s, "\
"last error was %s" % (keyspace, tableName, servers, str(lastError)))
def create_keyspace(self):
"""
Creates the Cassandra Keyspace (if not exist)
"""
sys_mgr = None
try:
sys_mgr = SystemManager()
try:
sys_mgr.describe_ring(settings.KEYSPACE)
except:
logger.info("create_keyspace(): Creating keyspace %s",
settings.KEYSPACE)
sys_mgr.create_keyspace(settings.KEYSPACE, SIMPLE_STRATEGY,
{'replication_factor': '1'})
finally:
if sys_mgr:
sys_mgr.close()
def _cassandra_system_manager(self):
# Retry till cassandra is up
server_idx = 0
connected = False
while not connected:
try:
cass_server = self._server_list[server_idx]
sys_mgr = SystemManager(cass_server, credentials=self._credential)
connected = True
except Exception:
# TODO do only for
# thrift.transport.TTransport.TTransportException
server_idx = (server_idx + 1) % self._num_dbnodes
time.sleep(3)
return sys_mgr
def __init__(self, engine, **kw):
super(CassandraStore, self).__init__(engine, **kw)
spliturl = urlparse.urlsplit(engine)
_, keyspace, column_family = spliturl[2].split('/')
try:
self._pool = pycassa.connect(keyspace, [spliturl[1]])
self._store = pycassa.ColumnFamily(self._pool, column_family)
except pycassa.InvalidRequestException:
from pycassa.system_manager import SystemManager
system_manager = SystemManager(spliturl[1])
system_manager.create_keyspace(keyspace, kw.get('replication', 1))
system_manager.create_column_family(keyspace, column_family)
self._pool = pycassa.connect(keyspace, [spliturl[1]])
self._store = pycassa.ColumnFamily(self._pool, column_family)
def _create_column_family(self,
family,
bytes_columns=[],
key_validation_class=TIME_UUID_TYPE):
'''
Creates a column family of the name 'family' and sets any of
the names in the bytes_column list to have the BYTES_TYPE.
key_validation_class defaults to TIME_UUID_TYPE and could also
be ASCII_TYPE for md5 hash keys, like we use for 'inbound'
'''
sm = SystemManager(random.choice(self.server_list))
# sys.create_column_family(self.namespace, family, super=False)
sm.create_column_family(self.namespace,
family,
super=False,
key_validation_class=key_validation_class,
default_validation_class=TIME_UUID_TYPE,
column_name_class=ASCII_TYPE)
for column in bytes_columns:
sm.alter_column(self.namespace, family, column, BYTES_TYPE)
sm.close()
def _init_db(server):
dbsys = SystemManager(server)
dbsys.create_keyspace(KEY_SPACE, SIMPLE_STRATEGY,
{"replication_factor": "1"})
column_family_type = types.CompositeType(
types.UTF8Type(),
types.UTF8Type(),
)
dbsys.create_column_family(
KEY_SPACE,
COLUMN_FAMILY,
comparator_type=column_family_type,
key_validation_class=types.UTF8Type(),
default_validation_class=types.UTF8Type(),
)
def __init__(self, engine, **kw):
super(CassandraStore, self).__init__(engine, **kw)
spliturl = urlsplit(engine)
_, keyspace, column_family = spliturl.path.split('/')
try:
self._store = pycassa.ColumnFamily(
pycassa.ConnectionPool(keyspace, [spliturl.hostname]),
column_family,
)
except pycassa.InvalidRequestException:
from pycassa.system_manager import SystemManager # @UnresolvedImport @IgnorePep8
system_manager = SystemManager(spliturl[1])
system_manager.create_keyspace(
keyspace,
pycassa.system_manager.SIMPLE_STRATEGY,
dict(replication_factor=native(kw.get('replication', 1))),
)
system_manager.create_column_family(keyspace, column_family)
self._store = pycassa.ColumnFamily(
pycassa.ConnectionPool(keyspace, [spliturl.netloc]),
column_family,
)
def _clean_db(server):
dbsys = SystemManager(server)
dbsys.drop_keyspace(KEY_SPACE)
dbsys.close()
def _sys_mgr():
'''
Return a pycassa system manager connection object
'''
return SystemManager('{0}:{1}'.format(host, thrift_port))
def db_export(self):
db_contents = {'cassandra': {},
'zookeeper': {}}
cassandra_contents = db_contents['cassandra']
creds = None
if self._api_args.cassandra_user and self._api_args.cassandra_password:
creds = {'username': self._api_args.cassandra_user,
'password': self._api_args.cassandra_password}
socket_factory = default_socket_factory
if ('cassandra_use_ssl' in self._api_args and
self._api_args.cassandra_use_ssl):
socket_factory = self._make_ssl_socket_factory(
self._api_args.cassandra_ca_certs, validate=False)
sys_mgr = SystemManager(
self._api_args.cassandra_server_list[0],
credentials=creds,
socket_factory=socket_factory)
existing_keyspaces = sys_mgr.list_keyspaces()
for ks_name in set(KEYSPACES) - set(self._args.omit_keyspaces or []):
if self._api_args.cluster_id:
full_ks_name = '%s_%s' %(self._api_args.cluster_id, ks_name)
else:
full_ks_name = ks_name
if full_ks_name not in existing_keyspaces:
continue
cassandra_contents[ks_name] = {}
pool = ConnectionPool(
full_ks_name, self._api_args.cassandra_server_list,
pool_timeout=120, max_retries=-1, timeout=5,
socket_factory=socket_factory, credentials=creds)
for cf_name in sys_mgr.get_keyspace_column_families(full_ks_name):
cassandra_contents[ks_name][cf_name] = {}
cf = ColumnFamily(pool, cf_name,
buffer_size=self._args.buffer_size)
for r,c in cf.get_range(column_count=10000000, include_timestamp=True):
cassandra_contents[ks_name][cf_name][r] = c
logger.info("Cassandra DB dumped")
def get_nodes(path):
if path[:-1].rpartition('/')[-1] in self._zk_ignore_list:
return []
try:
if not zk.get_children(path):
return [(path, zk.get(path))]
except kazoo.exceptions.NoNodeError:
return []
nodes = []
for child in zk.get_children(path):
nodes.extend(get_nodes('%s%s/' %(path, child)))
return nodes
zk = kazoo.client.KazooClient(self._api_args.zk_server_ip)
zk.start()
nodes = get_nodes(self._api_args.cluster_id+'/')
zk.stop()
db_contents['zookeeper'] = json.dumps(nodes)
logger.info("Zookeeper DB dumped")
f = open(self._args.export_to, 'w')
try:
f.write(json.dumps(db_contents))
finally:
f.close()
logger.info("DB dump wrote to file %s" % self._args.export_to)
def setup_module():
global sys
sys = SystemManager()
def create_cfs(self):
"""
Creates the Cassandra Column Families (if not exist)
"""
sys_mgr = None
pool = None
try:
sys_mgr = SystemManager()
pool = ConnectionPool(settings.KEYSPACE,
server_list=settings.CASSANDRA_HOSTS)
try:
cf = ColumnFamily(pool, CF_LOGS)
except:
logger.info("create_cfs(): Creating column family %s", CF_LOGS)
#========================================
# Column key -> CompositeType
#========================================
# 1. UUID + Timestamp
# 2. Host / Origin
# 3. Application
# 4. Severiry
comparator = CompositeType(TimeUUIDType(), UTF8Type(),
UTF8Type(), UTF8Type())
sys_mgr.create_column_family(settings.KEYSPACE,
CF_LOGS,
comparator_type=comparator)
cf = ColumnFamily(pool, CF_LOGS)
# cf.get_count(str(uuid.uuid4()))
try:
cf = ColumnFamily(pool, CF_METADATA)
except:
logger.info("create_cfs(): Creating column family %s",
CF_METADATA)
sys_mgr.create_column_family(settings.KEYSPACE,
CF_METADATA,
comparator_type=UTF8Type())
cf = ColumnFamily(pool, CF_METADATA)
cf.get_count(str(uuid.uuid4()))
try:
cf = ColumnFamily(pool, CF_TIMESTAMP_BITMAP)
except:
logger.info("create_cfs(): Creating column family %s",
CF_TIMESTAMP_BITMAP)
sys_mgr.create_column_family(settings.KEYSPACE,
CF_TIMESTAMP_BITMAP,
comparator_type=IntegerType())
cf = ColumnFamily(pool, CF_TIMESTAMP_BITMAP)
try:
cf = ColumnFamily(pool, CF_MULTI_MESSAGELOGS)
except:
logger.info("create_cfs(): Creating column family %s",
CF_MULTI_MESSAGELOGS)
sys_mgr.create_column_family(settings.KEYSPACE,
CF_MULTI_MESSAGELOGS,
comparator_type=UTF8Type())
cf = ColumnFamily(pool, CF_MULTI_MESSAGELOGS)
sys_mgr.create_index(settings.KEYSPACE,
CF_MULTI_MESSAGELOGS,
'meta:host',
UTF8_TYPE,
index_name='multimsg_host_index')
sys_mgr.create_index(settings.KEYSPACE,
CF_MULTI_MESSAGELOGS,
'meta:application',
UTF8_TYPE,
index_name='multimsg_application_index')
sys_mgr.create_index(settings.KEYSPACE,
CF_MULTI_MESSAGELOGS,
'meta:status',
UTF8_TYPE,
index_name='multimsg_finish_status_index')
finally:
if pool:
pool.dispose()
if sys_mgr:
sys_mgr.close()
class CCli(Cmd, object):
prompt = '> '
continuation_prompt = '. '
timing = True
colors = True
debug = True
# O'Key, guys, you created good library with a lot of parameters, but,
# I think, you should test it before release, shouldn't?
case_insensitive = False
max_data_size = 35
max_rows = 50
def __init__(self, *args, **kwargs):
super(CCli, self).__init__(*args, **kwargs)
self.sm = None
self.pool = None
self.server = None
self.keyspace = None
self.settable['max_data_size'] = 'Maximum value symbols [0 = no truncating]'
self.settable['max_rows'] = 'Maximum rows to receive by one get'
def func_named(self, arg):
return super(CCli, self).func_named(arg[0])
@parse(Optional(server, default='locahost:9160'))
def do_connect(self, server):
try:
self.sm = SystemManager(server)
except TTransportException:
return self.perror('Can not connect to %s' % server)
self.server = server
self.prompt = '%s> ' % server
self.keyspace = None
self.pool = None
print 'Successfully connected to %s' % server
@check_connection
@parse(keyspace)
def do_use(self, keyspace):
if keyspace not in self.sm.list_keyspaces():
return self.perror('Unknown keyspace %s' % keyspace)
self.prompt = '%s/%s> ' % (self.server, keyspace)
self.keyspace = keyspace
self.pool = ConnectionPool(keyspace, server_list=[self.server])
print 'Using %s as default keyspace' % keyspace
def complete_use(self, text, line, begidx, endidx):
if not self.sm:
return []
return [x for x in self.sm.list_keyspaces() if x.startswith(text)]
@check_connection
@parse(Keyword('keyspaces') | Keyword('columnfamilies'))
def do_list(self, space):
return getattr(self, 'list_%s' % space)()
def list_keyspaces(self):
pt = PrettyTable()
pt.field_names = ['Keyspaces']
pt.align = 'l'
for ks in self.sm.list_keyspaces():
pt.add_row([ks])
print pt.get_string(sortby="Keyspaces")
@check_keyspace
def list_columnfamilies(self):
pt = PrettyTable()
pt.field_names = ['ColumnFamilies']
pt.align = 'l'
for cf in self.sm.get_keyspace_column_families(self.keyspace).keys():
pt.add_row([cf])
print pt.get_string(sortby="ColumnFamilies")
def complete_list(self, text, line, begidx, endidx):
return [x for x in ['keyspaces', 'columnfamilies'] if x.startswith(text)]
@check_connection
@parse(Keyword('keyspace') + Optional(keyspace, default=None) | Keyword('columnfamily') + columnfamily)
def do_describe(self, space, name):
return getattr(self, 'describe_%s' % space)(name)
def describe_keyspace(self, keyspace):
keyspace = keyspace or self.keyspace
if not keyspace:
return self.perror('Please, select working keyspace or define it as command parameter')
try:
options = self.sm.get_keyspace_properties(keyspace)
except NotFoundException:
return self.perror('Unknown keyspace %s' % keyspace)
pt = PrettyTable()
pt.field_names = ['Keyspace', keyspace]
pt.align["Keyspace"] = "l"
pt.align[keyspace] = 'r'
pt.add_row(['replication_strategy', options['replication_strategy']])
for k, v in options['strategy_options'].items():
pt.add_row([k, "\033[34m%s\033[0m" % v])
print pt.get_string(sortby='Keyspace')
@check_keyspace
def describe_columnfamily(self, columnfamily):
try:
options = self.sm.get_keyspace_column_families(self.keyspace, use_dict_for_col_metadata=True)[columnfamily]
except KeyError:
return self.perror('Unknown columnfamily %s' % columnfamily)
pt = PrettyTable()
pt.field_names = ['ColumnFamily', columnfamily]
pt.align["ColumnFamily"] = "l"
pt.align[columnfamily] = 'r'
for k, v in options.__dict__.items():
if k == 'column_metadata' and len(v):
continue
pt.add_row([k, v])
print pt.get_string(sortby='ColumnFamily')
if len(options.column_metadata):
pt = PrettyTable()
pt.field_names = ['Column \ Options'] + options.column_metadata.values()[0].__dict__.keys()
for k, v in options.column_metadata.items():
pt.add_row([k] + v.__dict__.values())
print pt.get_string(sortby='Column \ Options')
def complete_describe(self, text, line, begidx, endidx):
return [x for x in ['keyspace', 'columnfamily'] if x.startswith(text)]
def completenames(self, text, *ignored):
names = super(CCli, self).completenames(text, *ignored)
if self.keyspace:
names.extend(cf for cf in
self.sm.get_keyspace_column_families(self.keyspace).keys()
if cf.startswith(text))
return names
def default(self, line):
# Ugly, ugly things...
line = (' '.join(line.parsed)).strip()
if not self.server and not self.keyspace:
return super(CCli, self).default(line)
return self.simple_select(line)
@parse(columnfamily + Optional('[' + Combine(Optional(key, default='') +
Optional(':' + Optional(key, default='') + Optional(':' +
Optional(count, default='')))) + ']')
)
def simple_select(self, columnfamily, *args):
slice = ['', '', self.max_rows]
key = None
if args and args[1]:
if ':' not in args[1]:
key = args[1]
for i, part in enumerate(args[1].split(':', 2)):
slice[i] = part
try:
cf = ColumnFamily(self.pool, columnfamily)
except NotFoundException:
return super(CCli, self).default(' '.join([columnfamily] + list(args)))
if key:
pt = PrettyTable()
pt.field_names = ['Key', key]
pt.align["Key"] = "l"
pt.align[key] = 'r'
for k, v in cf.get(key).items():
pt.add_row([k, (v[:self.max_data_size - 3] + '...' if self.max_data_size and len(v) > self.max_data_size else v)])
print pt.get_string(sortby='Key')
return
data = dict(cf.get_range(start=slice[0], finish=slice[1], row_count=int(slice[2])))
columns = []
for key, row in data.items():
columns.extend(row.keys())
columns = list(set(columns))
columns.sort()
pt = PrettyTable()
pt.field_names = ['Key / Column'] + columns
pt.align["Key / Column"] = "l"
for column in columns:
pt.align[column] = "r"
for key, row in data.items():
prow = [key]
for column in columns:
value = row.get(column, '---')
if len(value) > self.max_data_size:
value = value[:self.max_data_size - 3] + '...'
prow.append(value)
pt.add_row(prow)
print pt.get_string(sortby='Key / Column')