def setup_databases(self, **kwargs):
settings.KEYSPACE = settings.KEYSPACE_TESTS
print "Patched value of KEYSPACE to '{0}'".format(settings.KEYSPACE)
if "dont_drop_daedalus_tests" not in os.environ:
sys_mgr = SystemManager()
try:
sys_mgr.drop_keyspace(settings.KEYSPACE)
print "Keyspace {0} droped OK".format(settings.KEYSPACE)
except:
pass
finally:
sys_mgr.close()
sys_mgr = SystemManager()
# keyspace 'settings.KEYSPACE' shouldn't exists
if settings.KEYSPACE in sys_mgr.list_keyspaces():
print "ERROR: keyspace {0} EXISTS after been droped".format(
settings.KEYSPACE)
print " - Keyspaces:", sys_mgr.list_keyspaces()
assert False, "ERROR"
sys_mgr.close()
storage.get_service().create_keyspace_and_cfs()
return super(CassandraDjangoTestSuiteRunner,
self).setup_databases(**kwargs)
def ensure_cassandra_cf(self):
s = SystemManager()
if self.keyspace not in s.list_keyspaces():
s.create_keyspace(self.keyspace, SIMPLE_STRATEGY, {'replication_factor': '1'})
if self.cassandra_columns_family not in s.get_keyspace_column_families(self.keyspace):
s.create_column_family(self.keyspace, self.cassandra_columns_family)
self.columnfamily = ColumnFamily(self.cassandra_session, self.cassandra_columns_family)
def CreateCF(connStr, keySpace, cfs, columesTypeList = None):
sysmgt = SystemManager(connStr)
list = sysmgt.list_keyspaces()
if keySpace in list:
#print('pyagent is existed')
pass
else:
sysmgt.create_keyspace(keySpace, SIMPLE_STRATEGY, {'replication_factor': '3'})
cfDict = sysmgt.get_keyspace_column_families(keySpace)
allCF = cfDict.keys()
for i in range(len(cfs)):
columnFamily = cfs[i]
#print(columnFamily)
if columesTypeList != None:
columes = columesTypeList[i]
else:
columes = None
if columnFamily in allCF:
#print(columnFamily + ' is existed')
pass
else:
#print(columnFamily + ' is creating')
sysmgt.create_column_family(keySpace, columnFamily, super=False,
comparator_type=UTF8_TYPE,
key_validation_class=ASCII_TYPE,
default_validation_class=UTF8_TYPE,
column_validation_classes=columes,
gc_grace_seconds=1000)
sysmgt.close()
class Cluster(object):
def __init__(self, name):
self.keyspaces = OrderedDict()
self.name = name
self._sm_client = None
cmcache.append('clusters', self)
def setclient(self, servers_list=[connection.DEFAULT_SERVER]):
for server in servers_list:
try:
self._sm_client = SystemManager(server)
return
except:
pass
if self._sm_client is None:
raise NoServerAvailable('Cannot create system_manager connections on server list %s' % str(servers_list))
def registerKeyspace(self, name, keyspc):
self.keyspaces[name] = keyspc
if self._sm_client is None:
self.setclient()
# try to actually register this keyspace to the system via
if not name in self._sm_client.list_keyspaces():
self._sm_client.create_keyspace(name, keyspec.strategy_class)
def __str__(self):
return self.name
def initializeTableLayout(keyspace, server_list, replicationStrategy,
strategyOptions, localDCName, credentials):
sys_manager = SystemManager(server_list[0], credentials=credentials)
# Make sure the the keyspace exists
if keyspace not in sys_manager.list_keyspaces():
sys_manager.create_keyspace(keyspace, replicationStrategy,
strategyOptions)
cf_defs = sys_manager.get_keyspace_column_families(keyspace)
# Create UTF8 CF's
for tablename in ["global_nodes", "metadata"]:
if tablename not in cf_defs.keys():
createUTF8ColumnFamily(sys_manager, keyspace, tablename)
if localDCName:
dcNodes = "dc_%s_nodes" % (localDCName,)
if dcNodes not in cf_defs.keys():
createUTF8ColumnFamily(sys_manager, keyspace, dcNodes)
else:
# TODO Log we do not have the DC name
pass
if "node_slices" not in cf_defs.keys():
sys_manager.create_column_family(
keyspace,
"node_slices",
super=False,
comparator_type=pycassa_types.LongType(),
key_validation_class=pycassa_types.UTF8Type(),
default_validation_class=pycassa_types.LongType()
)
def ensure_cassandra_cf(self):
s = SystemManager()
if self.keyspace not in s.list_keyspaces():
s.create_keyspace(self.keyspace, SIMPLE_STRATEGY,
{'replication_factor': '1'})
if self.cassandra_columns_family not in s.get_keyspace_column_families(
self.keyspace):
s.create_column_family(self.keyspace,
self.cassandra_columns_family)
self.columnfamily = ColumnFamily(self.cassandra_session,
self.cassandra_columns_family)
def main(argv):
usage = "Usage: dbdiscover.py"
args, kwargs = parse(argv)
host = kwargs.get('host', settings.DEFAULT_CASSANDRA_HOST)
port = kwargs.get('port', settings.DEFAULT_CASSANDRA_PORT)
try:
system = SystemManager("%s:%s" % (host, port))
ksprops = system.get_keyspace_properties("system").keys()
# header = ["keyspace"] + sorted(ksprops)
header = ["keyspace", "column_family"]
writer = csv.writer(output)
writer.writerow(header)
for keyspace in system.list_keyspaces():
ksinfo = system.get_keyspace_column_families(keyspace, use_dict_for_col_metadata=True)
attrs = [
"id",
"column_type",
"comment",
"comparator_type",
"default_validation_class",
"gc_grace_seconds",
"key_alias",
"key_cache_save_period_in_seconds",
"key_cache_size",
"key_validation_class",
"max_compaction_threshold",
"memtable_operations_in_millions",
"memtable_throughput_in_mb",
"memtable_flush_after_mins",
"merge_shards_chance",
"min_compaction_threshold",
"read_repair_chance",
"replicate_on_write",
"row_cache_provider",
"row_cache_save_period_in_seconds",
"row_cache_size",
"subcomparator_type",
]
for cfname, cfdef in ksinfo.iteritems():
ks = ("keyspace="+ keyspace)
cf = ("column_family="+ cfname)
row =(ks, cf)
writer.writerow(row)
return
except:
error(output, excinfo(), 2)
def main(argv):
usage = "Usage: dbdiscover.py"
args, kwargs = parse(argv)
host = kwargs.get('host', settings.DEFAULT_CASSANDRA_HOST)
port = kwargs.get('port', settings.DEFAULT_CASSANDRA_PORT)
try:
system = SystemManager("%s:%s" % (host, port))
ksprops = system.get_keyspace_properties("system").keys()
# header = ["keyspace"] + sorted(ksprops)
header = ["keyspace", "column_family"]
writer = csv.writer(output)
writer.writerow(header)
for keyspace in system.list_keyspaces():
ksinfo = system.get_keyspace_column_families(
keyspace, use_dict_for_col_metadata=True)
attrs = [
"id",
"column_type",
"comment",
"comparator_type",
"default_validation_class",
"gc_grace_seconds",
"key_alias",
"key_cache_save_period_in_seconds",
"key_cache_size",
"key_validation_class",
"max_compaction_threshold",
"memtable_operations_in_millions",
"memtable_throughput_in_mb",
"memtable_flush_after_mins",
"merge_shards_chance",
"min_compaction_threshold",
"read_repair_chance",
"replicate_on_write",
"row_cache_provider",
"row_cache_save_period_in_seconds",
"row_cache_size",
"subcomparator_type",
]
for cfname, cfdef in ksinfo.iteritems():
ks = ("keyspace=" + keyspace)
cf = ("column_family=" + cfname)
row = (ks, cf)
writer.writerow(row)
return
except:
error(output, excinfo(), 2)
def setup_databases(self, **kwargs):
settings.KEYSPACE = settings.KEYSPACE_TESTS
print "Patched value of KEYSPACE to '{0}'".format(settings.KEYSPACE)
if "dont_drop_daedalus_tests" not in os.environ:
sys_mgr = SystemManager()
try:
sys_mgr.drop_keyspace(settings.KEYSPACE)
print "Keyspace {0} droped OK".format(settings.KEYSPACE)
except:
pass
finally:
sys_mgr.close()
sys_mgr = SystemManager()
# keyspace 'settings.KEYSPACE' shouldn't exists
if settings.KEYSPACE in sys_mgr.list_keyspaces():
print "ERROR: keyspace {0} EXISTS after been droped".format(settings.KEYSPACE)
print " - Keyspaces:", sys_mgr.list_keyspaces()
assert False, "ERROR"
sys_mgr.close()
storage.get_service().create_keyspace_and_cfs()
return super(CassandraDjangoTestSuiteRunner, self).setup_databases(**kwargs)
def cassandra_reset():
from feedly import settings
hostname = settings.FEEDLY_CASSANDRA_HOSTS[0]
keyspace = "test_feedly"
sys = SystemManager(hostname)
# sys.drop_keyspace(keyspace)
if keyspace not in sys.list_keyspaces():
sys.create_keyspace(keyspace, SIMPLE_STRATEGY, {"replication_factor": "1"})
sys.create_column_family(keyspace, "activity", comparator_type=UTF8_TYPE)
sys.create_column_family(keyspace, "timeline", comparator_type=IntegerType(reversed=True))
def cassandra_reset():
from feedly import settings
hostname = settings.FEEDLY_CASSANDRA_HOSTS[0]
keyspace = 'test_feedly'
sys = SystemManager(hostname)
# sys.drop_keyspace(keyspace)
if keyspace not in sys.list_keyspaces():
sys.create_keyspace(keyspace, SIMPLE_STRATEGY,
{'replication_factor': '1'})
sys.create_column_family(keyspace,
'activity',
comparator_type=UTF8_TYPE)
sys.create_column_family(keyspace,
'timeline',
comparator_type=IntegerType(reversed=True))
def setup_package():
sys = SystemManager()
if TEST_KS in sys.list_keyspaces():
sys.drop_keyspace(TEST_KS)
try:
sys.create_keyspace(TEST_KS, 'SimpleStrategy', {'replication_factor': '1'})
sys.create_column_family(TEST_KS, 'Standard1')
sys.create_column_family(TEST_KS, 'Super1', super=True)
sys.create_column_family(TEST_KS, 'Indexed1')
sys.create_index(TEST_KS, 'Indexed1', 'birthdate', 'LongType')
sys.create_column_family(TEST_KS, 'Counter1',
default_validation_class='CounterColumnType')
sys.create_column_family(TEST_KS, 'SuperCounter1', super=True,
default_validation_class='CounterColumnType')
except Exception, e:
print e
try:
sys.drop_keyspace(TEST_KS)
except:
pass
raise e
def teardown_package():
sys = SystemManager()
if TEST_KS in sys.list_keyspaces():
sys.drop_keyspace(TEST_KS)
sys.close()
class CCli(Cmd, object):
prompt = '> '
continuation_prompt = '. '
timing = True
colors = True
debug = True
# O'Key, guys, you created good library with a lot of parameters, but,
# I think, you should test it before release, shouldn't?
case_insensitive = False
max_data_size = 35
max_rows = 50
def __init__(self, *args, **kwargs):
super(CCli, self).__init__(*args, **kwargs)
self.sm = None
self.pool = None
self.server = None
self.keyspace = None
self.settable['max_data_size'] = 'Maximum value symbols [0 = no truncating]'
self.settable['max_rows'] = 'Maximum rows to receive by one get'
def func_named(self, arg):
return super(CCli, self).func_named(arg[0])
@parse(Optional(server, default='locahost:9160'))
def do_connect(self, server):
try:
self.sm = SystemManager(server)
except TTransportException:
return self.perror('Can not connect to %s' % server)
self.server = server
self.prompt = '%s> ' % server
self.keyspace = None
self.pool = None
print 'Successfully connected to %s' % server
@check_connection
@parse(keyspace)
def do_use(self, keyspace):
if keyspace not in self.sm.list_keyspaces():
return self.perror('Unknown keyspace %s' % keyspace)
self.prompt = '%s/%s> ' % (self.server, keyspace)
self.keyspace = keyspace
self.pool = ConnectionPool(keyspace, server_list=[self.server])
print 'Using %s as default keyspace' % keyspace
def complete_use(self, text, line, begidx, endidx):
if not self.sm:
return []
return [x for x in self.sm.list_keyspaces() if x.startswith(text)]
@check_connection
@parse(Keyword('keyspaces') | Keyword('columnfamilies'))
def do_list(self, space):
return getattr(self, 'list_%s' % space)()
def list_keyspaces(self):
pt = PrettyTable()
pt.field_names = ['Keyspaces']
pt.align = 'l'
for ks in self.sm.list_keyspaces():
pt.add_row([ks])
print pt.get_string(sortby="Keyspaces")
@check_keyspace
def list_columnfamilies(self):
pt = PrettyTable()
pt.field_names = ['ColumnFamilies']
pt.align = 'l'
for cf in self.sm.get_keyspace_column_families(self.keyspace).keys():
pt.add_row([cf])
print pt.get_string(sortby="ColumnFamilies")
def complete_list(self, text, line, begidx, endidx):
return [x for x in ['keyspaces', 'columnfamilies'] if x.startswith(text)]
@check_connection
@parse(Keyword('keyspace') + Optional(keyspace, default=None) | Keyword('columnfamily') + columnfamily)
def do_describe(self, space, name):
return getattr(self, 'describe_%s' % space)(name)
def describe_keyspace(self, keyspace):
keyspace = keyspace or self.keyspace
if not keyspace:
return self.perror('Please, select working keyspace or define it as command parameter')
try:
options = self.sm.get_keyspace_properties(keyspace)
except NotFoundException:
return self.perror('Unknown keyspace %s' % keyspace)
pt = PrettyTable()
pt.field_names = ['Keyspace', keyspace]
pt.align["Keyspace"] = "l"
pt.align[keyspace] = 'r'
pt.add_row(['replication_strategy', options['replication_strategy']])
for k, v in options['strategy_options'].items():
pt.add_row([k, "\033[34m%s\033[0m" % v])
print pt.get_string(sortby='Keyspace')
@check_keyspace
def describe_columnfamily(self, columnfamily):
try:
options = self.sm.get_keyspace_column_families(self.keyspace, use_dict_for_col_metadata=True)[columnfamily]
except KeyError:
return self.perror('Unknown columnfamily %s' % columnfamily)
pt = PrettyTable()
pt.field_names = ['ColumnFamily', columnfamily]
pt.align["ColumnFamily"] = "l"
pt.align[columnfamily] = 'r'
for k, v in options.__dict__.items():
if k == 'column_metadata' and len(v):
continue
pt.add_row([k, v])
print pt.get_string(sortby='ColumnFamily')
if len(options.column_metadata):
pt = PrettyTable()
pt.field_names = ['Column \ Options'] + options.column_metadata.values()[0].__dict__.keys()
for k, v in options.column_metadata.items():
pt.add_row([k] + v.__dict__.values())
print pt.get_string(sortby='Column \ Options')
def complete_describe(self, text, line, begidx, endidx):
return [x for x in ['keyspace', 'columnfamily'] if x.startswith(text)]
def completenames(self, text, *ignored):
names = super(CCli, self).completenames(text, *ignored)
if self.keyspace:
names.extend(cf for cf in
self.sm.get_keyspace_column_families(self.keyspace).keys()
if cf.startswith(text))
return names
def default(self, line):
# Ugly, ugly things...
line = (' '.join(line.parsed)).strip()
if not self.server and not self.keyspace:
return super(CCli, self).default(line)
return self.simple_select(line)
@parse(columnfamily + Optional('[' + Combine(Optional(key, default='') +
Optional(':' + Optional(key, default='') + Optional(':' +
Optional(count, default='')))) + ']')
)
def simple_select(self, columnfamily, *args):
slice = ['', '', self.max_rows]
key = None
if args and args[1]:
if ':' not in args[1]:
key = args[1]
for i, part in enumerate(args[1].split(':', 2)):
slice[i] = part
try:
cf = ColumnFamily(self.pool, columnfamily)
except NotFoundException:
return super(CCli, self).default(' '.join([columnfamily] + list(args)))
if key:
pt = PrettyTable()
pt.field_names = ['Key', key]
pt.align["Key"] = "l"
pt.align[key] = 'r'
for k, v in cf.get(key).items():
pt.add_row([k, (v[:self.max_data_size - 3] + '...' if self.max_data_size and len(v) > self.max_data_size else v)])
print pt.get_string(sortby='Key')
return
data = dict(cf.get_range(start=slice[0], finish=slice[1], row_count=int(slice[2])))
columns = []
for key, row in data.items():
columns.extend(row.keys())
columns = list(set(columns))
columns.sort()
pt = PrettyTable()
pt.field_names = ['Key / Column'] + columns
pt.align["Key / Column"] = "l"
for column in columns:
pt.align[column] = "r"
for key, row in data.items():
prow = [key]
for column in columns:
value = row.get(column, '---')
if len(value) > self.max_data_size:
value = value[:self.max_data_size - 3] + '...'
prow.append(value)
pt.add_row(prow)
print pt.get_string(sortby='Key / Column')
def db_export(self):
db_contents = {'cassandra': {},
'zookeeper': {}}
cassandra_contents = db_contents['cassandra']
creds = None
if self._api_args.cassandra_user and self._api_args.cassandra_password:
creds = {'username': self._api_args.cassandra_user,
'password': self._api_args.cassandra_password}
socket_factory = default_socket_factory
if ('cassandra_use_ssl' in self._api_args and
self._api_args.cassandra_use_ssl):
socket_factory = self._make_ssl_socket_factory(
self._api_args.cassandra_ca_certs, validate=False)
sys_mgr = SystemManager(
self._api_args.cassandra_server_list[0],
credentials=creds,
socket_factory=socket_factory)
existing_keyspaces = sys_mgr.list_keyspaces()
for ks_name in set(KEYSPACES) - set(self._args.omit_keyspaces or []):
if self._api_args.cluster_id:
full_ks_name = '%s_%s' %(self._api_args.cluster_id, ks_name)
else:
full_ks_name = ks_name
if full_ks_name not in existing_keyspaces:
continue
cassandra_contents[ks_name] = {}
pool = ConnectionPool(
full_ks_name, self._api_args.cassandra_server_list,
pool_timeout=120, max_retries=-1, timeout=5,
socket_factory=socket_factory, credentials=creds)
for cf_name in sys_mgr.get_keyspace_column_families(full_ks_name):
cassandra_contents[ks_name][cf_name] = {}
cf = ColumnFamily(pool, cf_name,
buffer_size=self._args.buffer_size)
for r,c in cf.get_range(column_count=10000000, include_timestamp=True):
cassandra_contents[ks_name][cf_name][r] = c
logger.info("Cassandra DB dumped")
def get_nodes(path):
if path[:-1].rpartition('/')[-1] in self._zk_ignore_list:
return []
try:
if not zk.get_children(path):
return [(path, zk.get(path))]
except kazoo.exceptions.NoNodeError:
return []
nodes = []
for child in zk.get_children(path):
nodes.extend(get_nodes('%s%s/' %(path, child)))
return nodes
zk = kazoo.client.KazooClient(self._api_args.zk_server_ip)
zk.start()
nodes = get_nodes(self._api_args.cluster_id+'/')
zk.stop()
db_contents['zookeeper'] = json.dumps(nodes)
logger.info("Zookeeper DB dumped")
f = open(self._args.export_to, 'w')
try:
f.write(json.dumps(db_contents))
finally:
f.close()
logger.info("DB dump wrote to file %s" % self._args.export_to)
from pycassa.system_manager import SystemManager
from pycassa.system_manager import SIMPLE_STRATEGY
from pycassa.system_manager import UTF8_TYPE
from pycassa.system_manager import LONG_TYPE
from storage.cassandra_settings import HOSTS
from storage.cassandra_settings import KEYSPACE_NAME
sys = SystemManager(HOSTS[0])
if KEYSPACE_NAME in sys.list_keyspaces():
sys.drop_keyspace(KEYSPACE_NAME)
sys.create_keyspace(KEYSPACE_NAME, SIMPLE_STRATEGY, {'replication_factor': '1'})
sys.create_column_family(KEYSPACE_NAME, 'LoveActivity', comparator_type=UTF8_TYPE)
sys.create_column_family(KEYSPACE_NAME, 'Feed', comparator_type=UTF8_TYPE)
sys.create_column_family(KEYSPACE_NAME, 'AggregatedFeed', comparator_type=LONG_TYPE, super=True)
import json
from pycassa import ConnectionPool
from pycassa.columnfamily import ColumnFamily
from pycassa.system_manager import SystemManager
from pycassa.system_manager import SIMPLE_STRATEGY
from pycassa.system_manager import UTF8_TYPE
from pycassa.system_manager import ASCII_TYPE
import time
############################## Create Keyspace ################################
server_list = ['cassandra1:9160', 'cassandra2:9160', 'cassandra3:9160']
sys = SystemManager(server_list[0])
sys.list_keyspaces()
if 'entries' in sys.list_keyspaces():
sys.drop_keyspace('entries')
sys.create_keyspace('entries', SIMPLE_STRATEGY, {'replication_factor': '1'})
############################## Connection Pooling ############################
#pool = ConnectionPool('entries', server_list=server_list, pool_size=20)
pool = ConnectionPool('entries', server_list=server_list)
############################## Create Column Family ###########################
sys.create_column_family('entries', 'Author', comparator_type=UTF8_TYPE)
author_cf = ColumnFamily(pool, 'Author')
################################ INSERT #######################################
# Insert a row with a Column
author_cf.insert('sacharya', {'first_name': 'Sudarshan'})
# Insert a row with multiple columns
