def CreateCF(connStr, keySpace, cfs, columesTypeList = None):
sysmgt = SystemManager(connStr)
list = sysmgt.list_keyspaces()
if keySpace in list:
#print('pyagent is existed')
pass
else:
sysmgt.create_keyspace(keySpace, SIMPLE_STRATEGY, {'replication_factor': '3'})
cfDict = sysmgt.get_keyspace_column_families(keySpace)
allCF = cfDict.keys()
for i in range(len(cfs)):
columnFamily = cfs[i]
#print(columnFamily)
if columesTypeList != None:
columes = columesTypeList[i]
else:
columes = None
if columnFamily in allCF:
#print(columnFamily + ' is existed')
pass
else:
#print(columnFamily + ' is creating')
sysmgt.create_column_family(keySpace, columnFamily, super=False,
comparator_type=UTF8_TYPE,
key_validation_class=ASCII_TYPE,
default_validation_class=UTF8_TYPE,
column_validation_classes=columes,
gc_grace_seconds=1000)
sysmgt.close()
def ensure_cassandra_cf(self):
s = SystemManager()
if self.keyspace not in s.list_keyspaces():
s.create_keyspace(self.keyspace, SIMPLE_STRATEGY, {'replication_factor': '1'})
if self.cassandra_columns_family not in s.get_keyspace_column_families(self.keyspace):
s.create_column_family(self.keyspace, self.cassandra_columns_family)
self.columnfamily = ColumnFamily(self.cassandra_session, self.cassandra_columns_family)
def initializeTableLayout(keyspace, server_list, replicationStrategy,
strategyOptions, localDCName, credentials):
sys_manager = SystemManager(server_list[0], credentials=credentials)
# Make sure the the keyspace exists
if keyspace not in sys_manager.list_keyspaces():
sys_manager.create_keyspace(keyspace, replicationStrategy,
strategyOptions)
cf_defs = sys_manager.get_keyspace_column_families(keyspace)
# Create UTF8 CF's
for tablename in ["global_nodes", "metadata"]:
if tablename not in cf_defs.keys():
createUTF8ColumnFamily(sys_manager, keyspace, tablename)
if localDCName:
dcNodes = "dc_%s_nodes" % (localDCName,)
if dcNodes not in cf_defs.keys():
createUTF8ColumnFamily(sys_manager, keyspace, dcNodes)
else:
# TODO Log we do not have the DC name
pass
if "node_slices" not in cf_defs.keys():
sys_manager.create_column_family(
keyspace,
"node_slices",
super=False,
comparator_type=pycassa_types.LongType(),
key_validation_class=pycassa_types.UTF8Type(),
default_validation_class=pycassa_types.LongType()
)
def db_export(self):
db_contents = {'cassandra': {}, 'zookeeper': {}}
cassandra_contents = db_contents['cassandra']
for ks_name in (set(KEYSPACES) - set(self._args.omit_keyspaces or [])):
if self._api_args.cluster_id:
full_ks_name = '%s_%s' % (self._api_args.cluster_id, ks_name)
else:
full_ks_name = ks_name
cassandra_contents[ks_name] = {}
socket_factory = pycassa.connection.default_socket_factory
if self._api_args.cassandra_use_ssl:
socket_factory = pycassa.connection.make_ssl_socket_factory(
self._api_args.cassandra_ca_certs, validate=False)
pool = pycassa.ConnectionPool(full_ks_name,
self._api_args.cassandra_server_list,
pool_timeout=120,
max_retries=-1,
timeout=5,
socket_factory=socket_factory)
creds = None
if (self._api_args.cassandra_user
and self._api_args.cassandra_password):
creds = {
'username': self._api_args.cassandra_user,
'password': self._api_args.cassandra_password
}
sys_mgr = SystemManager(self._api_args.cassandra_server_list[0],
credentials=creds)
for cf_name in sys_mgr.get_keyspace_column_families(full_ks_name):
cassandra_contents[ks_name][cf_name] = {}
cf = pycassa.ColumnFamily(pool, cf_name)
for r, c in cf.get_range(column_count=10000000,
include_timestamp=True):
cassandra_contents[ks_name][cf_name][r] = c
def get_nodes(path):
if not zk.get_children(path):
return [(path, zk.get(path))]
nodes = []
for child in zk.get_children(path):
nodes.extend(get_nodes('%s%s/' % (path, child)))
return nodes
zk = kazoo.client.KazooClient(self._api_args.zk_server_ip)
zk.start()
nodes = get_nodes(self._api_args.cluster_id + '/')
zk.stop()
db_contents['zookeeper'] = json.dumps(nodes)
f = open(self._args.export_to, 'w')
try:
f.write(json.dumps(db_contents))
finally:
f.close()
def db_export(self):
db_contents = {'cassandra': {},
'zookeeper': {}}
cassandra_contents = db_contents['cassandra']
for ks_name in (set(KEYSPACES) -
set(self._args.omit_keyspaces or [])):
if self._api_args.cluster_id:
full_ks_name = '%s_%s' %(self._api_args.cluster_id, ks_name)
else:
full_ks_name = ks_name
cassandra_contents[ks_name] = {}
socket_factory = pycassa.connection.default_socket_factory
if self._api_args.cassandra_use_ssl:
socket_factory = pycassa.connection.make_ssl_socket_factory(
self._api_args.cassandra_ca_certs, validate=False)
pool = pycassa.ConnectionPool(
full_ks_name, self._api_args.cassandra_server_list,
pool_timeout=120, max_retries=-1, timeout=5,
socket_factory=socket_factory)
creds = None
if (self._api_args.cassandra_user and
self._api_args.cassandra_password):
creds = {'username': self._api_args.cassandra_user,
'password': self._api_args.cassandra_password}
sys_mgr = SystemManager(self._api_args.cassandra_server_list[0],
credentials=creds)
for cf_name in sys_mgr.get_keyspace_column_families(full_ks_name):
cassandra_contents[ks_name][cf_name] = {}
cf = pycassa.ColumnFamily(pool, cf_name)
for r,c in cf.get_range(column_count=10000000, include_timestamp=True):
cassandra_contents[ks_name][cf_name][r] = c
def get_nodes(path):
if not zk.get_children(path):
return [(path, zk.get(path))]
nodes = []
for child in zk.get_children(path):
nodes.extend(get_nodes('%s%s/' %(path, child)))
return nodes
zk = kazoo.client.KazooClient(self._api_args.zk_server_ip)
zk.start()
nodes = get_nodes(self._api_args.cluster_id+'/')
zk.stop()
db_contents['zookeeper'] = json.dumps(nodes)
f = open(self._args.export_to, 'w')
try:
f.write(json.dumps(db_contents))
finally:
f.close()
def ensure_cassandra_cf(self):
s = SystemManager()
if self.keyspace not in s.list_keyspaces():
s.create_keyspace(self.keyspace, SIMPLE_STRATEGY,
{'replication_factor': '1'})
if self.cassandra_columns_family not in s.get_keyspace_column_families(
self.keyspace):
s.create_column_family(self.keyspace,
self.cassandra_columns_family)
self.columnfamily = ColumnFamily(self.cassandra_session,
self.cassandra_columns_family)
def main(argv):
usage = "Usage: dbdiscover.py"
args, kwargs = parse(argv)
host = kwargs.get('host', settings.DEFAULT_CASSANDRA_HOST)
port = kwargs.get('port', settings.DEFAULT_CASSANDRA_PORT)
try:
system = SystemManager("%s:%s" % (host, port))
ksprops = system.get_keyspace_properties("system").keys()
# header = ["keyspace"] + sorted(ksprops)
header = ["keyspace", "column_family"]
writer = csv.writer(output)
writer.writerow(header)
for keyspace in system.list_keyspaces():
ksinfo = system.get_keyspace_column_families(keyspace, use_dict_for_col_metadata=True)
attrs = [
"id",
"column_type",
"comment",
"comparator_type",
"default_validation_class",
"gc_grace_seconds",
"key_alias",
"key_cache_save_period_in_seconds",
"key_cache_size",
"key_validation_class",
"max_compaction_threshold",
"memtable_operations_in_millions",
"memtable_throughput_in_mb",
"memtable_flush_after_mins",
"merge_shards_chance",
"min_compaction_threshold",
"read_repair_chance",
"replicate_on_write",
"row_cache_provider",
"row_cache_save_period_in_seconds",
"row_cache_size",
"subcomparator_type",
]
for cfname, cfdef in ksinfo.iteritems():
ks = ("keyspace="+ keyspace)
cf = ("column_family="+ cfname)
row =(ks, cf)
writer.writerow(row)
return
except:
error(output, excinfo(), 2)
def main(argv):
usage = "Usage: dbdiscover.py"
args, kwargs = parse(argv)
host = kwargs.get('host', settings.DEFAULT_CASSANDRA_HOST)
port = kwargs.get('port', settings.DEFAULT_CASSANDRA_PORT)
try:
system = SystemManager("%s:%s" % (host, port))
ksprops = system.get_keyspace_properties("system").keys()
# header = ["keyspace"] + sorted(ksprops)
header = ["keyspace", "column_family"]
writer = csv.writer(output)
writer.writerow(header)
for keyspace in system.list_keyspaces():
ksinfo = system.get_keyspace_column_families(
keyspace, use_dict_for_col_metadata=True)
attrs = [
"id",
"column_type",
"comment",
"comparator_type",
"default_validation_class",
"gc_grace_seconds",
"key_alias",
"key_cache_save_period_in_seconds",
"key_cache_size",
"key_validation_class",
"max_compaction_threshold",
"memtable_operations_in_millions",
"memtable_throughput_in_mb",
"memtable_flush_after_mins",
"merge_shards_chance",
"min_compaction_threshold",
"read_repair_chance",
"replicate_on_write",
"row_cache_provider",
"row_cache_save_period_in_seconds",
"row_cache_size",
"subcomparator_type",
]
for cfname, cfdef in ksinfo.iteritems():
ks = ("keyspace=" + keyspace)
cf = ("column_family=" + cfname)
row = (ks, cf)
writer.writerow(row)
return
except:
error(output, excinfo(), 2)
def handle_noargs(self, **options):
sys = SystemManager(server=settings.CASSANDRA_SERVERS[0])
REPLICATION_STRATEGY = getattr(pycassa.system_manager, settings.CASSANDRA_REPLICATION_STRATEGY)
existing_cfs = sys.get_keyspace_column_families(settings.CASSANDRA_KEYSPACE).keys()
if 'APIConsumers' not in existing_cfs:
print 'Creating missing column family: APIConsumers'
sys.create_column_family(settings.CASSANDRA_KEYSPACE, 'APIConsumers', comparator_type=UTF8_TYPE)
sys.alter_column(settings.CASSANDRA_KEYSPACE, 'APIConsumers', 'consumer_key', UTF8_TYPE)
sys.alter_column(settings.CASSANDRA_KEYSPACE, 'APIConsumers', 'consumer_secret', UTF8_TYPE)
sys.alter_column(settings.CASSANDRA_KEYSPACE, 'APIConsumers', 'username', UTF8_TYPE)
print 'All done!'
class CCli(Cmd, object):
prompt = '> '
continuation_prompt = '. '
timing = True
colors = True
debug = True
# O'Key, guys, you created good library with a lot of parameters, but,
# I think, you should test it before release, shouldn't?
case_insensitive = False
max_data_size = 35
max_rows = 50
def __init__(self, *args, **kwargs):
super(CCli, self).__init__(*args, **kwargs)
self.sm = None
self.pool = None
self.server = None
self.keyspace = None
self.settable['max_data_size'] = 'Maximum value symbols [0 = no truncating]'
self.settable['max_rows'] = 'Maximum rows to receive by one get'
def func_named(self, arg):
return super(CCli, self).func_named(arg[0])
@parse(Optional(server, default='locahost:9160'))
def do_connect(self, server):
try:
self.sm = SystemManager(server)
except TTransportException:
return self.perror('Can not connect to %s' % server)
self.server = server
self.prompt = '%s> ' % server
self.keyspace = None
self.pool = None
print 'Successfully connected to %s' % server
@check_connection
@parse(keyspace)
def do_use(self, keyspace):
if keyspace not in self.sm.list_keyspaces():
return self.perror('Unknown keyspace %s' % keyspace)
self.prompt = '%s/%s> ' % (self.server, keyspace)
self.keyspace = keyspace
self.pool = ConnectionPool(keyspace, server_list=[self.server])
print 'Using %s as default keyspace' % keyspace
def complete_use(self, text, line, begidx, endidx):
if not self.sm:
return []
return [x for x in self.sm.list_keyspaces() if x.startswith(text)]
@check_connection
@parse(Keyword('keyspaces') | Keyword('columnfamilies'))
def do_list(self, space):
return getattr(self, 'list_%s' % space)()
def list_keyspaces(self):
pt = PrettyTable()
pt.field_names = ['Keyspaces']
pt.align = 'l'
for ks in self.sm.list_keyspaces():
pt.add_row([ks])
print pt.get_string(sortby="Keyspaces")
@check_keyspace
def list_columnfamilies(self):
pt = PrettyTable()
pt.field_names = ['ColumnFamilies']
pt.align = 'l'
for cf in self.sm.get_keyspace_column_families(self.keyspace).keys():
pt.add_row([cf])
print pt.get_string(sortby="ColumnFamilies")
def complete_list(self, text, line, begidx, endidx):
return [x for x in ['keyspaces', 'columnfamilies'] if x.startswith(text)]
@check_connection
@parse(Keyword('keyspace') + Optional(keyspace, default=None) | Keyword('columnfamily') + columnfamily)
def do_describe(self, space, name):
return getattr(self, 'describe_%s' % space)(name)
def describe_keyspace(self, keyspace):
keyspace = keyspace or self.keyspace
if not keyspace:
return self.perror('Please, select working keyspace or define it as command parameter')
try:
options = self.sm.get_keyspace_properties(keyspace)
except NotFoundException:
return self.perror('Unknown keyspace %s' % keyspace)
pt = PrettyTable()
pt.field_names = ['Keyspace', keyspace]
pt.align["Keyspace"] = "l"
pt.align[keyspace] = 'r'
pt.add_row(['replication_strategy', options['replication_strategy']])
for k, v in options['strategy_options'].items():
pt.add_row([k, "\033[34m%s\033[0m" % v])
print pt.get_string(sortby='Keyspace')
@check_keyspace
def describe_columnfamily(self, columnfamily):
try:
options = self.sm.get_keyspace_column_families(self.keyspace, use_dict_for_col_metadata=True)[columnfamily]
except KeyError:
return self.perror('Unknown columnfamily %s' % columnfamily)
pt = PrettyTable()
pt.field_names = ['ColumnFamily', columnfamily]
pt.align["ColumnFamily"] = "l"
pt.align[columnfamily] = 'r'
for k, v in options.__dict__.items():
if k == 'column_metadata' and len(v):
continue
pt.add_row([k, v])
print pt.get_string(sortby='ColumnFamily')
if len(options.column_metadata):
pt = PrettyTable()
pt.field_names = ['Column \ Options'] + options.column_metadata.values()[0].__dict__.keys()
for k, v in options.column_metadata.items():
pt.add_row([k] + v.__dict__.values())
print pt.get_string(sortby='Column \ Options')
def complete_describe(self, text, line, begidx, endidx):
return [x for x in ['keyspace', 'columnfamily'] if x.startswith(text)]
def completenames(self, text, *ignored):
names = super(CCli, self).completenames(text, *ignored)
if self.keyspace:
names.extend(cf for cf in
self.sm.get_keyspace_column_families(self.keyspace).keys()
if cf.startswith(text))
return names
def default(self, line):
# Ugly, ugly things...
line = (' '.join(line.parsed)).strip()
if not self.server and not self.keyspace:
return super(CCli, self).default(line)
return self.simple_select(line)
@parse(columnfamily + Optional('[' + Combine(Optional(key, default='') +
Optional(':' + Optional(key, default='') + Optional(':' +
Optional(count, default='')))) + ']')
)
def simple_select(self, columnfamily, *args):
slice = ['', '', self.max_rows]
key = None
if args and args[1]:
if ':' not in args[1]:
key = args[1]
for i, part in enumerate(args[1].split(':', 2)):
slice[i] = part
try:
cf = ColumnFamily(self.pool, columnfamily)
except NotFoundException:
return super(CCli, self).default(' '.join([columnfamily] + list(args)))
if key:
pt = PrettyTable()
pt.field_names = ['Key', key]
pt.align["Key"] = "l"
pt.align[key] = 'r'
for k, v in cf.get(key).items():
pt.add_row([k, (v[:self.max_data_size - 3] + '...' if self.max_data_size and len(v) > self.max_data_size else v)])
print pt.get_string(sortby='Key')
return
data = dict(cf.get_range(start=slice[0], finish=slice[1], row_count=int(slice[2])))
columns = []
for key, row in data.items():
columns.extend(row.keys())
columns = list(set(columns))
columns.sort()
pt = PrettyTable()
pt.field_names = ['Key / Column'] + columns
pt.align["Key / Column"] = "l"
for column in columns:
pt.align[column] = "r"
for key, row in data.items():
prow = [key]
for column in columns:
value = row.get(column, '---')
if len(value) > self.max_data_size:
value = value[:self.max_data_size - 3] + '...'
prow.append(value)
pt.add_row(prow)
print pt.get_string(sortby='Key / Column')
def db_export(self):
db_contents = {'cassandra': {},
'zookeeper': {}}
cassandra_contents = db_contents['cassandra']
creds = None
if self._api_args.cassandra_user and self._api_args.cassandra_password:
creds = {'username': self._api_args.cassandra_user,
'password': self._api_args.cassandra_password}
socket_factory = default_socket_factory
if ('cassandra_use_ssl' in self._api_args and
self._api_args.cassandra_use_ssl):
socket_factory = self._make_ssl_socket_factory(
self._api_args.cassandra_ca_certs, validate=False)
sys_mgr = SystemManager(
self._api_args.cassandra_server_list[0],
credentials=creds,
socket_factory=socket_factory)
existing_keyspaces = sys_mgr.list_keyspaces()
for ks_name in set(KEYSPACES) - set(self._args.omit_keyspaces or []):
if self._api_args.cluster_id:
full_ks_name = '%s_%s' %(self._api_args.cluster_id, ks_name)
else:
full_ks_name = ks_name
if full_ks_name not in existing_keyspaces:
continue
cassandra_contents[ks_name] = {}
pool = ConnectionPool(
full_ks_name, self._api_args.cassandra_server_list,
pool_timeout=120, max_retries=-1, timeout=5,
socket_factory=socket_factory, credentials=creds)
for cf_name in sys_mgr.get_keyspace_column_families(full_ks_name):
cassandra_contents[ks_name][cf_name] = {}
cf = ColumnFamily(pool, cf_name,
buffer_size=self._args.buffer_size)
for r,c in cf.get_range(column_count=10000000, include_timestamp=True):
cassandra_contents[ks_name][cf_name][r] = c
logger.info("Cassandra DB dumped")
def get_nodes(path):
if path[:-1].rpartition('/')[-1] in self._zk_ignore_list:
return []
try:
if not zk.get_children(path):
return [(path, zk.get(path))]
except kazoo.exceptions.NoNodeError:
return []
nodes = []
for child in zk.get_children(path):
nodes.extend(get_nodes('%s%s/' %(path, child)))
return nodes
zk = kazoo.client.KazooClient(self._api_args.zk_server_ip)
zk.start()
nodes = get_nodes(self._api_args.cluster_id+'/')
zk.stop()
db_contents['zookeeper'] = json.dumps(nodes)
logger.info("Zookeeper DB dumped")
f = open(self._args.export_to, 'w')
try:
f.write(json.dumps(db_contents))
finally:
f.close()
logger.info("DB dump wrote to file %s" % self._args.export_to)
def db_export(self):
db_contents = {'cassandra': {}, 'zookeeper': {}}
cassandra_contents = db_contents['cassandra']
for ks_name in [
'config_db_uuid',
'useragent',
'to_bgp_keyspace',
'svc_monitor_keyspace',
'DISCOVERY_SERVER',
]:
cassandra_contents[ks_name] = {}
if ks_name == 'DISCOVERY_SERVER':
# stringify key as composite column is used
stringify_col_name = True
else:
stringify_col_name = False
pool = pycassa.ConnectionPool(
ks_name, [self._api_args.cassandra_server_list],
pool_timeout=120,
max_retries=-1,
timeout=5)
sys_mgr = SystemManager(self._api_args.cassandra_server_list[0],
credentials={
'username':
self._api_args.cassandra_user,
'password':
self._api_args.cassandra_password
})
for cf_name in sys_mgr.get_keyspace_column_families(ks_name):
cassandra_contents[ks_name][cf_name] = {}
cf = pycassa.ColumnFamily(pool, cf_name)
for r, c in cf.get_range(column_count=10000000,
include_timestamp=True):
if stringify_col_name:
cassandra_contents[ks_name][cf_name][r] = dict(
(str(k), v) for k, v in c.items())
else:
cassandra_contents[ks_name][cf_name][r] = c
def get_nodes(path):
if not zk.get_children(path):
return [(path, zk.get(path))]
nodes = []
for child in zk.get_children(path):
nodes.extend(get_nodes('%s%s/' % (path, child)))
return nodes
zk = kazoo.client.KazooClient(self._api_args.zk_server_ip)
zk.start()
nodes = get_nodes('/')
zk.stop()
db_contents['zookeeper'] = json.dumps(nodes)
f = open(self._args.export_to, 'w')
try:
f.write(json.dumps(db_contents))
finally:
f.close()
def create():
keyspace = config.cassandra_keyspace
creds = {'username': config.cassandra_username,
'password': config.cassandra_password}
mgr = SystemManager(config.cassandra_hosts[0], credentials=creds)
cfs = mgr.get_keyspace_column_families(keyspace).keys()
try:
if 'Indexes' not in cfs:
workaround_1779(mgr.create_column_family, keyspace, 'Indexes',
comparator_type=UTF8_TYPE)
if 'Stacktrace' not in cfs:
workaround_1779(mgr.create_column_family, keyspace, 'Stacktrace',
comparator_type=UTF8_TYPE,
default_validation_class=UTF8_TYPE)
if 'AwaitingRetrace' not in cfs:
workaround_1779(mgr.create_column_family, keyspace, 'AwaitingRetrace',
key_validation_class=UTF8_TYPE,
comparator_type=UTF8_TYPE,
default_validation_class=UTF8_TYPE)
if 'RetraceStats' not in cfs:
workaround_1779(mgr.create_column_family, keyspace, 'RetraceStats',
comparator_type=UTF8_TYPE,
default_validation_class=CounterColumnType())
if 'UniqueUsers90Days' not in cfs:
workaround_1779(mgr.create_column_family, keyspace, 'UniqueUsers90Days',
comparator_type=UTF8_TYPE,
key_validation_class=UTF8_TYPE,
default_validation_class=LONG_TYPE)
if 'BadRequest' not in cfs:
workaround_1779(mgr.create_column_family, keyspace, 'BadRequest',
default_validation_class=CounterColumnType())
if 'UserBinaryPackages' not in cfs:
workaround_1779(mgr.create_column_family, keyspace, 'UserBinaryPackages',
# The key_validation_class is a launchpad team ID, which is
# always ascii.
# The comparator is a binary package name, which is always
# ascii according to Debian policy.
# default_validation_class is bytes as it's always NULL.
key_validation_class=ASCII_TYPE,
comparator_type=ASCII_TYPE)
if 'BugToCrashSignatures' not in cfs:
workaround_1779(mgr.create_column_family, keyspace, 'BugToCrashSignatures',
key_validation_class=INT_TYPE,
comparator_type=UTF8_TYPE)
if 'CouldNotBucket' not in cfs:
workaround_1779(mgr.create_column_family, keyspace, 'CouldNotBucket',
comparator_type=TIME_UUID_TYPE)
if 'TimeToRetrace' not in cfs:
workaround_1779(mgr.create_column_family, keyspace, 'TimeToRetrace',
default_validation_class=FLOAT_TYPE)
if 'UniqueSystemsForErrorsByRelease' not in cfs:
workaround_1779(mgr.create_column_family, keyspace,
'UniqueSystemsForErrorsByRelease',
comparator_type=DateType(),
default_validation_class=LONG_TYPE)
if 'SystemImages' not in cfs:
workaround_1779(mgr.create_column_family, keyspace, 'SystemImages',
key_validation_class=UTF8_TYPE,
comparator_type=UTF8_TYPE)
finally:
mgr.close()
