def setup_databases(self, **kwargs):
settings.KEYSPACE = settings.KEYSPACE_TESTS
print "Patched value of KEYSPACE to '{0}'".format(settings.KEYSPACE)
if "dont_drop_daedalus_tests" not in os.environ:
sys_mgr = SystemManager()
try:
sys_mgr.drop_keyspace(settings.KEYSPACE)
print "Keyspace {0} droped OK".format(settings.KEYSPACE)
except:
pass
finally:
sys_mgr.close()
sys_mgr = SystemManager()
# keyspace 'settings.KEYSPACE' shouldn't exists
if settings.KEYSPACE in sys_mgr.list_keyspaces():
print "ERROR: keyspace {0} EXISTS after been droped".format(
settings.KEYSPACE)
print " - Keyspaces:", sys_mgr.list_keyspaces()
assert False, "ERROR"
sys_mgr.close()
storage.get_service().create_keyspace_and_cfs()
return super(CassandraDjangoTestSuiteRunner,
self).setup_databases(**kwargs)
def _sys_mgr():
'''
Return a pycassa system manager connection object
'''
thrift_port = str(__salt__['config.option']('cassandra.THRIFT_PORT'))
host = __salt__['config.option']('cassandra.host')
return SystemManager('{0}:{1}'.format(host, thrift_port))
def tearDown(self):
if self.store._store is not None:
self.store.clear()
self.store.close()
from pycassa.system_manager import SystemManager # @UnresolvedImport @IgnorePep8
system_manager = SystemManager('localhost:9160')
system_manager.drop_column_family('Murk', 'shove')
def main(argv):
usage = "Usage: dbschema [options] [{resource}]"
args, kwargs = parse(argv)
host = kwargs.get('host', settings.DEFAULT_CASSANDRA_HOST)
port = kwargs.get('port', settings.DEFAULT_CASSANDRA_PORT)
try:
system = SystemManager("%s:%s" % (host, port))
# FOR TESTING ONLY
# print str(argv)
if len(argv) == 0:
list_keyspaces(system)
return
if len(argv) == 1:
ksname = argv[0]
list_keyspace(system, ksname)
if len(argv) == 2:
ksname = argv[0]
cfname = argv[1]
if cfname != 'ANY':
list_column_family(system, ksname, cfname)
else:
list_keyspace(system, ksname)
except:
error(output, excinfo(), 2)
def _sys_mgr():
"""
Return a pycassa system manager connection object
"""
thrift_port = str(__salt__["config.option"]("cassandra.THRIFT_PORT"))
host = __salt__["config.option"]("cassandra.host")
return SystemManager("{}:{}".format(host, thrift_port))
def initializeTableLayout(keyspace, server_list, replicationStrategy,
strategyOptions, localDCName, credentials):
sys_manager = SystemManager(server_list[0], credentials=credentials)
# Make sure the the keyspace exists
if keyspace not in sys_manager.list_keyspaces():
sys_manager.create_keyspace(keyspace, replicationStrategy,
strategyOptions)
cf_defs = sys_manager.get_keyspace_column_families(keyspace)
# Create UTF8 CF's
for tablename in ["global_nodes", "metadata"]:
if tablename not in cf_defs.keys():
createUTF8ColumnFamily(sys_manager, keyspace, tablename)
if localDCName:
dcNodes = "dc_%s_nodes" % (localDCName,)
if dcNodes not in cf_defs.keys():
createUTF8ColumnFamily(sys_manager, keyspace, dcNodes)
else:
# TODO Log we do not have the DC name
pass
if "node_slices" not in cf_defs.keys():
sys_manager.create_column_family(
keyspace,
"node_slices",
super=False,
comparator_type=pycassa_types.LongType(),
key_validation_class=pycassa_types.UTF8Type(),
default_validation_class=pycassa_types.LongType()
)
def create_cfs(self):
"""
Creates the Cassandra Column Families (if not exist)
"""
sys_mgr = None
pool = None
try:
sys_mgr = SystemManager()
pool = ConnectionPool(settings.KEYSPACE,
server_list=settings.CASSANDRA_HOSTS)
for cf_name in [
CF_LOGS, CF_LOGS_BY_APP, CF_LOGS_BY_HOST,
CF_LOGS_BY_SEVERITY
]:
try:
cf = ColumnFamily(pool, cf_name)
except:
logger.info("create_cfs(): Creating column family %s",
cf_name)
sys_mgr.create_column_family(
settings.KEYSPACE,
cf_name,
comparator_type=TimeUUIDType())
cf = ColumnFamily(pool, cf_name)
cf.get_count(str(uuid.uuid4()))
finally:
if pool:
pool.dispose()
if sys_mgr:
sys_mgr.close()
def _cassandra_ensure_keyspace(self, server_list, keyspace_name,
cf_info_list):
# Retry till cassandra is up
server_idx = 0
num_dbnodes = len(self._server_list)
connected = False
while not connected:
try:
cass_server = self._server_list[server_idx]
sys_mgr = SystemManager(cass_server)
connected = True
except Exception as e:
# TODO do only for
# thrift.transport.TTransport.TTransportException
server_idx = (server_idx + 1) % num_dbnodes
time.sleep(3)
if self._reset_config:
try:
sys_mgr.drop_keyspace(keyspace_name)
except pycassa.cassandra.ttypes.InvalidRequestException as e:
# TODO verify only EEXISTS
self._logger("Warning! " + str(e), level=SandeshLevel.SYS_WARN)
try:
sys_mgr.create_keyspace(keyspace_name, SIMPLE_STRATEGY,
{'replication_factor': str(num_dbnodes)})
except pycassa.cassandra.ttypes.InvalidRequestException as e:
# TODO verify only EEXISTS
self._logger("Warning! " + str(e), level=SandeshLevel.SYS_WARN)
gc_grace_sec = 0
if num_dbnodes > 1:
gc_grace_sec = 60
for cf_info in cf_info_list:
try:
(cf_name, comparator_type) = cf_info
if comparator_type:
sys_mgr.create_column_family(
keyspace_name,
cf_name,
comparator_type=comparator_type,
gc_grace_seconds=gc_grace_sec,
default_validation_class='UTF8Type')
else:
sys_mgr.create_column_family(
keyspace_name,
cf_name,
gc_grace_seconds=gc_grace_sec,
default_validation_class='UTF8Type')
except pycassa.cassandra.ttypes.InvalidRequestException as e:
# TODO verify only EEXISTS
self._logger("Warning! " + str(e), level=SandeshLevel.SYS_WARN)
sys_mgr.alter_column_family(
keyspace_name,
cf_name,
gc_grace_seconds=gc_grace_sec,
default_validation_class='UTF8Type')
def db_export(self):
db_contents = {'cassandra': {}, 'zookeeper': {}}
cassandra_contents = db_contents['cassandra']
for ks_name in (set(KEYSPACES) - set(self._args.omit_keyspaces or [])):
if self._api_args.cluster_id:
full_ks_name = '%s_%s' % (self._api_args.cluster_id, ks_name)
else:
full_ks_name = ks_name
cassandra_contents[ks_name] = {}
socket_factory = pycassa.connection.default_socket_factory
if self._api_args.cassandra_use_ssl:
socket_factory = pycassa.connection.make_ssl_socket_factory(
self._api_args.cassandra_ca_certs, validate=False)
pool = pycassa.ConnectionPool(full_ks_name,
self._api_args.cassandra_server_list,
pool_timeout=120,
max_retries=-1,
timeout=5,
socket_factory=socket_factory)
creds = None
if (self._api_args.cassandra_user
and self._api_args.cassandra_password):
creds = {
'username': self._api_args.cassandra_user,
'password': self._api_args.cassandra_password
}
sys_mgr = SystemManager(self._api_args.cassandra_server_list[0],
credentials=creds)
for cf_name in sys_mgr.get_keyspace_column_families(full_ks_name):
cassandra_contents[ks_name][cf_name] = {}
cf = pycassa.ColumnFamily(pool, cf_name)
for r, c in cf.get_range(column_count=10000000,
include_timestamp=True):
cassandra_contents[ks_name][cf_name][r] = c
def get_nodes(path):
if not zk.get_children(path):
return [(path, zk.get(path))]
nodes = []
for child in zk.get_children(path):
nodes.extend(get_nodes('%s%s/' % (path, child)))
return nodes
zk = kazoo.client.KazooClient(self._api_args.zk_server_ip)
zk.start()
nodes = get_nodes(self._api_args.cluster_id + '/')
zk.stop()
db_contents['zookeeper'] = json.dumps(nodes)
f = open(self._args.export_to, 'w')
try:
f.write(json.dumps(db_contents))
finally:
f.close()
def setUp(self):
from shove import Shove
from pycassa.system_manager import SystemManager # @UnresolvedImport @IgnorePep8
system_manager = SystemManager('localhost:9160')
try:
system_manager.create_column_family('Murk', 'shove')
except:
pass
self.store = Shove('cassandra://localhost:9160/Murk/shove')
def setup_module():
global pool, cf, scf, counter_cf, super_counter_cf, sysman
credentials = {'username': '******', 'password': '******'}
pool = ConnectionPool(keyspace='PycassaTestKeyspace', credentials=credentials)
cf = ColumnFamily(pool, 'Standard1')
scf = ColumnFamily(pool, 'Super1')
sysman = SystemManager()
counter_cf = ColumnFamily(pool, 'Counter1')
super_counter_cf = ColumnFamily(pool, 'SuperCounter1')
def setUp(self):
from shove import Shove
from pycassa.system_manager import SystemManager
system_manager = SystemManager('localhost:9160')
try:
system_manager.create_column_family('Foo', 'shove')
except:
pass
self.store = Shove('cassandra://localhost:9160/Foo/shove')
def ensure_cassandra_cf(self):
s = SystemManager()
if self.keyspace not in s.list_keyspaces():
s.create_keyspace(self.keyspace, SIMPLE_STRATEGY,
{'replication_factor': '1'})
if self.cassandra_columns_family not in s.get_keyspace_column_families(
self.keyspace):
s.create_column_family(self.keyspace,
self.cassandra_columns_family)
self.columnfamily = ColumnFamily(self.cassandra_session,
self.cassandra_columns_family)
def main(argv):
usage = "Usage: dbdiscover.py"
args, kwargs = parse(argv)
host = kwargs.get('host', settings.DEFAULT_CASSANDRA_HOST)
port = kwargs.get('port', settings.DEFAULT_CASSANDRA_PORT)
try:
system = SystemManager("%s:%s" % (host, port))
ksprops = system.get_keyspace_properties("system").keys()
# header = ["keyspace"] + sorted(ksprops)
header = ["keyspace", "column_family"]
writer = csv.writer(output)
writer.writerow(header)
for keyspace in system.list_keyspaces():
ksinfo = system.get_keyspace_column_families(
keyspace, use_dict_for_col_metadata=True)
attrs = [
"id",
"column_type",
"comment",
"comparator_type",
"default_validation_class",
"gc_grace_seconds",
"key_alias",
"key_cache_save_period_in_seconds",
"key_cache_size",
"key_validation_class",
"max_compaction_threshold",
"memtable_operations_in_millions",
"memtable_throughput_in_mb",
"memtable_flush_after_mins",
"merge_shards_chance",
"min_compaction_threshold",
"read_repair_chance",
"replicate_on_write",
"row_cache_provider",
"row_cache_save_period_in_seconds",
"row_cache_size",
"subcomparator_type",
]
for cfname, cfdef in ksinfo.iteritems():
ks = ("keyspace=" + keyspace)
cf = ("column_family=" + cfname)
row = (ks, cf)
writer.writerow(row)
return
except:
error(output, excinfo(), 2)
def setUp(self):
self.sysman = SystemManager()
self.sysman.create_keyspace(
TEST_KS, SIMPLE_STRATEGY,
dict(replication_factor='1')
)
self.sysman.create_column_family(TEST_KS, TEST_CF)
self.pool = ConnectionPool(TEST_KS)
self.cf = ColumnFamily(self.pool, TEST_CF)
def do_connect(self, server):
try:
self.sm = SystemManager(server)
except TTransportException:
return self.perror('Can not connect to %s' % server)
self.server = server
self.prompt = '%s> ' % server
self.keyspace = None
self.pool = None
print 'Successfully connected to %s' % server
def __init__(self, engine, **kw):
super(CassandraStore, self).__init__(engine, **kw)
spliturl = urlparse.urlsplit(engine)
_, keyspace, column_family = spliturl[2].split('/')
try:
self._pool = pycassa.connect(keyspace, [spliturl[1]])
self._store = pycassa.ColumnFamily(self._pool, column_family)
except pycassa.InvalidRequestException:
from pycassa.system_manager import SystemManager
system_manager = SystemManager(spliturl[1])
system_manager.create_keyspace(keyspace, kw.get('replication', 1))
system_manager.create_column_family(keyspace, column_family)
self._pool = pycassa.connect(keyspace, [spliturl[1]])
self._store = pycassa.ColumnFamily(self._pool, column_family)
def createTSColumnFamily(servers, keyspace, tableName, credentials=None):
"""Create a tsXX Column Family using one of the servers in the ``servers``
list and the ``keysapce`` and ``tableName``.
"""
for server in servers:
try:
manager = SystemManager(server, credentials=credentials)
createUTF8ColumnFamily(manager, keyspace, tableName, ts_table=True)
return None
except (Exception) as e:
# TODO: log when we know how to log
lastError = e
raise RuntimeError("Failed to create CF %s.%s using the server list %s, "\
"last error was %s" % (keyspace, tableName, servers, str(lastError)))
def _cassandra_system_manager(self):
# Retry till cassandra is up
server_idx = 0
connected = False
while not connected:
try:
cass_server = self._server_list[server_idx]
sys_mgr = SystemManager(cass_server, credentials=self._credential)
connected = True
except Exception:
# TODO do only for
# thrift.transport.TTransport.TTransportException
server_idx = (server_idx + 1) % self._num_dbnodes
time.sleep(3)
return sys_mgr
def create_keyspace(self):
"""
Creates the Cassandra Keyspace (if not exist)
"""
sys_mgr = None
try:
sys_mgr = SystemManager()
try:
sys_mgr.describe_ring(settings.KEYSPACE)
except:
logger.info("create_keyspace(): Creating keyspace %s",
settings.KEYSPACE)
sys_mgr.create_keyspace(settings.KEYSPACE, SIMPLE_STRATEGY,
{'replication_factor': '1'})
finally:
if sys_mgr:
sys_mgr.close()
def _init_db(server):
dbsys = SystemManager(server)
dbsys.create_keyspace(KEY_SPACE, SIMPLE_STRATEGY,
{"replication_factor": "1"})
column_family_type = types.CompositeType(
types.UTF8Type(),
types.UTF8Type(),
)
dbsys.create_column_family(
KEY_SPACE,
COLUMN_FAMILY,
comparator_type=column_family_type,
key_validation_class=types.UTF8Type(),
default_validation_class=types.UTF8Type(),
)
def cassandra_reset():
from feedly import settings
hostname = settings.FEEDLY_CASSANDRA_HOSTS[0]
keyspace = 'test_feedly'
sys = SystemManager(hostname)
# sys.drop_keyspace(keyspace)
if keyspace not in sys.list_keyspaces():
sys.create_keyspace(keyspace, SIMPLE_STRATEGY,
{'replication_factor': '1'})
sys.create_column_family(keyspace,
'activity',
comparator_type=UTF8_TYPE)
sys.create_column_family(keyspace,
'timeline',
comparator_type=IntegerType(reversed=True))
def setup_package():
sys = SystemManager()
if TEST_KS in sys.list_keyspaces():
sys.drop_keyspace(TEST_KS)
try:
sys.create_keyspace(TEST_KS, 'SimpleStrategy', {'replication_factor': '1'})
sys.create_column_family(TEST_KS, 'Standard1')
sys.create_column_family(TEST_KS, 'Super1', super=True)
sys.create_column_family(TEST_KS, 'Indexed1')
sys.create_index(TEST_KS, 'Indexed1', 'birthdate', 'LongType')
sys.create_column_family(TEST_KS, 'Counter1',
default_validation_class='CounterColumnType')
sys.create_column_family(TEST_KS, 'SuperCounter1', super=True,
default_validation_class='CounterColumnType')
except Exception, e:
print e
try:
sys.drop_keyspace(TEST_KS)
except:
pass
raise e
def __init__(self, engine, **kw):
super(CassandraStore, self).__init__(engine, **kw)
spliturl = urlsplit(engine)
_, keyspace, column_family = spliturl.path.split('/')
try:
self._store = pycassa.ColumnFamily(
pycassa.ConnectionPool(keyspace, [spliturl.hostname]),
column_family,
)
except pycassa.InvalidRequestException:
from pycassa.system_manager import SystemManager # @UnresolvedImport @IgnorePep8
system_manager = SystemManager(spliturl[1])
system_manager.create_keyspace(
keyspace,
pycassa.system_manager.SIMPLE_STRATEGY,
dict(replication_factor=native(kw.get('replication', 1))),
)
system_manager.create_column_family(keyspace, column_family)
self._store = pycassa.ColumnFamily(
pycassa.ConnectionPool(keyspace, [spliturl.netloc]),
column_family,
)
def _create_column_family(self,
family,
bytes_columns=[],
key_validation_class=TIME_UUID_TYPE):
'''
Creates a column family of the name 'family' and sets any of
the names in the bytes_column list to have the BYTES_TYPE.
key_validation_class defaults to TIME_UUID_TYPE and could also
be ASCII_TYPE for md5 hash keys, like we use for 'inbound'
'''
sm = SystemManager(random.choice(self.server_list))
# sys.create_column_family(self.namespace, family, super=False)
sm.create_column_family(self.namespace,
family,
super=False,
key_validation_class=key_validation_class,
default_validation_class=TIME_UUID_TYPE,
column_name_class=ASCII_TYPE)
for column in bytes_columns:
sm.alter_column(self.namespace, family, column, BYTES_TYPE)
sm.close()
def detect_nodes(seeds, keyspace):
from feedly import settings
if not settings.FEEDLY_DISCOVER_CASSANDRA_NODES:
logger.warning('cassandra nodes discovery is off')
return seeds
nodes = frozenset(seeds)
logging.info('retrieve nodes from seeds %r' % seeds)
for seed in seeds:
try:
sys_manager = SystemManager(seed)
except TTransportException:
logging.warning('%s is not a seed or is not reachable' % seed)
continue
ring_description = sys_manager.describe_ring(keyspace)
for ring_range in ring_description:
endpoint_details = ring_range.endpoint_details[0]
hostname = endpoint_details.host
port = getattr(endpoint_details, 'port', 9160)
nodes = nodes.union({
'%s:%s' % (hostname, port),
}, nodes)
break
return nodes
def _clean_db(server):
dbsys = SystemManager(server)
dbsys.drop_keyspace(KEY_SPACE)
dbsys.close()
def _sys_mgr():
'''
Return a pycassa system manager connection object
'''
return SystemManager('{0}:{1}'.format(host, thrift_port))
def setup_module():
global sys
sys = SystemManager()
def db_export(self):
db_contents = {'cassandra': {},
'zookeeper': {}}
cassandra_contents = db_contents['cassandra']
creds = None
if self._api_args.cassandra_user and self._api_args.cassandra_password:
creds = {'username': self._api_args.cassandra_user,
'password': self._api_args.cassandra_password}
socket_factory = default_socket_factory
if ('cassandra_use_ssl' in self._api_args and
self._api_args.cassandra_use_ssl):
socket_factory = self._make_ssl_socket_factory(
self._api_args.cassandra_ca_certs, validate=False)
sys_mgr = SystemManager(
self._api_args.cassandra_server_list[0],
credentials=creds,
socket_factory=socket_factory)
existing_keyspaces = sys_mgr.list_keyspaces()
for ks_name in set(KEYSPACES) - set(self._args.omit_keyspaces or []):
if self._api_args.cluster_id:
full_ks_name = '%s_%s' %(self._api_args.cluster_id, ks_name)
else:
full_ks_name = ks_name
if full_ks_name not in existing_keyspaces:
continue
cassandra_contents[ks_name] = {}
pool = ConnectionPool(
full_ks_name, self._api_args.cassandra_server_list,
pool_timeout=120, max_retries=-1, timeout=5,
socket_factory=socket_factory, credentials=creds)
for cf_name in sys_mgr.get_keyspace_column_families(full_ks_name):
cassandra_contents[ks_name][cf_name] = {}
cf = ColumnFamily(pool, cf_name,
buffer_size=self._args.buffer_size)
for r,c in cf.get_range(column_count=10000000, include_timestamp=True):
cassandra_contents[ks_name][cf_name][r] = c
logger.info("Cassandra DB dumped")
def get_nodes(path):
if path[:-1].rpartition('/')[-1] in self._zk_ignore_list:
return []
try:
if not zk.get_children(path):
return [(path, zk.get(path))]
except kazoo.exceptions.NoNodeError:
return []
nodes = []
for child in zk.get_children(path):
nodes.extend(get_nodes('%s%s/' %(path, child)))
return nodes
zk = kazoo.client.KazooClient(self._api_args.zk_server_ip)
zk.start()
nodes = get_nodes(self._api_args.cluster_id+'/')
zk.stop()
db_contents['zookeeper'] = json.dumps(nodes)
logger.info("Zookeeper DB dumped")
f = open(self._args.export_to, 'w')
try:
f.write(json.dumps(db_contents))
finally:
f.close()
logger.info("DB dump wrote to file %s" % self._args.export_to)
