def load_schema(keyspace=None):
# Only loads schemas insode ./.temp
temp_path = sys.path[0] + '/.temp'
host = get_rpc_address()
# unzip schemas.zip
print('Unzipping schemas.zip')
z = zipfile.ZipFile(temp_path + '/schemas.zip', 'r', allowZip64=True)
z.extractall(temp_path)
if keyspace:
for ks in keyspace:
print('Loading keyspace: %s' % ks)
_load(host, temp_path + '/' + ks + '/' + ks + '_schema.cql')
else:
_load(host, temp_path + '/schema.cql')
def load_schema(keyspace = None):
# Only loads schemas insode ./.temp
temp_path = sys.path[0] + '/.temp'
host = get_rpc_address()
# unzip schemas.zip
print('Unzipping schemas.zip')
z = zipfile.ZipFile(temp_path + '/schemas.zip', 'r')
z.extractall(temp_path)
if keyspace:
for ks in keyspace:
print('Loading keyspace: %s' % ks)
_load(host, temp_path + '/' + ks + '/' + ks + '_schema.cql')
else:
_load(host, temp_path + '/schema.cql')
def restore(hosts, keyspace_arg = None, table_arg = None):
cqlsh_host = get_rpc_address()
snapshot_path = sys.path[0] + '/.snapshots'
temp_path = sys.path[0] + '/.temp'
print('Unzipping snapshot file')
if make_dir(temp_path):
clean_dir(temp_path)
zip_path = snapshot_path + '/' + cqlsh_host + '.zip'
zipf = zipfile.ZipFile(zip_path, 'r', allowZip64=True)
zipf.extractall(temp_path)
zipf.close()
print('Checking keyspace and table arguments . . .')
keyspaces = os.listdir(temp_path)
if keyspace_arg:
for ks in keyspace_arg:
if ks not in keyspaces:
print('ERROR: Keyspace arg not in snapshot file')
exit(1)
if table_arg:
for tb in table_arg:
if tb not in os.listdir(temp_path + '/' + ks):
print('ERROR: Table arg not in snapshot file')
exit(1)
else:
tables = table_arg
else:
keyspaces = keyspace_arg
print('Loading snapshot data . . .')
for ks in keyspaces:
if not table_arg:
tables = os.listdir(temp_path + '/' + ks)
print('Loading keyspace: %s' % ks)
for tb in tables:
print('\tLoading table: %s' % tb)
tb_dir = temp_path + '/' + ks + '/' + tb
subprocess.call(['/bin/sstableloader', '-d', ','.join(hosts), tb_dir])
print('Restoration complete')
def restore(hosts, keyspace_arg = None, table_arg = None):
cqlsh_host = get_rpc_address()
snapshot_path = sys.path[0] + '/.snapshots'
temp_path = sys.path[0] + '/.temp'
print('Unzipping snapshot file')
if make_dir(temp_path):
clean_dir(temp_path)
zip_path = snapshot_path + '/' + cqlsh_host + '.zip'
zipf = zipfile.ZipFile(zip_path, 'r')
zipf.extractall(temp_path)
zipf.close()
print('Checking keyspace and table arguments . . .')
keyspaces = os.listdir(temp_path)
if keyspace_arg:
for ks in keyspace_arg:
if ks not in keyspaces:
print('ERROR: Keyspace arg not in snapshot file')
exit(1)
if table_arg:
for tb in table_arg:
if tb not in os.listdir(temp_path + '/' + ks):
print('ERROR: Table arg not in snapshot file')
exit(1)
else:
tables = table_arg
else:
keyspaces = keyspace_arg
print('Loading snapshot data . . .')
for ks in keyspaces:
if not table_arg:
tables = os.listdir(temp_path + '/' + ks)
print('Loading keyspace: %s' % ks)
for tb in tables:
print('\tLoading table: %s' % tb)
tb_dir = temp_path + '/' + ks + '/' + tb
subprocess.call(['/bin/sstableloader', '-d', ','.join(hosts), tb_dir])
print('Restoration complete')
def save_schema(keyspace_arg=None):
host = get_rpc_address()
save_path = sys.path[0] + '/.snapshots/schemas'
keyspaces = get_keyspaces(host)
if keyspace_arg:
for ks in keyspace_arg:
if ks not in keyspaces:
print('ERROR: Invalid keyspace argument')
exit(1)
print('Saving schema . . .')
print_save_path = write_schema(host, save_path)
print('Saved schema as %s' % print_save_path)
for ks in keyspaces:
print_save_path = write_schema(host, save_path, ks)
print('Saved keyspace schema as %s' % print_save_path)
print('Compressing schema file')
shutil.make_archive(save_path, 'zip', save_path)
print('Saving ring information . . .')
write_ring_info(sys.path[0] + '/.snapshots')
def start():
print('Starting service')
startservice = subprocess.Popen(('sudo', 'service', 'cassandra', 'start'))
startservice.wait()
# process will not continue without calling script with nohup
print('Starting Cassandra')
start_cassandra = subprocess.Popen(('/usr/sbin/cassandra'), shell=True)
start = time.time()
current = start
host = get_rpc_address()
while check_host(host) != 0:
print('Time elapsed waiting for Cassandra: %s' % (current - start))
current = time.time()
if current - start > _TIMEOUT:
print('ERROR: Timed out waiting for cassandra to start.' +
'Try increasing _TIMEOUT value.')
exit(1)
time.sleep(_SLEEP)
print('Hard reset complete')
def start():
print('Starting service')
startservice = subprocess.Popen(('sudo', 'service', 'cassandra', 'start'))
startservice.wait()
# process will not continue without calling script with nohup
print('Starting Cassandra')
start_cassandra = subprocess.Popen(('/usr/sbin/cassandra'), shell=True)
start = time.time()
current = start
host = get_rpc_address()
while check_host(host) != 0:
print('Time elapsed waiting for Cassandra: %s' % (current - start))
current = time.time()
if current - start > _TIMEOUT:
print('ERROR: Timed out waiting for cassandra to start.' +
'Try increasing _TIMEOUT value.')
exit(1)
time.sleep(_SLEEP)
print('Hard reset complete')
from cass_functions import cassandra_query, get_keyspaces, get_rpc_address
def destroy_schema(host):
keyspaces = get_keyspaces(host)
for k in keyspaces:
print('Dropping keyspace: %s' % k)
cassandra_query(host, 'DROP KEYSPACE %s;' % k)
if __name__ == '__main__':
destroy_schema(get_rpc_address())
table_dirs.add(structure[keyspace][table])
inactive_dirs = data_dirs - table_dirs
print('Removing inactive directories . . .')
for d in inactive_dirs:
print('\tDeleting: ' + cass_data_dir + '/' + keyspace + '/' +
d)
shutil.rmtree(cass_data_dir + '/' + keyspace + '/' + d)
if backups:
print('Removing old backup db files')
for d in table_dirs:
clean_directory(cass_data_dir + '/' + keyspace + '/' + d +
'/backups')
print('\nClearing old snapshots . . .')
subprocess.call(['nodetool', 'clearsnapshot'])
def clean_directory(table_directory):
for f in os.listdir(table_directory):
if os.isfile(f):
os.remove(table_directory + '/' + f)
if __name__ == '__main__':
data_cleaner(get_rpc_address(), backups=True)
table_dirs.add(structure[keyspace][table])
inactive_dirs = data_dirs - table_dirs
print('Removing inactive directories . . .')
for d in inactive_dirs:
print('\tDeleting: ' + cass_data_dir + '/' + keyspace + '/' + d)
shutil.rmtree(cass_data_dir + '/' + keyspace + '/' + d)
if backups:
print('Removing old backup db files')
for d in table_dirs:
clean_directory(cass_data_dir + '/' + keyspace + '/' + d + '/backups')
print('\nClearing old snapshots . . .')
subprocess.call(['nodetool', 'clearsnapshot'])
def clean_directory(table_directory):
for f in os.listdir(table_directory):
if os.isfile(f):
os.remove(table_directory + '/' + f)
if __name__ == '__main__':
data_cleaner(get_rpc_address(), backups=True)
def snapshot(keyspace_arg=None, table_arg=None):
# nodetool can only run localhost and cqlsh can only run on host argument
host = get_rpc_address()
title = host # all local snapshots are named by its ip address or rpc_address
save_root = sys.path[0] + '/.snapshots/'
if check_host(host) != 0:
print('ERROR: Invalid host, check rpc_address in this node\'s yaml file')
exit(1)
keyspaces = get_keyspaces(host) # set; retrieves through cqlsh
if len(keyspaces) == 0: # edge case
print('ERROR: No keyspaces found')
exit(1)
print('Checking keyspace arguments . . .')
if keyspace_arg:
for ks in keyspace_arg:
if ks not in keyspaces:
print('ERROR: Keyspaces "%s" not found.' % ks)
exit(1)
else:
keyspaces = set(keyspace_arg)
else:
print('No keyspace arguments.')
structure = get_dir_structure(host, keyspaces) # basic schema in json format
print('Checking table arguments . . .')
if table_arg:
if not keyspace_arg or len(keyspace_arg) != 1:
print('ERROR: Only one keyspace can be specified with table arg')
exit(1)
ks = next(iter(keyspaces)) # retrieve only element in set
for tb in table_arg:
if tb not in structure[ks]:
print('ERROR: Table "%s" not found in keyspace "%s"' % (tb, ks))
exit(1)
else:
tables = set(table_arg)
else:
print('No table arguments.')
print('Valid arguments.\n')
print('Clearing previous cassandra data snapshots . . .')
subprocess.call(['nodetool', 'clearsnapshot'])
if os.path.isdir(save_root): # remove old snapshots from .snapshot
for f in os.listdir(save_root):
if os.path.isdir(save_root + f):
shutil.rmtree(save_root + f)
else:
os.remove(save_root + f)
save_path = save_root + title
if os.path.exists(save_path):
print('ERROR: Snapshot save path conflict')
exit(1)
print('Saving snapshot into %s . . .' % save_path)
print('Producing snapshots . . .')
if keyspace_arg:
if table_arg:
ks = next(iter(keyspaces))
for table in tables:
run_snapshot(title, ks, table)
else:
run_snapshot(title, ' '.join(keyspaces))
else:
run_snapshot(title)
cassandra_data_dir = get_data_dir()
for ks in keyspaces:
if not table_arg:
tables = structure[ks]
for tb in tables:
save_table_path = '%(save_path)s/%(keyspace)s/%(table)s/' \
% dict(save_path = save_path,
keyspace = ks,
table = tb)
load_dir = '%(data_dir)s/%(keyspace)s/%(table_dir)s/snapshots/%(ss_title)s' \
% dict(data_dir = cassandra_data_dir,
keyspace = ks,
table_dir = structure[ks][tb],
ss_title = title)
print('Storing %s in %s' % (tb, save_table_path))
shutil.copytree(load_dir, save_table_path)
print('Compressing snapshot file')
shutil.make_archive(save_path, 'zip', save_path)
print('\nProcess complete. Snapshot stored in %s\n' % save_path)
from cass_functions import cassandra_query, get_keyspaces, get_rpc_address
def destroy_schema(host):
keyspaces = get_keyspaces(host)
for k in keyspaces:
print('Dropping keyspace: %s' % k)
cassandra_query(host, 'DROP KEYSPACE %s;' % k)
if __name__ == '__main__':
destroy_schema(get_rpc_address())