예제 #1
0
def load_schema(keyspace=None):

    # Only loads schemas insode ./.temp
    temp_path = sys.path[0] + '/.temp'
    host = get_rpc_address()

    # unzip schemas.zip
    print('Unzipping schemas.zip')
    z = zipfile.ZipFile(temp_path + '/schemas.zip', 'r', allowZip64=True)
    z.extractall(temp_path)

    if keyspace:
        for ks in keyspace:
            print('Loading keyspace: %s' % ks)
            _load(host, temp_path + '/' + ks + '/' + ks + '_schema.cql')
    else:
        _load(host, temp_path + '/schema.cql')
예제 #2
0
def load_schema(keyspace = None):

    # Only loads schemas insode ./.temp
    temp_path = sys.path[0] + '/.temp'
    host = get_rpc_address()

    # unzip schemas.zip
    print('Unzipping schemas.zip')
    z = zipfile.ZipFile(temp_path + '/schemas.zip', 'r')
    z.extractall(temp_path)

    if keyspace:
        for ks in keyspace:
            print('Loading keyspace: %s' % ks)
            _load(host, temp_path + '/' + ks + '/' + ks + '_schema.cql')
    else:
        _load(host, temp_path + '/schema.cql') 
예제 #3
0
def restore(hosts, keyspace_arg = None, table_arg = None):

    cqlsh_host = get_rpc_address()
    snapshot_path = sys.path[0] + '/.snapshots'
    temp_path = sys.path[0] + '/.temp'

    print('Unzipping snapshot file')
    if make_dir(temp_path):
        clean_dir(temp_path)

    zip_path = snapshot_path + '/' + cqlsh_host + '.zip'
    zipf = zipfile.ZipFile(zip_path, 'r', allowZip64=True)
    zipf.extractall(temp_path)
    zipf.close()

    print('Checking keyspace and table arguments . . .')
    keyspaces = os.listdir(temp_path)
    if keyspace_arg:
        for ks in keyspace_arg:
            if ks not in keyspaces:
                print('ERROR: Keyspace arg not in snapshot file')
                exit(1)
            if table_arg:
                for tb in table_arg:
                    if tb not in os.listdir(temp_path + '/' + ks):
                        print('ERROR: Table arg not in snapshot file')
                        exit(1)
                else:
                    tables = table_arg
        else:
            keyspaces = keyspace_arg

    print('Loading snapshot data . . .')
    for ks in keyspaces:
        if not table_arg:
            tables = os.listdir(temp_path + '/' + ks)
        print('Loading keyspace: %s' % ks)
        for tb in tables:
            print('\tLoading table: %s' % tb)
            tb_dir = temp_path + '/' + ks + '/' + tb
            subprocess.call(['/bin/sstableloader', '-d', ','.join(hosts), tb_dir])

    print('Restoration complete')
예제 #4
0
def restore(hosts, keyspace_arg = None, table_arg = None):

    cqlsh_host = get_rpc_address()
    snapshot_path = sys.path[0] + '/.snapshots'
    temp_path = sys.path[0] + '/.temp'

    print('Unzipping snapshot file')
    if make_dir(temp_path):
        clean_dir(temp_path)

    zip_path = snapshot_path + '/' + cqlsh_host + '.zip'
    zipf = zipfile.ZipFile(zip_path, 'r')
    zipf.extractall(temp_path)
    zipf.close()

    print('Checking keyspace and table arguments . . .')
    keyspaces = os.listdir(temp_path)
    if keyspace_arg:
        for ks in keyspace_arg:
            if ks not in keyspaces:
                print('ERROR: Keyspace arg not in snapshot file')
                exit(1)
            if table_arg:
                for tb in table_arg:
                    if tb not in os.listdir(temp_path + '/' + ks):
                        print('ERROR: Table arg not in snapshot file')
                        exit(1)
                else:
                    tables = table_arg
        else:
            keyspaces = keyspace_arg

    print('Loading snapshot data . . .')
    for ks in keyspaces:
        if not table_arg:
            tables = os.listdir(temp_path + '/' + ks)
        print('Loading keyspace: %s' % ks)
        for tb in tables:
            print('\tLoading table: %s' % tb)
            tb_dir = temp_path + '/' + ks + '/' + tb
            subprocess.call(['/bin/sstableloader', '-d', ','.join(hosts), tb_dir])

    print('Restoration complete')
def save_schema(keyspace_arg=None):

    host = get_rpc_address()
    save_path = sys.path[0] + '/.snapshots/schemas'
    keyspaces = get_keyspaces(host)
    if keyspace_arg:
        for ks in keyspace_arg:
            if ks not in keyspaces:
                print('ERROR: Invalid keyspace argument')
                exit(1)

    print('Saving schema . . .')
    print_save_path = write_schema(host, save_path)
    print('Saved schema as %s' % print_save_path)
    for ks in keyspaces:
        print_save_path = write_schema(host, save_path, ks)
        print('Saved keyspace schema as %s' % print_save_path)

    print('Compressing schema file')
    shutil.make_archive(save_path, 'zip', save_path)

    print('Saving ring information . . .')
    write_ring_info(sys.path[0] + '/.snapshots')
예제 #6
0
def start():

    print('Starting service')
    startservice = subprocess.Popen(('sudo', 'service', 'cassandra', 'start'))
    startservice.wait()

    # process will not continue without calling script with nohup
    print('Starting Cassandra')
    start_cassandra = subprocess.Popen(('/usr/sbin/cassandra'), shell=True)

    start = time.time()
    current = start
    host = get_rpc_address()
    while check_host(host) != 0:
        print('Time elapsed waiting for Cassandra: %s' % (current - start))
        current = time.time()
        if current - start > _TIMEOUT:
            print('ERROR: Timed out waiting for cassandra to start.' +
                  'Try increasing _TIMEOUT value.')
            exit(1)
        time.sleep(_SLEEP)

    print('Hard reset complete')
예제 #7
0
def start():

    print('Starting service')
    startservice = subprocess.Popen(('sudo', 'service', 'cassandra', 'start'))
    startservice.wait()

    # process will not continue without calling script with nohup
    print('Starting Cassandra') 
    start_cassandra = subprocess.Popen(('/usr/sbin/cassandra'), shell=True)

    start = time.time()
    current = start
    host = get_rpc_address()
    while check_host(host) != 0:
        print('Time elapsed waiting for Cassandra: %s' % (current - start))
        current = time.time()
        if current - start > _TIMEOUT:
            print('ERROR: Timed out waiting for cassandra to start.' +
                  'Try increasing _TIMEOUT value.')
            exit(1)
        time.sleep(_SLEEP)

    print('Hard reset complete')
예제 #8
0
from cass_functions import cassandra_query, get_keyspaces, get_rpc_address

def destroy_schema(host):

    keyspaces = get_keyspaces(host)

    for k in keyspaces:
        print('Dropping keyspace: %s' % k)
        cassandra_query(host, 'DROP KEYSPACE %s;' % k)

if __name__ == '__main__':
    destroy_schema(get_rpc_address())
예제 #9
0
                table_dirs.add(structure[keyspace][table])

            inactive_dirs = data_dirs - table_dirs

            print('Removing inactive directories . . .')
            for d in inactive_dirs:
                print('\tDeleting: ' + cass_data_dir + '/' + keyspace + '/' +
                      d)
                shutil.rmtree(cass_data_dir + '/' + keyspace + '/' + d)

            if backups:
                print('Removing old backup db files')
                for d in table_dirs:
                    clean_directory(cass_data_dir + '/' + keyspace + '/' + d +
                                    '/backups')

    print('\nClearing old snapshots . . .')
    subprocess.call(['nodetool', 'clearsnapshot'])


def clean_directory(table_directory):

    for f in os.listdir(table_directory):
        if os.isfile(f):
            os.remove(table_directory + '/' + f)


if __name__ == '__main__':

    data_cleaner(get_rpc_address(), backups=True)
예제 #10
0
                table_dirs.add(structure[keyspace][table])

            inactive_dirs = data_dirs - table_dirs

            print('Removing inactive directories . . .')
            for d in inactive_dirs:
                print('\tDeleting: ' + cass_data_dir + '/' + keyspace + '/' + d)
                shutil.rmtree(cass_data_dir + '/' + keyspace + '/' + d)

            if backups:
                print('Removing old backup db files')
                for d in table_dirs:
                    clean_directory(cass_data_dir + '/' + keyspace + '/' + d + '/backups')

    print('\nClearing old snapshots . . .')
    subprocess.call(['nodetool', 'clearsnapshot'])


def clean_directory(table_directory):

    for f in os.listdir(table_directory):
        if os.isfile(f):
            os.remove(table_directory + '/' + f)


if __name__ == '__main__':

    data_cleaner(get_rpc_address(), backups=True)


def snapshot(keyspace_arg=None, table_arg=None):

    # nodetool can only run localhost and cqlsh can only run on host argument
    host = get_rpc_address()
    title = host # all local snapshots are named by its ip address or rpc_address
    save_root = sys.path[0] + '/.snapshots/'

    if check_host(host) != 0:
        print('ERROR: Invalid host, check rpc_address in this node\'s yaml file')
        exit(1)
    keyspaces = get_keyspaces(host) # set; retrieves through cqlsh
    if len(keyspaces) == 0: # edge case
        print('ERROR: No keyspaces found')
        exit(1)

    print('Checking keyspace arguments . . .')
    if keyspace_arg:
        for ks in keyspace_arg:
            if ks not in keyspaces:
                print('ERROR: Keyspaces "%s" not found.' % ks)
                exit(1)
        else:
            keyspaces = set(keyspace_arg)
    else:
        print('No keyspace arguments.')

    structure = get_dir_structure(host, keyspaces) # basic schema in json format
    print('Checking table arguments . . .')
    if table_arg:
        if not keyspace_arg or len(keyspace_arg) != 1:
            print('ERROR: Only one keyspace can be specified with table arg')
            exit(1)
        ks = next(iter(keyspaces)) # retrieve only element in set
        for tb in table_arg:
            if tb not in structure[ks]:
                print('ERROR: Table "%s" not found in keyspace "%s"' % (tb, ks))
                exit(1)
        else:
            tables = set(table_arg)
    else:
        print('No table arguments.')
    print('Valid arguments.\n')

    print('Clearing previous cassandra data snapshots . . .')
    subprocess.call(['nodetool', 'clearsnapshot'])
    if os.path.isdir(save_root): # remove old snapshots from .snapshot
        for f in os.listdir(save_root):
            if os.path.isdir(save_root + f):
                shutil.rmtree(save_root + f)
            else:
                os.remove(save_root + f)

    save_path = save_root + title
    if os.path.exists(save_path):
        print('ERROR: Snapshot save path conflict')
        exit(1)

    print('Saving snapshot into %s . . .' % save_path)
    print('Producing snapshots . . .')
    if keyspace_arg:
        if table_arg:
            ks = next(iter(keyspaces))
            for table in tables:
                run_snapshot(title, ks, table)
        else:
            run_snapshot(title, ' '.join(keyspaces))
    else:
        run_snapshot(title)

    cassandra_data_dir = get_data_dir()
    for ks in keyspaces:
        if not table_arg:
            tables = structure[ks]
        for tb in tables:
            save_table_path = '%(save_path)s/%(keyspace)s/%(table)s/' \
                          % dict(save_path = save_path,
                                 keyspace  = ks,
                                 table     = tb)
            load_dir = '%(data_dir)s/%(keyspace)s/%(table_dir)s/snapshots/%(ss_title)s' \
                   % dict(data_dir  = cassandra_data_dir,
                          keyspace  = ks,
                          table_dir = structure[ks][tb],
                          ss_title  = title)
            print('Storing %s in %s' % (tb, save_table_path))
            shutil.copytree(load_dir, save_table_path)

    print('Compressing snapshot file')
    shutil.make_archive(save_path, 'zip', save_path)

    print('\nProcess complete. Snapshot stored in %s\n' % save_path)
예제 #12
0
from cass_functions import cassandra_query, get_keyspaces, get_rpc_address


def destroy_schema(host):

    keyspaces = get_keyspaces(host)

    for k in keyspaces:
        print('Dropping keyspace: %s' % k)
        cassandra_query(host, 'DROP KEYSPACE %s;' % k)


if __name__ == '__main__':
    destroy_schema(get_rpc_address())