def create_snap(dataset, quiet=False): ''' ''' zettaknight_utils.zlog("create_snap started", "DEBUG") zettaknight_utils.check_quiet(quiet) snap = "{0}@{1}".format(dataset, zettaknight_globs.today_date) snap_list = zettaknight_utils.spawn_job("/sbin/zfs list -r -t snapshot -o name -H {0}".format(snap)) if int(0) not in snap_list.iterkeys(): ret = zettaknight_utils.spawn_job("/sbin/zfs snapshot -r {0}".format(snap)) else: ret = {0: "Snapshot {0} already exists.".format(snap)} zettaknight_utils.zlog("snapshot {0} already exists".format(snap), "INFO") for exit_status, output in ret.iteritems(): if str(exit_status) == "0" and str(output) == "Job succeeded": ret[exit_status] = "Snapshot created: {0}".format(snap) zettaknight_utils.zlog("snapshot created: {0}".format(snap), "SUCCESS") if not quiet: snap_out = {} snap_out[dataset] = {} snap_out[dataset][inspect.stack()[0][3]] = ret zettaknight_utils.parse_output(snap_out) zettaknight_utils.zlog("create_snap exiting", "DEBUG") return ret
def create_snap(dataset, quiet=False): ''' ''' zettaknight_utils.zlog("create_snap started", "DEBUG") zettaknight_utils.check_quiet(quiet) snap = "{0}@{1}".format(dataset, zettaknight_globs.today_date) snap_list = zettaknight_utils.spawn_job( "/sbin/zfs list -r -t snapshot -o name -H {0}".format(snap)) if int(0) not in snap_list.iterkeys(): ret = zettaknight_utils.spawn_job( "/sbin/zfs snapshot -r {0}".format(snap)) else: ret = {0: "Snapshot {0} already exists.".format(snap)} zettaknight_utils.zlog("snapshot {0} already exists".format(snap), "INFO") for exit_status, output in ret.iteritems(): if str(exit_status) == "0" and str(output) == "Job succeeded": ret[exit_status] = "Snapshot created: {0}".format(snap) zettaknight_utils.zlog("snapshot created: {0}".format(snap), "SUCCESS") if not quiet: snap_out = {} snap_out[dataset] = {} snap_out[dataset][inspect.stack()[0][3]] = ret zettaknight_utils.parse_output(snap_out) zettaknight_utils.zlog("create_snap exiting", "DEBUG") return ret
def sync(*args, **kwargs): ''' ''' zettaknight_utils.zlog("kwargs recieved by sync\n\t{0}".format(kwargs), "DEBUG") if zettaknight_globs.help_flag: ret = """Sync: Syncs snapshots to a remote server. Usage: zettaknight sync <dataset> <remote_ssh> Required Arguments: dataset Specifies the dataset to sync. remote_ssh Specifies remote server to sync snapshots to.""" return ret #if 'priority' in zettaknight_globs.zfs_conf[dataset]: # priority = zettaknight_globs.zfs_conf[dataset]['priority'] # print(priority) # if isinstance(priority, int): # print("priority is an integer") # os.nice(int(priority)) try: if 'dataset' and 'remote_ssh' in kwargs.iterkeys(): sync_cmd = "bash {0} -d {1} -s {2}".format( zettaknight_globs.sync_script, kwargs['dataset'], kwargs['remote_ssh']) else: raise Exception( "dataset and remote_ssh is are required kwargs for sync") if 'identity_file' in kwargs.iterkeys(): sync_cmd = "{0} -i {1}".format(sync_cmd, kwargs['identity_file']) if 'pull_snap' in kwargs.iterkeys(): if kwargs['pull_snap']: sync_cmd = "{0} -p".format(sync_cmd) if 'priority' in kwargs.iterkeys(): sync_cmd = "{0} -n {1}".format(sync_cmd, kwargs['priority']) if str(inspect.stack()[1][3]) is 'sync_all': zettaknight_utils.zlog( "starting sync job:\n\t{0}".format(sync_cmd), "INFO") ret = zettaknight_utils.spawn_job(sync_cmd) else: ret = {} ret[dataset] = {} ret[dataset]['Snapshot sync'] = zettaknight_utils.spawn_job( sync_cmd) except Exception as e: zettaknight_utils.zlog("{0}".format(e), "CRITICAL") sys.exit(1) return ret
def replace_string(string, new_string, file): ''' function is take a given string, replace with another string in a specified file ''' ret = {} ret[zettaknight_globs.fqdn] = {} try: string = str(string) new_string = str(new_string) for line in fileinput.input(file, inplace=1): if string in line: line = line.replace(string, new_string) ret = "replaced {0} with {1}".format(string, line) #ret[zettaknight_globs.fqdn]['replace_string'] = {0 : "replaced {0} with {1}".format(string, line)} sys.stdout.write(line) except Exception as e: zettaknight_utils.zlog("{0}".format(e), "ERROR") sys.exit(1) return ret
def replace_string(string, new_string, file): ''' function is take a given string, replace with another string in a specified file ''' ret = {} ret[zettaknight_globs.fqdn] = {} try: string = str(string) new_string = str(new_string) for line in fileinput.input(file, inplace=1): if string in line: line = line.replace(string,new_string) ret = "replaced {0} with {1}".format(string, line) #ret[zettaknight_globs.fqdn]['replace_string'] = {0 : "replaced {0} with {1}".format(string, line)} sys.stdout.write(line) except Exception as e: zettaknight_utils.zlog("{0}".format(e), "ERROR") sys.exit(1) return ret
def sync(*args, **kwargs): ''' ''' zettaknight_utils.zlog("kwargs recieved by sync\n\t{0}".format(kwargs), "DEBUG") if zettaknight_globs.help_flag: ret = """Sync: Syncs snapshots to a remote server. Usage: zettaknight sync <dataset> <remote_ssh> Required Arguments: dataset Specifies the dataset to sync. remote_ssh Specifies remote server to sync snapshots to.""" return ret #if 'priority' in zettaknight_globs.zfs_conf[dataset]: # priority = zettaknight_globs.zfs_conf[dataset]['priority'] # print(priority) # if isinstance(priority, int): # print("priority is an integer") # os.nice(int(priority)) try: if 'dataset' and 'remote_ssh' in kwargs.iterkeys(): sync_cmd = "bash {0} -d {1} -s {2}".format(zettaknight_globs.sync_script, kwargs['dataset'], kwargs['remote_ssh']) else: raise Exception("dataset and remote_ssh is are required kwargs for sync") if 'identity_file' in kwargs.iterkeys(): sync_cmd = "{0} -i {1}".format(sync_cmd, kwargs['identity_file']) if 'pull_snap' in kwargs.iterkeys(): if kwargs['pull_snap']: sync_cmd = "{0} -p".format(sync_cmd) if 'priority' in kwargs.iterkeys(): sync_cmd = "{0} -n {1}".format(sync_cmd, kwargs['priority']) if str(inspect.stack()[1][3]) is 'sync_all': zettaknight_utils.zlog("starting sync job:\n\t{0}".format(sync_cmd),"INFO") ret = zettaknight_utils.spawn_job(sync_cmd) else: ret = {} ret[dataset] = {} ret[dataset]['Snapshot sync'] = zettaknight_utils.spawn_job(sync_cmd) except Exception as e: zettaknight_utils.zlog("{0}".format(e),"CRITICAL") sys.exit(1) return ret
def build_out_config(force=False): ''' This function reads in the pool config file and creates the zfs data structures defined in it ''' ret = {} ret[zettaknight_globs.fqdn] = {} ret[zettaknight_globs.fqdn]['Build Config'] = {} create_config = {} create_config['create_config'] = False create_config['nfs'] = False try: #create pools defined in zpool conf if zettaknight_globs.zpool_conf: for zpool in zettaknight_globs.zpool_conf.iterkeys(): zettaknight_utils.zlog( "determining if zpool {0} exists:\n\t[build_out_config] --> spawn_job : /sbin/zpool list -H {0}" .format(zpool), "DEBUG") d = zettaknight_utils.spawn_job( "/sbin/zpool list -H '{0}'".format(zpool)) chk_code, chk_msg = d.popitem() if int(chk_code) is not 0: ret[zettaknight_globs.fqdn]['Create {0}'.format( zpool)] = {} zettaknight_utils.zlog( "creating {0}:\n\t[build_out_config] --> zettaknight_zpool.create_zpool" .format(zpool), "DEBUG") out1 = zettaknight_zpool.create_zpool( zpool, **zettaknight_globs.zpool_conf[zpool]) ret[zettaknight_globs.fqdn]['Create {0}'.format( zpool)] = out1[zpool]['Create Zpool'] #print(ret) #create datasets defined in zfs_conf for dataset in zettaknight_globs.zfs_conf.iterkeys(): zettaknight_utils.zlog( "determining if dataset {0} exists:\n\t[build_out_config] --> spawn_job : /sbin/zfs list -H {0}" .format(dataset), "DEBUG") d = zettaknight_utils.spawn_job( "/sbin/zfs list -H '{0}'".format(dataset)) chk_code, chk_msg = d.popitem() if int(chk_code) is not 0: zettaknight_utils.zlog( "creating {0}:\n\t[build_out_config] --> add_dataset". format(dataset), "DEBUG") out2 = add_dataset(dataset, **create_config) ret[zettaknight_globs.fqdn]['Create dataset {0}'.format( dataset)] = out2 ret[zettaknight_globs. fqdn]['Build Config']['0'] = "Everything Looks Okay Here" except Exception as e: zettaknight_utils.zlog("{0}".format(e), "ERROR") ret[zettaknight_globs.fqdn]['Build Config']['1'] = e zettaknight_utils.zlog("ret for [build_out_config]:\n\t{0}".format(ret), "DEBUG") return ret
def rename_dataset(**kwargs): ''' ''' import paramiko try: conf_file = zettaknight_globs.config_file_new if not 'keyfile' in kwargs.iterkeys(): keyfile = zettaknight_globs.identity_file if not 'dataset' in kwargs.iterkeys(): raise ValueError('A very specific bad thing happened') #required argument, show_help else: dataset = kwargs['dataset'] if not 'new_dataset' in kwargs.iterkeys(): #required argument, show_help raise ValueError('A very specific bad thing happened') else: new_dataset = kwargs['new_dataset'] if not 'user' in kwargs.iterkeys(): user = zettaknight_globs.zfs_conf[dataset]['user'] print("user is {0}".format(user)) remote_server = [] for r_server in zettaknight_globs.zfs_conf[dataset]['snap'][ 'remote_server']: remote_server.append(r_server) print(remote_server) except Exception as e: zettaknight_utils.zlog("{0}".format(e), "ERROR") sys.exit(1) ret = {} ret[zettaknight_globs.fqdn] = {} ret[zettaknight_globs.fqdn]['rename {0}'.format(dataset)] = {} try: for r in remote_server: ssh = paramiko.SSHClient() ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) ssh.connect(r, username=user, key_filename=keyfile) remote_sudo_cmd = "zfs rename {0} {1}".format(dataset, new_dataset) ssh_stdin, ssh_stdout, ssh_stderr = ssh.exec_command( remote_sudo_cmd) print ssh_stdout.read() #call function replace_string replace_string(dataset, new_dataset, conf_file) cmd = "zfs rename {0} {1}".format(dataset, new_dataset) zettaknight_utils.spawn_job(cmd) except Exception as e: zettaknight_utils.zlog("{0}".format(e), "ERROR") sys.exit(1) ret[zettaknight_globs.fqdn]['rename {0}'.format(dataset)] = { 0: "successfully renamed {0} to {1}, all records have been updated". format(dataset, new_dataset) } return ret
def sync_all(**kwargs): ''' ''' zettaknight_utils.zlog("kwargs passed to sync_all:\n\t{0}".format(kwargs), "DEBUG") ret = {} if 'parallel' in kwargs.iterkeys(): parallel = zettaknight_utils._str_to_bool(kwargs['parallel']) else: parallel = False zettaknight_utils.zlog( "[sync_all] started, parallel is {0}".format(parallel), "INFO") if zettaknight_globs.help_flag: ret = """Sync All: Syncs snapshots for all defined datasets. Datasets to sync and remote targets are pulled from the Zettaknight configuration files. Usage: zettaknight sync_all """ return ret protocol = "ssh" sync_list = [] index = 1 list_pos = 0 for dataset in zettaknight_globs.zfs_conf.iterkeys(): if not parallel: ret[dataset] = {} if 'snap' in zettaknight_globs.zfs_conf[dataset].iterkeys(): if zettaknight_globs.zfs_conf[dataset]['snap']: if 'remote_server' in zettaknight_globs.zfs_conf[dataset][ 'snap'].iterkeys(): for remote_server in zettaknight_globs.zfs_conf[dataset][ 'snap']['remote_server']: try: if str(zettaknight_globs.zfs_conf[dataset] ['primary']) != str(zettaknight_globs.fqdn): if str(zettaknight_globs.zfs_conf[dataset] ['primary']) == str(remote_server): pull_snap = True else: pull_snap = False else: pull_snap = False except KeyError: pull_snap = False pass my_kwargs = {} my_kwargs['dataset'] = dataset my_kwargs['remote_ssh'] = "{0}@{1}".format( zettaknight_globs.zfs_conf[dataset]['user'], remote_server) my_kwargs[ 'identity_file'] = zettaknight_globs.identity_file my_kwargs['pull_snap'] = pull_snap if parallel: if 'dataset' and 'remote_ssh' in my_kwargs.iterkeys( ): sync_cmd = "bash {0} -d {1} -s {2}".format( zettaknight_globs.sync_script, my_kwargs['dataset'], my_kwargs['remote_ssh']) else: raise Exception( "dataset and remote_ssh are required kwargs for sync" ) if 'identity_file' in my_kwargs.iterkeys(): sync_cmd = "{0} -i {1}".format( sync_cmd, my_kwargs['identity_file']) if 'pull_snap' in my_kwargs.iterkeys(): if my_kwargs['pull_snap']: sync_cmd = "{0} -p".format(sync_cmd) if 'priority' in my_kwargs.iterkeys(): sync_cmd = "{0} -n {1}".format( sync_cmd, my_kwargs['priority']) if sync_cmd: sync_list.append(sync_cmd) else: ret[dataset]['Snapshot sync with {0}'.format( remote_server)] = sync(**my_kwargs) if parallel: job_list = zettaknight_utils.spawn_jobs(sync_list) ret[zettaknight_globs.fqdn] = {} for job in job_list: ret[zettaknight_globs.fqdn]['Snapshot sync job {0}'.format( index)] = job_list[list_pos] index += 1 list_pos += 1 zettaknight_utils.zlog("[sync_all] return:\n\t{0}".format(ret), "DEBUG") return ret
def create_zpool(pool=False, disk_list=False, raid=False, luks=False, slog=False, create_config=False, ldap=False, recordsize=False, ashift=False, keyfile=False): if zettaknight_globs.help_flag: ret = """Create Zpool: See help entry for create function. zettaknight help create """ return ret ret = {} ret[pool] = {} ret[pool]['Create Zpool'] = {} if not raid: raid = "12+2" try: disks, z_level = raid.split("+") except Exception as e: ret[pool]['Create Zpool'] = {'1': "{0}\nargument raid must be in x+y format, i.e. 2+1".format(e)} zettaknight_utils.parse_output(ret) sys.exit(0) create_cmd = "bash {0} -d {1} -z {2}".format(zettaknight_globs.zpool_create_script, disks, z_level) if disk_list: create_cmd = "{0} -f '{1}'".format(create_cmd, disk_list) if pool: create_cmd = "{0} -p '{1}'".format(create_cmd, pool) if luks: create_cmd = "{0} -l".format(create_cmd) if slog: create_cmd = "{0} -s '{1}'".format(create_cmd, slog) if ldap: create_cmd = "{0} -i".format(create_cmd) if recordsize: if any(i in recordsize for i in 'KM'): create_cmd = "{0} -r {1}".format(create_cmd, recordsize) else: print(zettaknight_utils.printcolors("Recordsize must be in number/unit format. ie. 1M, or 512K", "FAIL")) sys.exit(0) if ashift: create_cmd = "{0} -a {1}".format(create_cmd, ashift) if keyfile: create_cmd = "{0} -k {1}".format(create_cmd, keyfile) try: ret[pool]['Create Zpool'] = zettaknight_utils.spawn_job(create_cmd) except Exception as e: zettaknight_utils.zlog("{0}".format(e), "ERROR") ret[pool]['Create Zpool']['1'] = e return ret
def recover(snapshot, filename, relocate=None): ''' ''' if zettaknight_globs.help_flag: ret = """Recover: Usage: zettaknight recover <snapshot> <filepath> (<directory to recover file to>)*optional Recovers a previous version of a file or folder from a specified snapshot. Information to use in calling zettaknight recover can be found in the output from the find_versions function. By default, files/folders will be recovered to their original location with .R appended to the end of the name. Required Arguments: snapshot Specifies the snapshot to recover files from. filename Defines target file(s) to recover. This should be the full path to a file or folder (/zfs_data/<some dataset>/<some filename>.<ext>) Optional Arguments: relocate Defines an alternate path to recover files/folders to. If used, .R will not be appended, and the recover will overwrite any existing files.""" return ret file = filename snap = snapshot dataset, snap_date = snap.split('@', 1) find_snap = find_versions(dataset, file, "quiet") find_snap_keys = find_snap.iterkeys() snap_basedir = "/{0}/.zfs/snapshot".format(dataset) active_basedir = "/{0}".format(dataset) dir_flag = False ret = {} ret[dataset] = {} try: if str(dataset) in str(file): file = file.split(dataset, 1)[1] last_ref = file.rsplit("/", 1)[1] except Exception as e: pass if dataset in file.rsplit("/", 1): file = file.split(dataset, 1)[1] last_ref = file.rsplit("/",1)[1] if snap not in find_snap_keys: try: raise Exception("Recoverable versions of {0} not found in dataset {1}".format(file, dataset)) except Exception as e: zettaknight_utils.zlog("{0}".format(e), "ERROR") path_list1 = find_snap[snap] path_list = [] if os.path.isdir("/{0}{1}".format(dataset, file)): dir_flag = True for line in path_list1: if line.startswith("-") or line.startswith("M") or line.startswith("R"): a, b = line.rsplit("/", 1) if not dir_flag: try: if str(b) == str(last_ref): path_list.append(line) except: if str(b) == str(file): path_list.append(line) pass else: if a.startswith("-") or line.startswith("M") or line.startswith("R"): a, b = line.rsplit(dataset, 2) if os.path.isdir("{0}/{1}{2}".format(snap_basedir, snap_date, b)): dir_flag = True path_list.append(line) if len(path_list) > 1: exc_out = "" for i in path_list: if exc_out: exc_out = "{0}\n\t{1}".format(exc_out, i) else: exc_out = str(i) err_msg = "Matching files/folders in snapshot:" print(zettaknight_utils.printcolors("\nAmbiguous recover request. Multiple matches for {0}.".format(file), "FAIL")) print("{0} {1}".format(zettaknight_utils.printcolors(err_msg, "FAIL"), zettaknight_utils.printcolors(snap, "OKBLUE"))) print("\t{0}".format(zettaknight_utils.printcolors(exc_out, "WARNING"))) print(zettaknight_utils.printcolors("Re-run with explicit path to file.\n", "FAIL")) try: raise Exception("Ambiguous file reference.") except Exception as e: zettaknight_utils.zlog("{0}".format(e), "ERROR") sys.exit(0) if len(path_list) < 1: try: raise Exception("No restorable files or folders identified. \nRe-run with explicit path to file?") except Exception as e: zettaknight_utils.zlog("{0}".format(e), "ERROR") sys.exit(0) a, p = path_list[0].split(dataset, 1) path, dirs = p.split(file, 1) if dir_flag: a, b = path_list[0].rsplit(dataset, 2) file_loc = "{0}/{1}{2}".format(snap_basedir, snap_date, b) #bad, dir_path = path_list[0].split(file, 1) #list_dirs = dir_path.rsplit("/", 1) #try: # out_dir = list_dirs[0] #except: out_dir = "" # pass #file_loc = "{0}/{1}/{2}{3}/".format(snap_basedir, snap_date, file, dir_path ) #mkdir_cmd = "/bin/mkdir {0}/{1}.R".format(active_basedir, file) #mkdir_run = zettaknight_utils.spawn_job(mkdir_cmd) #mkdir_cmd = "/bin/mkdir -p {0}/{1}.R/{2}".format(active_basedir, file, dir_path) #mkdir_run = zettaknight_utils.spawn_job(mkdir_cmd) out_path = "{0}{1}{2}.R{3}/".format(active_basedir, str(path), file, out_dir) else: file_loc = "{0}/{1}/{2}".format(snap_basedir, snap_date, p) out_path = "{0}{1}{2}.R".format(active_basedir, str(path), file) if relocate: out_path = relocate rec_cmd = "/bin/cp ""-r -p ""{0}"" ""{1}".format(file_loc, out_path) rec_run = zettaknight_utils.spawn_job(rec_cmd) rec_dict = eval(str(rec_run)) for k,v in rec_dict.iteritems(): if str(k) is "0": print("Recover operation succeeded.") print("\tFilename: {0}".format(zettaknight_utils.printcolors(file, "OKBLUE"))) print("\t\tFile(s) restored to version: {0}".format(zettaknight_utils.printcolors(snap_date, "OKGREEN"))) print("\t\tFile(s) restored to: {0}".format(zettaknight_utils.printcolors(out_path, "OKGREEN"))) if str(k) is not "0": print("Recover operation failed.") print("\tFilename: {0}".format(zettaknight_utils.printcolors(file, "OKBLUE"))) print("\t\tRestore failed with error: {0}".format(zettaknight_utils.printcolors(v, "FAIL"))) return
def zfs_maintain(dset=False): zettaknight_utils.zlog("{0} object passed to [zfs_maintain]:\n\t{1}".format(type(dset), dset),"DEBUG") ''' The zfs_maintain function reads in dataset and maintenance requirements from /opt/clemson/zfs_scripts/maintain.conf Accepted configuration keys are: - remote_server - retention - reservation - quota - user - contact ''' ret = {} if zettaknight_globs.help_flag: ret = """ZFS Maintain: The zfs_maintain function reads in dataset and maintenance requirements from configuration files and enforces defined configurations. Usage: zettaknight zfs_maintain (<dataset>) Optional Arguments: dataset Specifies the dataset to run maintenance functions on. If not provided, all defined datasets will have maintenance performed.""" return ret protocol = "ssh" if dset and str(dset) not in zettaknight_globs.zfs_conf.iterkeys(): ret[dset] = {} ret[dset]['zfs maintain'] = {1: "{0} is not a Zettaknight controlled dataset.".format(dset)} #zettaknight_utils.parse_output(ret) return ret for dataset in zettaknight_globs.zfs_conf.iterkeys(): if dset: if str(dset) != str(dataset): continue ret[dataset] = {} #return config file ret[dataset]['Config'] = {0: zettaknight_globs.zfs_conf[dataset]} ret[dataset]['Cleanup'] = cleanup_snaps(dataset, zettaknight_globs.zfs_conf[dataset]['retention']) if 'snap' in zettaknight_globs.zfs_conf[dataset].iterkeys(): ret[dataset]['snapshot'] = create_snap(dataset, "quiet") if zettaknight_globs.zfs_conf[dataset]['quota']: ret[dataset]['Quota'] = set_quota(dataset, zettaknight_globs.zfs_conf[dataset]['quota']) if zettaknight_globs.zfs_conf[dataset]['refquota']: ret[dataset]['Refquota'] = set_refquota(dataset, zettaknight_globs.zfs_conf[dataset]['refquota']) if zettaknight_globs.zfs_conf[dataset]['reservation']: ret[dataset]['Reservation'] = set_reservation(dataset, zettaknight_globs.zfs_conf[dataset]['reservation']) if zettaknight_globs.zfs_conf[dataset]['refreservation']: ret[dataset]['Refreservation'] = set_refreservation(dataset, zettaknight_globs.zfs_conf[dataset]['refreservation']) #print(ret) #parse_output(ret) return ret
def rename_dataset(**kwargs): ''' ''' import paramiko try: conf_file = zettaknight_globs.config_file_new if not 'keyfile' in kwargs.iterkeys(): keyfile = zettaknight_globs.identity_file if not 'dataset' in kwargs.iterkeys(): raise ValueError('A very specific bad thing happened') #required argument, show_help else: dataset = kwargs['dataset'] if not 'new_dataset' in kwargs.iterkeys(): #required argument, show_help raise ValueError('A very specific bad thing happened') else: new_dataset = kwargs['new_dataset'] if not 'user' in kwargs.iterkeys(): user = zettaknight_globs.zfs_conf[dataset]['user'] print("user is {0}".format(user)) remote_server = [] for r_server in zettaknight_globs.zfs_conf[dataset]['snap']['remote_server']: remote_server.append(r_server) print(remote_server) except Exception as e: zettaknight_utils.zlog("{0}".format(e), "ERROR") sys.exit(1) ret = {} ret[zettaknight_globs.fqdn] = {} ret[zettaknight_globs.fqdn]['rename {0}'.format(dataset)] = {} try: for r in remote_server: ssh = paramiko.SSHClient() ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) ssh.connect(r, username=user, key_filename=keyfile) remote_sudo_cmd = "zfs rename {0} {1}".format(dataset, new_dataset) ssh_stdin, ssh_stdout, ssh_stderr = ssh.exec_command(remote_sudo_cmd) print ssh_stdout.read() #call function replace_string replace_string(dataset, new_dataset, conf_file) cmd = "zfs rename {0} {1}".format(dataset, new_dataset) zettaknight_utils.spawn_job(cmd) except Exception as e: zettaknight_utils.zlog("{0}".format(e), "ERROR") sys.exit(1) ret[zettaknight_globs.fqdn]['rename {0}'.format(dataset)] = {0 : "successfully renamed {0} to {1}, all records have been updated".format(dataset, new_dataset)} return ret
def sync_all(**kwargs): ''' ''' zettaknight_utils.zlog("kwargs passed to sync_all:\n\t{0}".format(kwargs), "DEBUG") ret = {} if 'parallel' in kwargs.iterkeys(): parallel = zettaknight_utils._str_to_bool(kwargs['parallel']) else: parallel = False zettaknight_utils.zlog("[sync_all] started, parallel is {0}".format(parallel), "INFO") if zettaknight_globs.help_flag: ret = """Sync All: Syncs snapshots for all defined datasets. Datasets to sync and remote targets are pulled from the Zettaknight configuration files. Usage: zettaknight sync_all """ return ret protocol = "ssh" sync_list = [] index = 1 list_pos = 0 for dataset in zettaknight_globs.zfs_conf.iterkeys(): if not parallel: ret[dataset] = {} if 'snap' in zettaknight_globs.zfs_conf[dataset].iterkeys(): if zettaknight_globs.zfs_conf[dataset]['snap']: if 'remote_server' in zettaknight_globs.zfs_conf[dataset]['snap'].iterkeys(): for remote_server in zettaknight_globs.zfs_conf[dataset]['snap']['remote_server']: try: if str(zettaknight_globs.zfs_conf[dataset]['primary']) != str(zettaknight_globs.fqdn): if str(zettaknight_globs.zfs_conf[dataset]['primary']) == str(remote_server): pull_snap = True else: pull_snap = False else: pull_snap = False except KeyError: pull_snap = False pass my_kwargs = {} my_kwargs['dataset'] = dataset my_kwargs['remote_ssh'] = "{0}@{1}".format(zettaknight_globs.zfs_conf[dataset]['user'], remote_server) my_kwargs['identity_file'] = zettaknight_globs.identity_file my_kwargs['pull_snap'] = pull_snap if parallel: if 'dataset' and 'remote_ssh' in my_kwargs.iterkeys(): sync_cmd = "bash {0} -d {1} -s {2}".format(zettaknight_globs.sync_script, my_kwargs['dataset'], my_kwargs['remote_ssh']) else: raise Exception("dataset and remote_ssh are required kwargs for sync") if 'identity_file' in my_kwargs.iterkeys(): sync_cmd = "{0} -i {1}".format(sync_cmd, my_kwargs['identity_file']) if 'pull_snap' in my_kwargs.iterkeys(): if my_kwargs['pull_snap']: sync_cmd = "{0} -p".format(sync_cmd) if 'priority' in my_kwargs.iterkeys(): sync_cmd = "{0} -n {1}".format(sync_cmd, my_kwargs['priority']) if sync_cmd: sync_list.append(sync_cmd) else: ret[dataset]['Snapshot sync with {0}'.format(remote_server)] = sync(**my_kwargs) if parallel: job_list = zettaknight_utils.spawn_jobs(sync_list) ret[zettaknight_globs.fqdn] = {} for job in job_list: ret[zettaknight_globs.fqdn]['Snapshot sync job {0}'.format(index)] = job_list[list_pos] index += 1 list_pos += 1 zettaknight_utils.zlog("[sync_all] return:\n\t{0}".format(ret), "DEBUG") return ret
def recover(snapshot, filename, relocate=None): ''' ''' if zettaknight_globs.help_flag: ret = """Recover: Usage: zettaknight recover <snapshot> <filepath> (<directory to recover file to>)*optional Recovers a previous version of a file or folder from a specified snapshot. Information to use in calling zettaknight recover can be found in the output from the find_versions function. By default, files/folders will be recovered to their original location with .R appended to the end of the name. Required Arguments: snapshot Specifies the snapshot to recover files from. filename Defines target file(s) to recover. This should be the full path to a file or folder (/zfs_data/<some dataset>/<some filename>.<ext>) Optional Arguments: relocate Defines an alternate path to recover files/folders to. If used, .R will not be appended, and the recover will overwrite any existing files.""" return ret file = filename snap = snapshot dataset, snap_date = snap.split('@', 1) find_snap = find_versions(dataset, file, "quiet") find_snap_keys = find_snap.iterkeys() snap_basedir = "/{0}/.zfs/snapshot".format(dataset) active_basedir = "/{0}".format(dataset) dir_flag = False ret = {} ret[dataset] = {} try: if str(dataset) in str(file): file = file.split(dataset, 1)[1] last_ref = file.rsplit("/", 1)[1] except Exception as e: pass if dataset in file.rsplit("/", 1): file = file.split(dataset, 1)[1] last_ref = file.rsplit("/", 1)[1] if snap not in find_snap_keys: try: raise Exception( "Recoverable versions of {0} not found in dataset {1}".format( file, dataset)) except Exception as e: zettaknight_utils.zlog("{0}".format(e), "ERROR") path_list1 = find_snap[snap] path_list = [] if os.path.isdir("/{0}{1}".format(dataset, file)): dir_flag = True for line in path_list1: if line.startswith("-") or line.startswith("M") or line.startswith( "R"): a, b = line.rsplit("/", 1) if not dir_flag: try: if str(b) == str(last_ref): path_list.append(line) except: if str(b) == str(file): path_list.append(line) pass else: if a.startswith("-") or line.startswith( "M") or line.startswith("R"): a, b = line.rsplit(dataset, 2) if os.path.isdir("{0}/{1}{2}".format( snap_basedir, snap_date, b)): dir_flag = True path_list.append(line) if len(path_list) > 1: exc_out = "" for i in path_list: if exc_out: exc_out = "{0}\n\t{1}".format(exc_out, i) else: exc_out = str(i) err_msg = "Matching files/folders in snapshot:" print( zettaknight_utils.printcolors( "\nAmbiguous recover request. Multiple matches for {0}.". format(file), "FAIL")) print("{0} {1}".format(zettaknight_utils.printcolors(err_msg, "FAIL"), zettaknight_utils.printcolors(snap, "OKBLUE"))) print("\t{0}".format(zettaknight_utils.printcolors(exc_out, "WARNING"))) print( zettaknight_utils.printcolors( "Re-run with explicit path to file.\n", "FAIL")) try: raise Exception("Ambiguous file reference.") except Exception as e: zettaknight_utils.zlog("{0}".format(e), "ERROR") sys.exit(0) if len(path_list) < 1: try: raise Exception( "No restorable files or folders identified. \nRe-run with explicit path to file?" ) except Exception as e: zettaknight_utils.zlog("{0}".format(e), "ERROR") sys.exit(0) a, p = path_list[0].split(dataset, 1) path, dirs = p.split(file, 1) if dir_flag: a, b = path_list[0].rsplit(dataset, 2) file_loc = "{0}/{1}{2}".format(snap_basedir, snap_date, b) #bad, dir_path = path_list[0].split(file, 1) #list_dirs = dir_path.rsplit("/", 1) #try: # out_dir = list_dirs[0] #except: out_dir = "" # pass #file_loc = "{0}/{1}/{2}{3}/".format(snap_basedir, snap_date, file, dir_path ) #mkdir_cmd = "/bin/mkdir {0}/{1}.R".format(active_basedir, file) #mkdir_run = zettaknight_utils.spawn_job(mkdir_cmd) #mkdir_cmd = "/bin/mkdir -p {0}/{1}.R/{2}".format(active_basedir, file, dir_path) #mkdir_run = zettaknight_utils.spawn_job(mkdir_cmd) out_path = "{0}{1}{2}.R{3}/".format(active_basedir, str(path), file, out_dir) else: file_loc = "{0}/{1}/{2}".format(snap_basedir, snap_date, p) out_path = "{0}{1}{2}.R".format(active_basedir, str(path), file) if relocate: out_path = relocate rec_cmd = "/bin/cp " "-r -p " "{0}" " " "{1}".format(file_loc, out_path) rec_run = zettaknight_utils.spawn_job(rec_cmd) rec_dict = eval(str(rec_run)) for k, v in rec_dict.iteritems(): if str(k) is "0": print("Recover operation succeeded.") print("\tFilename: {0}".format( zettaknight_utils.printcolors(file, "OKBLUE"))) print("\t\tFile(s) restored to version: {0}".format( zettaknight_utils.printcolors(snap_date, "OKGREEN"))) print("\t\tFile(s) restored to: {0}".format( zettaknight_utils.printcolors(out_path, "OKGREEN"))) if str(k) is not "0": print("Recover operation failed.") print("\tFilename: {0}".format( zettaknight_utils.printcolors(file, "OKBLUE"))) print("\t\tRestore failed with error: {0}".format( zettaknight_utils.printcolors(v, "FAIL"))) return
def find_versions(dataset, filename, quiet=False): ''' ''' if zettaknight_globs.help_flag: ret = """Find Versions: Usage: zettaknight find_versions zfs_data/<some dataset> <some filename> Searches snapshots of provided dataset for previous versions of filename. Required Arguments: dataset Specifies the dataset whose snapshots will be searched. filename Defines target file(s) to find previous versions of. This can be a full path to a file (/zfs_data/<some dataset>/<some filename>.<ext>), just a filename with or without an extension (<some filename> or <some filename>.<ext>), or just an extension (.<ext>)""" return ret zettaknight_utils.check_quiet(quiet) snaps = {} ret = {} ret[dataset] = {} snaplist_cmd = "/sbin/zfs list -r -t snapshot -o name -H {0}".format( dataset) snaplist_run = subprocess.Popen(shlex.split(snaplist_cmd), stdout=subprocess.PIPE, stderr=subprocess.STDOUT) snaplist_run.wait() snaplist_out = snaplist_run.stdout.read() if not snaplist_out: try: out_dict = {} out_dict[dataset] = {} job = inspect.stack()[0][3] if str(inspect.stack()[1][3]) is 'recover': job = inspect.stack()[1][3] out_dict[dataset][job] = {} out_dict[dataset][job]['1'] = "No snapshots found." raise Exception except Exception as e: zettaknight_utils.parse_output(out_dict) sys.exit(0) for snap in snaplist_out.split(): if snap.startswith("cannot"): try: raise Exception("{0}".format(snaplist_out)) except Exception as e: zettaknight_utils.zlog("{0}".format(e), "ERROR") sys.exit(0) snapdiff_cmd = "/sbin/zfs diff {0}".format(snap) gerp_cmd = "/bin/grep {0}".format(filename) gerp_run = zettaknight_utils.pipe_this(snapdiff_cmd, gerp_cmd) gerp_out = gerp_run.stdout.read() gerp_list = [] if gerp_out: for gerp in gerp_out.split('\n'): if gerp.startswith("-") or gerp.startswith( "M") or gerp.startswith("R"): gerp_list.append(gerp) if gerp_list: snaps[snap] = gerp_list gerp_msg = "" for z in gerp_list: if gerp_msg: gerp_msg = "{0}\n{1}".format(gerp_msg, z) else: gerp_msg = str(z) job = "Snapshot: {0}".format(snap) ret[dataset][job] = {} job_out = "Path:\n{0}".format(gerp_msg) ret[dataset][job]['0'] = job_out if not ret[dataset]: ret[dataset]['Snapshot'] = {} ret[dataset]['Snapshot'][ '1'] = "No modified versions of {0} found.".format( filename, dataset) if not quiet: zettaknight_utils.parse_output(ret) return snaps
def build_out_config(force=False): ''' This function reads in the pool config file and creates the zfs data structures defined in it ''' ret = {} ret[zettaknight_globs.fqdn] = {} ret[zettaknight_globs.fqdn]['Build Config'] = {} create_config = {} create_config['create_config'] = False create_config['nfs'] = False try: #create pools defined in zpool conf if zettaknight_globs.zpool_conf: for zpool in zettaknight_globs.zpool_conf.iterkeys(): zettaknight_utils.zlog("determining if zpool {0} exists:\n\t[build_out_config] --> spawn_job : /sbin/zpool list -H {0}".format(zpool), "DEBUG") d = zettaknight_utils.spawn_job("/sbin/zpool list -H '{0}'".format(zpool)) chk_code, chk_msg = d.popitem() if int(chk_code) is not 0: ret[zettaknight_globs.fqdn]['Create {0}'.format(zpool)] = {} zettaknight_utils.zlog("creating {0}:\n\t[build_out_config] --> zettaknight_zpool.create_zpool".format(zpool), "DEBUG") out1 = zettaknight_zpool.create_zpool(zpool, **zettaknight_globs.zpool_conf[zpool]) ret[zettaknight_globs.fqdn]['Create {0}'.format(zpool)] = out1[zpool]['Create Zpool'] #print(ret) #create datasets defined in zfs_conf for dataset in zettaknight_globs.zfs_conf.iterkeys(): zettaknight_utils.zlog("determining if dataset {0} exists:\n\t[build_out_config] --> spawn_job : /sbin/zfs list -H {0}".format(dataset), "DEBUG") d = zettaknight_utils.spawn_job("/sbin/zfs list -H '{0}'".format(dataset)) chk_code, chk_msg = d.popitem() if int(chk_code) is not 0: zettaknight_utils.zlog("creating {0}:\n\t[build_out_config] --> add_dataset".format(dataset), "DEBUG") out2 = add_dataset(dataset, **create_config) ret[zettaknight_globs.fqdn]['Create dataset {0}'.format(dataset)] = out2 ret[zettaknight_globs.fqdn]['Build Config']['0'] = "Everything Looks Okay Here" except Exception as e: zettaknight_utils.zlog("{0}".format(e), "ERROR") ret[zettaknight_globs.fqdn]['Build Config']['1'] = e zettaknight_utils.zlog("ret for [build_out_config]:\n\t{0}".format(ret), "DEBUG") return ret
def find_versions(dataset, filename, quiet=False): ''' ''' if zettaknight_globs.help_flag: ret = """Find Versions: Usage: zettaknight find_versions zfs_data/<some dataset> <some filename> Searches snapshots of provided dataset for previous versions of filename. Required Arguments: dataset Specifies the dataset whose snapshots will be searched. filename Defines target file(s) to find previous versions of. This can be a full path to a file (/zfs_data/<some dataset>/<some filename>.<ext>), just a filename with or without an extension (<some filename> or <some filename>.<ext>), or just an extension (.<ext>)""" return ret zettaknight_utils.check_quiet(quiet) snaps = {} ret = {} ret[dataset] = {} snaplist_cmd = "/sbin/zfs list -r -t snapshot -o name -H {0}".format(dataset) snaplist_run = subprocess.Popen(shlex.split(snaplist_cmd), stdout = subprocess.PIPE, stderr = subprocess.STDOUT) snaplist_run.wait() snaplist_out = snaplist_run.stdout.read() if not snaplist_out: try: out_dict = {} out_dict[dataset] = {} job = inspect.stack()[0][3] if str(inspect.stack()[1][3]) is 'recover': job = inspect.stack()[1][3] out_dict[dataset][job] = {} out_dict[dataset][job]['1'] = "No snapshots found." raise Exception except Exception as e: zettaknight_utils.parse_output(out_dict) sys.exit(0) for snap in snaplist_out.split(): if snap.startswith("cannot"): try: raise Exception("{0}".format(snaplist_out)) except Exception as e: zettaknight_utils.zlog("{0}".format(e), "ERROR") sys.exit(0) snapdiff_cmd = "/sbin/zfs diff {0}".format(snap) gerp_cmd = "/bin/grep {0}".format(filename) gerp_run = zettaknight_utils.pipe_this(snapdiff_cmd, gerp_cmd) gerp_out = gerp_run.stdout.read() gerp_list = [] if gerp_out: for gerp in gerp_out.split('\n'): if gerp.startswith("-") or gerp.startswith("M") or gerp.startswith("R"): gerp_list.append(gerp) if gerp_list: snaps[snap] = gerp_list gerp_msg = "" for z in gerp_list: if gerp_msg: gerp_msg = "{0}\n{1}".format(gerp_msg, z) else: gerp_msg = str(z) job = "Snapshot: {0}".format(snap) ret[dataset][job] = {} job_out = "Path:\n{0}".format(gerp_msg) ret[dataset][job]['0'] = job_out if not ret[dataset]: ret[dataset]['Snapshot'] = {} ret[dataset]['Snapshot']['1'] = "No modified versions of {0} found.".format(filename, dataset) if not quiet: zettaknight_utils.parse_output(ret) return snaps
def create_zpool(pool=False, disk_list=False, raid=False, luks=False, slog=False, create_config=False, ldap=False, recordsize=False, ashift=False, keyfile=False): if zettaknight_globs.help_flag: ret = """Create Zpool: See help entry for create function. zettaknight help create """ return ret ret = {} ret[pool] = {} ret[pool]['Create Zpool'] = {} if not raid: raid = "12+2" try: disks, z_level = raid.split("+") except Exception as e: ret[pool]['Create Zpool'] = { '1': "{0}\nargument raid must be in x+y format, i.e. 2+1".format(e) } zettaknight_utils.parse_output(ret) sys.exit(0) create_cmd = "bash {0} -d {1} -z {2}".format( zettaknight_globs.zpool_create_script, disks, z_level) if disk_list: create_cmd = "{0} -f '{1}'".format(create_cmd, disk_list) if pool: create_cmd = "{0} -p '{1}'".format(create_cmd, pool) if luks: create_cmd = "{0} -l".format(create_cmd) if slog: create_cmd = "{0} -s '{1}'".format(create_cmd, slog) if ldap: create_cmd = "{0} -i".format(create_cmd) if recordsize: if any(i in recordsize for i in 'KM'): create_cmd = "{0} -r {1}".format(create_cmd, recordsize) else: print( zettaknight_utils.printcolors( "Recordsize must be in number/unit format. ie. 1M, or 512K", "FAIL")) sys.exit(0) if ashift: create_cmd = "{0} -a {1}".format(create_cmd, ashift) if keyfile: create_cmd = "{0} -k {1}".format(create_cmd, keyfile) try: ret[pool]['Create Zpool'] = zettaknight_utils.spawn_job(create_cmd) except Exception as e: zettaknight_utils.zlog("{0}".format(e), "ERROR") ret[pool]['Create Zpool']['1'] = e return ret