def create_zpool(pool=False, disk_list=False, raid=False, luks=False, slog=False, create_config=False, ldap=False, recordsize=False, ashift=False, keyfile=False):
ret = {}
ret[pool] = {}
ret[pool]['Create Zpool'] = {}
if not raid:
raid = "12+2"
try:
disks, z_level = raid.split("+")
except Exception as e:
ret[pool]['Create Zpool'] = {'1': "{0}\nargument raid must be in x+y format, i.e. 2+1".format(e)}
zettaknight_utils.parse_output(ret)
sys.exit(0)
create_cmd = "bash {0} -d {1} -z {2}".format(zettaknight_globs.zpool_create_script, disks, z_level)
if disk_list:
create_cmd = "{0} -f '{1}'".format(create_cmd, disk_list)
if pool:
create_cmd = "{0} -p '{1}'".format(create_cmd, pool)
if luks:
create_cmd = "{0} -l".format(create_cmd)
if slog:
create_cmd = "{0} -s '{1}'".format(create_cmd, slog)
if ldap:
create_cmd = "{0} -i".format(create_cmd)
if recordsize:
if any(i in recordsize for i in 'KM'):
create_cmd = "{0} -r {1}".format(create_cmd, recordsize)
else:
print(zettaknight_utils.printcolors("Recordsize must be in number/unit format. ie. 1M, or 512K", "FAIL"))
sys.exit(0)
if ashift:
create_cmd = "{0} -a {1}".format(create_cmd, ashift)
if keyfile:
create_cmd = "{0} -k {1}".format(create_cmd, keyfile)
try:
ret[pool]['Create Zpool'] = zettaknight_utils.spawn_job(create_cmd)
zettaknight_utils.parse_output(ret)
if create_config:
zettaknight_utils.create_config(pool)
except Exception as e:
print(zettaknight_utils.printcolors(e, "FAIL"))
sys.exit(0)
return ret
def add_dataset(*args, **kwargs):
'''if create_config=True is passed, a configuration file
will be created for the dataset passed in to this function'''
#kwargs = {}
arg_list = list(args)
#kwargs = zettaknight_utils.create_kwargs_from_args(arg_list)
#print("This is kwargs : {0}".format(kwargs))
for arg in arg_list[0:]:
if "=" in arg:
k, v = arg.split("=", 1)
kwargs[k] = v
arg_list.remove(arg)
dataset = arg_list[0]
ret = {}
ret[dataset] = {}
gerp_run = zettaknight_utils.pipe_this2("zfs list | /bin/grep {0}".format(dataset))
gerp_out = gerp_run.stdout.read()
if 'create_config' not in kwargs.iterkeys():
kwargs['create_config'] = True
if kwargs['create_config'] == 'False':
kwargs['create_config'] == False
if kwargs['create_config']:
print(zettaknight_utils.printcolors("\nWill create {0} after completion of config file\n".format(dataset), "OKGREEN"))
conf_dict = {'dataset' : dataset}
zettaknight_utils.create_config(**conf_dict)
#create dataset after commit of file
if int(gerp_run.returncode) is not 0:
create_cmd = "zfs create -p {0}".format(dataset)
create_run = zettaknight_utils.spawn_job(create_cmd)
ret[dataset]['add_dataset'] = create_run
else:
ret[dataset]['add_dataset'] = {'0' : "{0} exists".format(dataset)}
question = zettaknight_utils.query_yes_no("Would you like to create an NFS share for: {0}".format(dataset))
if question:
nfs_share = zettaknight_utils.query_return_item("Where would you like to share it? <i.e 10.20.30.1/24 or my.server>")
zettaknight_utils.sharenfs(dataset, nfs_share)
#zettaknight_utils.parse_output(ret)
return ret
def recover(snapshot, filename, relocate=None):
'''
'''
file = filename
snap = snapshot
dataset, snap_date = snap.split('@', 1)
find_snap = find_versions(dataset, file, "quiet")
find_snap_keys = find_snap.iterkeys()
snap_basedir = "/{0}/.zfs/snapshot".format(dataset)
active_basedir = "/{0}".format(dataset)
dir_flag = False
ret = {}
ret[dataset] = {}
if dataset in file.split("/"):
file = file.split(dataset, 1)[1]
last_ref = file.rsplit("/",1)[1]
if snap not in find_snap_keys:
try:
raise Exception("Recoverable versions of {0} not found in dataset {1}".format(file, dataset))
except Exception as e:
print(zettaknight_utils.printcolors(e, "FAIL"))
path_list1 = find_snap[snap]
path_list = []
if os.path.isdir("/{0}/{1}".format(dataset, file)):
dir_flag = True
for line in path_list1:
if line.startswith("-") or line.startswith("M"):
a, b = line.rsplit("/", 1)
if not dir_flag:
try:
if str(b) == str(last_ref):
path_list.append(line)
except:
if str(b) == str(file):
path_list.append(line)
pass
else:
if a.startswith("-"):
a, b = line.rsplit(dataset, 2)
if os.path.isdir("{0}/{1}{2}".format(snap_basedir, snap_date, b)):
dir_flag = True
path_list.append(line)
if len(path_list) > 1:
exc_out = ""
for i in path_list:
if exc_out:
exc_out = "{0}\n\t{1}".format(exc_out, i)
else:
exc_out = str(i)
err_msg = "Matching files/folders in snapshot:"
print(zettaknight_utils.printcolors("\nAmbiguous recover request. Multiple matches for {0}.".format(file), "FAIL"))
print("{0} {1}".format(zettaknight_utils.printcolors(err_msg, "FAIL"), zettaknight_utils.printcolors(snap, "OKBLUE")))
print("\t{0}".format(zettaknight_utils.printcolors(exc_out, "WARNING")))
print(zettaknight_utils.printcolors("Re-run with explicit path to file.\n", "FAIL"))
try:
raise Exception("Ambiguous file reference.")
except Exception as e:
print(zettaknight_utils.printcolors(e, "FAIL"))
sys.exit(0)
if len(path_list) < 1:
try:
raise Exception("No restorable files or folders identified. \nRe-run with explicit path to file?")
except Exception as e:
print(zettaknight_utils.printcolors(e, "FAIL"))
sys.exit(0)
a, p = path_list[0].split(dataset, 1)
path, dirs = p.split(file, 1)
if dir_flag:
bad, dir_path = path_list[0].split(file, 1)
list_dirs = dir_path.rsplit("/", 1)
try:
out_dir = list_dirs[0]
except:
out_dir = ""
pass
file_loc = "{0}/{1}/{2}{3}/".format(snap_basedir, snap_date, file, dir_path )
mkdir_cmd = "/bin/mkdir {0}/{1}.R".format(active_basedir, file)
mkdir_run = zettaknight_utils.spawn_job(mkdir_cmd)
mkdir_cmd = "/bin/mkdir -p {0}/{1}.R/{2}".format(active_basedir, file, dir_path)
mkdir_run = zettaknight_utils.spawn_job(mkdir_cmd)
out_path = "{0}{1}{2}.R{3}/".format(active_basedir, str(path), file, out_dir)
else:
file_loc = "{0}/{1}/{2}".format(snap_basedir, snap_date, p)
out_path = "{0}{1}{2}.R".format(active_basedir, str(path), file)
if relocate:
out_path = relocate
rec_cmd = "/bin/cp ""-r ""{0}"" ""{1}".format(file_loc, out_path)
rec_run = zettaknight_utils.spawn_job(rec_cmd)
rec_dict = eval(str(rec_run))
for k,v in rec_dict.iteritems():
if str(k) is "0":
print("Recover operation succeeded.")
print("\tFilename: {0}".format(zettaknight_utils.printcolors(file, "OKBLUE")))
print("\t\tFile(s) restored to version: {0}".format(zettaknight_utils.printcolors(snap_date, "OKGREEN")))
print("\t\tFile(s) restored to: {0}".format(zettaknight_utils.printcolors(out_path, "OKGREEN")))
if str(k) is not "0":
print("Recover operation failed.")
print("\tFilename: {0}".format(zettaknight_utils.printcolors(file, "OKBLUE")))
print("\t\tRestore failed with error: {0}".format(zettaknight_utils.printcolors(v, "FAIL")))
return
def find_versions(dataset, filename, quiet=False):
'''
'''
zettaknight_utils.check_quiet(quiet)
snaps = {}
ret = {}
ret[dataset] = {}
snaplist_cmd = "/sbin/zfs list -r -t snapshot -o name -H {0}".format(dataset)
snaplist_run = subprocess.Popen(shlex.split(snaplist_cmd), stdout = subprocess.PIPE, stderr = subprocess.STDOUT)
snaplist_run.wait()
snaplist_out = snaplist_run.stdout.read()
if not snaplist_out:
try:
out_dict = {}
out_dict[dataset] = {}
job = inspect.stack()[0][3]
if str(inspect.stack()[1][3]) is 'recover':
job = inspect.stack()[1][3]
out_dict[dataset][job] = {}
out_dict[dataset][job]['1'] = "No snapshots found."
raise Exception
except Exception as e:
zettaknight_utils.parse_output(out_dict)
sys.exit(0)
for snap in snaplist_out.split():
if snap.startswith("cannot"):
try:
raise Exception("{0}".format(snaplist_out))
except Exception as e:
print(zettaknight_utils.printcolors(e, "FAIL"))
sys.exit(0)
snapdiff_cmd = "/sbin/zfs diff {0}".format(snap)
gerp_cmd = "/bin/grep {0}".format(filename)
gerp_run = zettaknight_utils.pipe_this(snapdiff_cmd, gerp_cmd)
gerp_out = gerp_run.stdout.read()
gerp_list = []
if gerp_out:
for gerp in gerp_out.split('\n'):
if gerp.startswith("-") or gerp.startswith("M"):
gerp_list.append(gerp)
if gerp_list:
snaps[snap] = gerp_list
gerp_msg = ""
for z in gerp_list:
if gerp_msg:
gerp_msg = "{0}\n{1}".format(gerp_msg, z)
else:
gerp_msg = str(z)
job = "Snapshot: {0}".format(snap)
ret[dataset][job] = {}
job_out = "Path:\n{0}".format(gerp_msg)
ret[dataset][job]['0'] = job_out
if not ret[dataset]:
ret[dataset]['Snapshot'] = {}
ret[dataset]['Snapshot']['1'] = "No modified versions of {0} found.".format(filename, dataset)
if not quiet:
zettaknight_utils.parse_output(ret)
return snaps
def recover(snapshot, filename, relocate=None):
'''
'''
if zettaknight_globs.help_flag:
ret = """Recover:
Usage:
zettaknight recover <snapshot> <filepath> (<directory to recover file to>)*optional
Recovers a previous version of a file or folder from a specified snapshot. Information to use in calling zettaknight recover can be found in the output from the find_versions function. By default, files/folders will be recovered to their original location with .R appended to the end of the name.
Required Arguments:
snapshot
Specifies the snapshot to recover files from.
filename
Defines target file(s) to recover. This should be the full path to a file or folder (/zfs_data/<some dataset>/<some filename>.<ext>)
Optional Arguments:
relocate
Defines an alternate path to recover files/folders to. If used, .R will not be appended, and the recover will overwrite any existing files."""
return ret
file = filename
snap = snapshot
dataset, snap_date = snap.split('@', 1)
find_snap = find_versions(dataset, file, "quiet")
find_snap_keys = find_snap.iterkeys()
snap_basedir = "/{0}/.zfs/snapshot".format(dataset)
active_basedir = "/{0}".format(dataset)
dir_flag = False
ret = {}
ret[dataset] = {}
try:
if str(dataset) in str(file):
file = file.split(dataset, 1)[1]
last_ref = file.rsplit("/", 1)[1]
except Exception as e:
pass
if dataset in file.rsplit("/", 1):
file = file.split(dataset, 1)[1]
last_ref = file.rsplit("/",1)[1]
if snap not in find_snap_keys:
try:
raise Exception("Recoverable versions of {0} not found in dataset {1}".format(file, dataset))
except Exception as e:
zettaknight_utils.zlog("{0}".format(e), "ERROR")
path_list1 = find_snap[snap]
path_list = []
if os.path.isdir("/{0}{1}".format(dataset, file)):
dir_flag = True
for line in path_list1:
if line.startswith("-") or line.startswith("M") or line.startswith("R"):
a, b = line.rsplit("/", 1)
if not dir_flag:
try:
if str(b) == str(last_ref):
path_list.append(line)
except:
if str(b) == str(file):
path_list.append(line)
pass
else:
if a.startswith("-") or line.startswith("M") or line.startswith("R"):
a, b = line.rsplit(dataset, 2)
if os.path.isdir("{0}/{1}{2}".format(snap_basedir, snap_date, b)):
dir_flag = True
path_list.append(line)
if len(path_list) > 1:
exc_out = ""
for i in path_list:
if exc_out:
exc_out = "{0}\n\t{1}".format(exc_out, i)
else:
exc_out = str(i)
err_msg = "Matching files/folders in snapshot:"
print(zettaknight_utils.printcolors("\nAmbiguous recover request. Multiple matches for {0}.".format(file), "FAIL"))
print("{0} {1}".format(zettaknight_utils.printcolors(err_msg, "FAIL"), zettaknight_utils.printcolors(snap, "OKBLUE")))
print("\t{0}".format(zettaknight_utils.printcolors(exc_out, "WARNING")))
print(zettaknight_utils.printcolors("Re-run with explicit path to file.\n", "FAIL"))
try:
raise Exception("Ambiguous file reference.")
except Exception as e:
zettaknight_utils.zlog("{0}".format(e), "ERROR")
sys.exit(0)
if len(path_list) < 1:
try:
raise Exception("No restorable files or folders identified. \nRe-run with explicit path to file?")
except Exception as e:
zettaknight_utils.zlog("{0}".format(e), "ERROR")
sys.exit(0)
a, p = path_list[0].split(dataset, 1)
path, dirs = p.split(file, 1)
if dir_flag:
a, b = path_list[0].rsplit(dataset, 2)
file_loc = "{0}/{1}{2}".format(snap_basedir, snap_date, b)
#bad, dir_path = path_list[0].split(file, 1)
#list_dirs = dir_path.rsplit("/", 1)
#try:
# out_dir = list_dirs[0]
#except:
out_dir = ""
# pass
#file_loc = "{0}/{1}/{2}{3}/".format(snap_basedir, snap_date, file, dir_path )
#mkdir_cmd = "/bin/mkdir {0}/{1}.R".format(active_basedir, file)
#mkdir_run = zettaknight_utils.spawn_job(mkdir_cmd)
#mkdir_cmd = "/bin/mkdir -p {0}/{1}.R/{2}".format(active_basedir, file, dir_path)
#mkdir_run = zettaknight_utils.spawn_job(mkdir_cmd)
out_path = "{0}{1}{2}.R{3}/".format(active_basedir, str(path), file, out_dir)
else:
file_loc = "{0}/{1}/{2}".format(snap_basedir, snap_date, p)
out_path = "{0}{1}{2}.R".format(active_basedir, str(path), file)
if relocate:
out_path = relocate
rec_cmd = "/bin/cp ""-r -p ""{0}"" ""{1}".format(file_loc, out_path)
rec_run = zettaknight_utils.spawn_job(rec_cmd)
rec_dict = eval(str(rec_run))
for k,v in rec_dict.iteritems():
if str(k) is "0":
print("Recover operation succeeded.")
print("\tFilename: {0}".format(zettaknight_utils.printcolors(file, "OKBLUE")))
print("\t\tFile(s) restored to version: {0}".format(zettaknight_utils.printcolors(snap_date, "OKGREEN")))
print("\t\tFile(s) restored to: {0}".format(zettaknight_utils.printcolors(out_path, "OKGREEN")))
if str(k) is not "0":
print("Recover operation failed.")
print("\tFilename: {0}".format(zettaknight_utils.printcolors(file, "OKBLUE")))
print("\t\tRestore failed with error: {0}".format(zettaknight_utils.printcolors(v, "FAIL")))
return
def recover(snapshot, filename, relocate=None):
'''
'''
if zettaknight_globs.help_flag:
ret = """Recover:
Usage:
zettaknight recover <snapshot> <filepath> (<directory to recover file to>)*optional
Recovers a previous version of a file or folder from a specified snapshot. Information to use in calling zettaknight recover can be found in the output from the find_versions function. By default, files/folders will be recovered to their original location with .R appended to the end of the name.
Required Arguments:
snapshot
Specifies the snapshot to recover files from.
filename
Defines target file(s) to recover. This should be the full path to a file or folder (/zfs_data/<some dataset>/<some filename>.<ext>)
Optional Arguments:
relocate
Defines an alternate path to recover files/folders to. If used, .R will not be appended, and the recover will overwrite any existing files."""
return ret
file = filename
snap = snapshot
dataset, snap_date = snap.split('@', 1)
find_snap = find_versions(dataset, file, "quiet")
find_snap_keys = find_snap.iterkeys()
snap_basedir = "/{0}/.zfs/snapshot".format(dataset)
active_basedir = "/{0}".format(dataset)
dir_flag = False
ret = {}
ret[dataset] = {}
try:
if str(dataset) in str(file):
file = file.split(dataset, 1)[1]
last_ref = file.rsplit("/", 1)[1]
except Exception as e:
pass
if dataset in file.rsplit("/", 1):
file = file.split(dataset, 1)[1]
last_ref = file.rsplit("/", 1)[1]
if snap not in find_snap_keys:
try:
raise Exception(
"Recoverable versions of {0} not found in dataset {1}".format(
file, dataset))
except Exception as e:
zettaknight_utils.zlog("{0}".format(e), "ERROR")
path_list1 = find_snap[snap]
path_list = []
if os.path.isdir("/{0}{1}".format(dataset, file)):
dir_flag = True
for line in path_list1:
if line.startswith("-") or line.startswith("M") or line.startswith(
"R"):
a, b = line.rsplit("/", 1)
if not dir_flag:
try:
if str(b) == str(last_ref):
path_list.append(line)
except:
if str(b) == str(file):
path_list.append(line)
pass
else:
if a.startswith("-") or line.startswith(
"M") or line.startswith("R"):
a, b = line.rsplit(dataset, 2)
if os.path.isdir("{0}/{1}{2}".format(
snap_basedir, snap_date, b)):
dir_flag = True
path_list.append(line)
if len(path_list) > 1:
exc_out = ""
for i in path_list:
if exc_out:
exc_out = "{0}\n\t{1}".format(exc_out, i)
else:
exc_out = str(i)
err_msg = "Matching files/folders in snapshot:"
print(
zettaknight_utils.printcolors(
"\nAmbiguous recover request. Multiple matches for {0}.".
format(file), "FAIL"))
print("{0} {1}".format(zettaknight_utils.printcolors(err_msg, "FAIL"),
zettaknight_utils.printcolors(snap, "OKBLUE")))
print("\t{0}".format(zettaknight_utils.printcolors(exc_out,
"WARNING")))
print(
zettaknight_utils.printcolors(
"Re-run with explicit path to file.\n", "FAIL"))
try:
raise Exception("Ambiguous file reference.")
except Exception as e:
zettaknight_utils.zlog("{0}".format(e), "ERROR")
sys.exit(0)
if len(path_list) < 1:
try:
raise Exception(
"No restorable files or folders identified. \nRe-run with explicit path to file?"
)
except Exception as e:
zettaknight_utils.zlog("{0}".format(e), "ERROR")
sys.exit(0)
a, p = path_list[0].split(dataset, 1)
path, dirs = p.split(file, 1)
if dir_flag:
a, b = path_list[0].rsplit(dataset, 2)
file_loc = "{0}/{1}{2}".format(snap_basedir, snap_date, b)
#bad, dir_path = path_list[0].split(file, 1)
#list_dirs = dir_path.rsplit("/", 1)
#try:
# out_dir = list_dirs[0]
#except:
out_dir = ""
# pass
#file_loc = "{0}/{1}/{2}{3}/".format(snap_basedir, snap_date, file, dir_path )
#mkdir_cmd = "/bin/mkdir {0}/{1}.R".format(active_basedir, file)
#mkdir_run = zettaknight_utils.spawn_job(mkdir_cmd)
#mkdir_cmd = "/bin/mkdir -p {0}/{1}.R/{2}".format(active_basedir, file, dir_path)
#mkdir_run = zettaknight_utils.spawn_job(mkdir_cmd)
out_path = "{0}{1}{2}.R{3}/".format(active_basedir, str(path), file,
out_dir)
else:
file_loc = "{0}/{1}/{2}".format(snap_basedir, snap_date, p)
out_path = "{0}{1}{2}.R".format(active_basedir, str(path), file)
if relocate:
out_path = relocate
rec_cmd = "/bin/cp " "-r -p " "{0}" " " "{1}".format(file_loc, out_path)
rec_run = zettaknight_utils.spawn_job(rec_cmd)
rec_dict = eval(str(rec_run))
for k, v in rec_dict.iteritems():
if str(k) is "0":
print("Recover operation succeeded.")
print("\tFilename: {0}".format(
zettaknight_utils.printcolors(file, "OKBLUE")))
print("\t\tFile(s) restored to version: {0}".format(
zettaknight_utils.printcolors(snap_date, "OKGREEN")))
print("\t\tFile(s) restored to: {0}".format(
zettaknight_utils.printcolors(out_path, "OKGREEN")))
if str(k) is not "0":
print("Recover operation failed.")
print("\tFilename: {0}".format(
zettaknight_utils.printcolors(file, "OKBLUE")))
print("\t\tRestore failed with error: {0}".format(
zettaknight_utils.printcolors(v, "FAIL")))
return
def first_step():
ret = {}
arg_dict = {}
try:
pool_name = "zfs_data" #default pool name
arg_dict["pool_name"] = pool_name
ret[pool_name] = {}
############### creation of the zpool ################
######################################################
#test if a zpool exists
zpool_list_cmd = "/sbin/zpool list -H"
zpool_list_run = zettaknight_utils.spawn_job(zpool_list_cmd)
chk_code, chk_msg = zpool_list_run.popitem()
if int(chk_code) != 0: #if zpool list -H returns empty
#query_return_item is defined in zettaknight_utils
create_pool_question = zettaknight_utils.query_yes_no(
"No pool detected, would you like to create one?")
if create_pool_question:
arg_dict["disk_list"] = zettaknight_utils.query_return_item(
"File containing all list of disks to be used for {0}, I.E /tmp/bunchofdisks.txt: "
.format(pool_name))
arg_dict["raid"] = zettaknight_utils.query_return_item(
"Raid level to be used [1-3]. I.E. 4+1, 5+2, 1+1: ")
arg_dict["ashift"] = zettaknight_utils.query_return_item(
"ashift value for {0}, value is 9 for 512 block disks and 12 for 4096 block disks: "
.format(pool_name))
arg_dict["luks"] = zettaknight_utils.query_yes_no(
"encrypt data disks with LUKS? ")
print("creating zpool: {0}".format(pool_name))
create_output = zettaknight_zpools.create(
pool_name, **arg_dict) #from zettaknight_zpools
#ret[pool_name]['Zpool Create'] = create_output[pool_name]['Zpool Create'] #'NoneType' object has no attribute '__getitem__'
else:
print("a pool already exists, moving on")
######################################################
######################################################
################## ssh key creation #################
print("checking for {0}".format(zettaknight_globs.identity_file))
if not os.path.isfile(zettaknight_globs.identity_file):
print(
zettaknight_utils.printcolors(
"zettaknight id file not found, generating.", "WARNING"))
keygen_output = zettaknight_utils.ssh_keygen(
zettaknight_globs.identity_file)
ret[pool_name]['Generate SSH Keygen'] = keygen_output[
zettaknight_globs.fqdn]['Generate SSH Key']
else:
print("{0} exists, moving on".format(
zettaknight_globs.identity_file))
######################################################
############# backup luks headers ####################
print("backing up luks headers for {0}".format(pool_name))
luks_backup_output = zettaknight_utils.backup_luks_headers()
ret[pool_name]['Backup Luks Headers'] = luks_backup_output[
zettaknight_globs.fqdn]['Backup Luks Headers']
######################################################
######## create zettaknight configuration file #######
if not os.path.isdir(zettaknight_globs.conf_dir_new
): #if new directory does not exists, make it
print("creating {0}".format(zettaknight_globs.conf_dir_new))
os.mkdir(zettaknight_globs.conf_dir_new)
######################################################
########### create zettaknight store #################
######################################################
if not os.path.isdir("/{0}".format(
zettaknight_globs.zettaknight_store)):
dataset_check_run = zettaknight_utils.pipe_this2(
"/sbin/zfs list -H | grep {0}".format(
zettaknight_globs.zettaknight_store))
if int(dataset_check_run.returncode) != 0:
print("creating configuration file for {0}".format(
zettaknight_globs.zettaknight_store))
zettaknight_utils.create_config(
dataset=zettaknight_globs.zettaknight_store)
gerp_run = zettaknight_utils.pipe_this2(
"/sbin/zfs list -H | /bin/grep {0}".format(
zettaknight_globs.zettaknight_store))
if int(gerp_run.returncode) is not 0:
zettaknight_zfs.add_dataset(zettaknight_globs.zettaknight_store)
#backup files to store
files = [
"/etc/exports", "/etc/crypttab",
"{0}".format(zettaknight_globs.config_file_new)
]
for file in files:
if os.path.exists(file):
destination_dir = "/{0}/{1}".format(
zettaknight_globs.zettaknight_store, zettaknight_globs.fqdn
) #add leading slash, zfs_share defined
filename = file.replace(
"/", ""
) #remove illegal characters from file path and save file as the concatenated version
if not os.path.isdir(destination_dir):
os.mkdir(destination_dir)
print("backing up {0} to {1}".format(file, destination_dir))
shutil.copyfile(file, "{0}/{1}".format(destination_dir,
filename))
######################################################
######################################################
############ create the first dataset ################
dataset = zettaknight_utils.query_return_list(
"Datasets to be created on {0}: ".format(pool_name))
for item in dataset:
dset_args = []
dataset_full = "{0}/{1}".format(pool_name, item)
dset_args.append(dataset_full)
dset_args.append("create_config=True")
add_dset_output = zettaknight_zfs.add_dataset(
*dset_args) #from zettaknight_utils
ret[pool_name]['Add dataset {0}'.format(
dataset_full)] = add_dset_output[dataset_full]['add_dataset']
######################################################
###### create server to server transfer pathing ######
replication_output = zettaknight_zfs.configure_replication(
) #from zettaknight_zfs
for repl_job, repl_job_output in replication_output[
dataset_full].itervalues():
ret[pool_name]['{0}'.format(repl_job)] = repl_job_output
######################################################
except Exception as e:
print(zettaknight_utils.printcolors(e, "FAIL"))
sys.exit(1)
zettaknight_globs.zfs_conf = _get_conf()
return ret
def check_usage(dset=False, quiet=False):
'''
'''
ret = {}
zettaknight_utils.check_quiet(quiet)
if dset and str(dset) not in zettaknight_globs.zfs_conf.iterkeys():
ret[dset] = {}
ret[dset]['Check Usage'] = {1: "{0} is not a Zettaknight controlled dataset.".format(dset)}
zettaknight_utils.parse_output(ret)
return ret
for dataset in zettaknight_globs.zfs_conf.iterkeys():
if dset:
if str(dset) != str(dataset):
continue
ret[dataset] = {}
ret[dataset]['Check Usage'] = {}
quota = zettaknight_globs.zfs_conf[dataset]['quota']
reservation = zettaknight_globs.zfs_conf[dataset]['reservation']
#find conversion multiplier to convert reservation to bytes
if 'G' in reservation:
res_x = float(1073741824)
res_unit = "G"
if 'T' in reservation:
res_x = float(1099511627776)
res_unit = "T"
if 'M' in reservation:
res_x = float(1048576)
res_unit = "M"
if str(reservation) == 'none':
res_x = float(1073741824)
res_unit = "G"
contact = zettaknight_globs.zfs_conf[dataset]['contact']
check_usage_cmd = "/sbin/zfs list -ro space {0} -H -p".format(dataset)
check_usage_run = zettaknight_utils.spawn_job(check_usage_cmd)
chk_code, chk_msg = check_usage_run.popitem()
chk_msg = re.sub("\n", "", chk_msg)
chk_msg_split = chk_msg.split("\t")
if not str(chk_code) == "0":
ret[dataset]['Check Usage']['1'] = "{0}Verify correct datasets are defined in configuration file\n".format(re.sub("[\[\"\]]", "", str(chk_msg_split)))
continue
if str(chk_msg_split[0]) == str(dataset):
avail = float(re.sub("[A-Za-z]", "", chk_msg_split[1])) #space available for dataset
used = float(re.sub("[A-Za-z]", "", chk_msg_split[2])) #how much space is used by the dataset
usnap = float(re.sub("[A-Za-z]", "", chk_msg_split[3])) #how much of the used space is consumed by snapshots
uds = float(re.sub("[A-Za-z]", "", chk_msg_split[4])) #how much of the space is consumed by the dataset itself
if str(reservation) != 'none':
res = (float(re.sub("[A-Za-z]", "", reservation)) * res_x)
avail_friendly = "{0:.2f}{1}".format((avail / res_x), res_unit)
used_friendly = "{0:.2f}{1}".format((used / res_x), res_unit)
usnap_friendly = "{0:.2f}{1}".format((usnap / res_x), res_unit)
uds_friendly = "{0:.2f}{1}".format((uds / res_x), res_unit)
if "e-" in str(used_friendly):
used_friendly = "{0}{1}".format(int(0), res_unit)
if "e-" in str(usnap_friendly):
usnap_friendly = "{0}{1}".format(int(0), res_unit)
if "e-" in str(uds_friendly):
uds_friendly = "{0}{1}".format(int(0), res_unit)
if str(reservation) != 'none' and used > res:
a = "{0}: {1}\n\t\t".format(zettaknight_utils.printcolors("Dataset", "OKBLUE"),zettaknight_utils.printcolors(dataset, "FAIL"))
b = "{0}: {1}\n{2}: {3}\n{4}: {5}\n{6}: {7}".format(zettaknight_utils.printcolors("Reservation", "OKBLUE"),zettaknight_utils.printcolors(reservation, "OKGREEN"),zettaknight_utils.printcolors("Total Used", "OKBLUE"),zettaknight_utils.printcolors(used_friendly, "WARNING"),zettaknight_utils.printcolors("Used by Snapshots", "OKBLUE"),zettaknight_utils.printcolors(usnap_friendly, "WARNING"),zettaknight_utils.printcolors("Active Dataset size", "OKBLUE"),zettaknight_utils.printcolors(uds_friendly, "WARNING"))
c = zettaknight_utils.printcolors("\nDataset exceeds space reservation", "WARNING")
msg = "{0}{1}{2}".format(a, b, c)
ret[dataset]['Check Usage']['1'] = "{0}{1}".format(b, c)
else:
a = "{0}: {1}\n\t\t".format(zettaknight_utils.printcolors("Dataset", "OKBLUE"),zettaknight_utils.printcolors(dataset, "OKGREEN"))
b = "{0}: {1}\n{2}: {3}\n{4}: {5}\n{6}: {7}".format(zettaknight_utils.printcolors("Reservation", "OKBLUE"),zettaknight_utils.printcolors(reservation, "OKGREEN"),zettaknight_utils.printcolors("Total Used", "OKBLUE"),zettaknight_utils.printcolors(used_friendly, "OKGREEN"),zettaknight_utils.printcolors("Used by Snapshots", "OKBLUE"),zettaknight_utils.printcolors(usnap_friendly, "OKGREEN"),zettaknight_utils.printcolors("Active Dataset size", "OKBLUE"),zettaknight_utils.printcolors(uds_friendly, "OKGREEN"))
msg = "{0}{1}".format(a, b)
ret[dataset]['Check Usage']['0'] = b
return ret
def rename_dataset(**kwargs):
'''
'''
import paramiko
try:
conf_file = zettaknight_globs.config_file_new
if not 'keyfile' in kwargs.iterkeys():
keyfile = zettaknight_globs.identity_file
if not 'dataset' in kwargs.iterkeys():
raise ValueError('A very specific bad thing happened')
#required argument, show_help
else:
dataset = kwargs['dataset']
if not 'new_dataset' in kwargs.iterkeys():
#required argument, show_help
raise ValueError('A very specific bad thing happened')
else:
new_dataset = kwargs['new_dataset']
if not 'user' in kwargs.iterkeys():
user = zettaknight_globs.zfs_conf[dataset]['user']
print("user is {0}".format(user))
remote_server = []
for r_server in zettaknight_globs.zfs_conf[dataset]['snap']['remote_server']:
remote_server.append(r_server)
print(remote_server)
except Exception as e:
print(zettaknight_utils.printcolors(e, "FAIL"))
sys.exit(1)
ret = {}
ret[zettaknight_globs.fqdn] = {}
ret[zettaknight_globs.fqdn]['rename {0}'.format(dataset)] = {}
try:
for r in remote_server:
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect(r, username=user, key_filename=keyfile)
remote_sudo_cmd = "zfs rename {0} {1}".format(dataset, new_dataset)
ssh_stdin, ssh_stdout, ssh_stderr = ssh.exec_command(remote_sudo_cmd)
print ssh_stdout.read()
#call function replace_string
replace_string(dataset, new_dataset, conf_file)
cmd = "zfs rename {0} {1}".format(dataset, new_dataset)
zettaknight_utils.spawn_job(cmd)
except Exception as e:
print(zettaknight_utils.printcolors(e, "FAIL"))
sys.exit(1)
ret[zettaknight_globs.fqdn]['rename {0}'.format(dataset)] = {0 : "successfully renamed {0} to {1}, all records have been updated".format(dataset, new_dataset)}
return ret
def check_usage(dset=False, quiet=False):
'''
'''
ret = {}
zettaknight_utils.check_quiet(quiet)
if dset and str(dset) not in zettaknight_globs.zfs_conf.iterkeys():
ret[dset] = {}
ret[dset]['Check Usage'] = {
1: "{0} is not a Zettaknight controlled dataset.".format(dset)
}
zettaknight_utils.parse_output(ret)
return ret
for dataset in zettaknight_globs.zfs_conf.iterkeys():
if dset:
if str(dset) != str(dataset):
continue
ret[dataset] = {}
ret[dataset]['Check Usage'] = {}
quota = zettaknight_globs.zfs_conf[dataset]['quota']
reservation = zettaknight_globs.zfs_conf[dataset]['reservation']
#find conversion multiplier to convert reservation to bytes
if 'G' in reservation:
res_x = float(1073741824)
res_unit = "G"
if 'T' in reservation:
res_x = float(1099511627776)
res_unit = "T"
if 'M' in reservation:
res_x = float(1048576)
res_unit = "M"
if str(reservation) == 'none':
res_x = float(1073741824)
res_unit = "G"
contact = zettaknight_globs.zfs_conf[dataset]['contact']
check_usage_cmd = "/sbin/zfs list -ro space {0} -H -p".format(dataset)
check_usage_run = zettaknight_utils.spawn_job(check_usage_cmd)
chk_code, chk_msg = check_usage_run.popitem()
chk_msg = re.sub("\n", "", chk_msg)
chk_msg_split = chk_msg.split("\t")
if not str(chk_code) == "0":
ret[dataset]['Check Usage'][
'1'] = "{0}Verify correct datasets are defined in configuration file\n".format(
re.sub("[\[\"\]]", "", str(chk_msg_split)))
continue
if str(chk_msg_split[0]) == str(dataset):
avail = float(
re.sub("[A-Za-z]", "",
chk_msg_split[1])) #space available for dataset
used = float(re.sub(
"[A-Za-z]", "",
chk_msg_split[2])) #how much space is used by the dataset
usnap = float(
re.sub("[A-Za-z]", "", chk_msg_split[3]
)) #how much of the used space is consumed by snapshots
uds = float(
re.sub("[A-Za-z]", "", chk_msg_split[4])
) #how much of the space is consumed by the dataset itself
if str(reservation) != 'none':
res = (float(re.sub("[A-Za-z]", "", reservation)) * res_x)
avail_friendly = "{0:.2f}{1}".format((avail / res_x), res_unit)
used_friendly = "{0:.2f}{1}".format((used / res_x), res_unit)
usnap_friendly = "{0:.2f}{1}".format((usnap / res_x), res_unit)
uds_friendly = "{0:.2f}{1}".format((uds / res_x), res_unit)
if "e-" in str(used_friendly):
used_friendly = "{0}{1}".format(int(0), res_unit)
if "e-" in str(usnap_friendly):
usnap_friendly = "{0}{1}".format(int(0), res_unit)
if "e-" in str(uds_friendly):
uds_friendly = "{0}{1}".format(int(0), res_unit)
if str(reservation) != 'none' and used > res:
a = "{0}: {1}\n\t\t".format(
zettaknight_utils.printcolors("Dataset", "OKBLUE"),
zettaknight_utils.printcolors(dataset, "FAIL"))
b = "{0}: {1}\n{2}: {3}\n{4}: {5}\n{6}: {7}".format(
zettaknight_utils.printcolors("Reservation", "OKBLUE"),
zettaknight_utils.printcolors(reservation, "OKGREEN"),
zettaknight_utils.printcolors("Total Used", "OKBLUE"),
zettaknight_utils.printcolors(used_friendly, "WARNING"),
zettaknight_utils.printcolors("Used by Snapshots",
"OKBLUE"),
zettaknight_utils.printcolors(usnap_friendly, "WARNING"),
zettaknight_utils.printcolors("Active Dataset size",
"OKBLUE"),
zettaknight_utils.printcolors(uds_friendly, "WARNING"))
c = zettaknight_utils.printcolors(
"\nDataset exceeds space reservation", "WARNING")
msg = "{0}{1}{2}".format(a, b, c)
ret[dataset]['Check Usage']['1'] = "{0}{1}".format(b, c)
else:
a = "{0}: {1}\n\t\t".format(
zettaknight_utils.printcolors("Dataset", "OKBLUE"),
zettaknight_utils.printcolors(dataset, "OKGREEN"))
b = "{0}: {1}\n{2}: {3}\n{4}: {5}\n{6}: {7}".format(
zettaknight_utils.printcolors("Reservation", "OKBLUE"),
zettaknight_utils.printcolors(reservation, "OKGREEN"),
zettaknight_utils.printcolors("Total Used", "OKBLUE"),
zettaknight_utils.printcolors(used_friendly, "OKGREEN"),
zettaknight_utils.printcolors("Used by Snapshots",
"OKBLUE"),
zettaknight_utils.printcolors(usnap_friendly, "OKGREEN"),
zettaknight_utils.printcolors("Active Dataset size",
"OKBLUE"),
zettaknight_utils.printcolors(uds_friendly, "OKGREEN"))
msg = "{0}{1}".format(a, b)
ret[dataset]['Check Usage']['0'] = b
return ret
def first_step():
ret = {}
arg_dict = {}
try:
pool_name = "zfs_data" #default pool name
arg_dict["pool_name"] = pool_name
ret[pool_name] = {}
############### creation of the zpool ################
######################################################
#test if a zpool exists
zpool_list_cmd = "/sbin/zpool list -H"
zpool_list_run = zettaknight_utils.spawn_job(zpool_list_cmd)
chk_code, chk_msg = zpool_list_run.popitem()
if int(chk_code) != 0: #if zpool list -H returns empty
#query_return_item is defined in zettaknight_utils
create_pool_question = zettaknight_utils.query_yes_no("No pool detected, would you like to create one?")
if create_pool_question:
arg_dict["disk_list"] = zettaknight_utils.query_return_item("File containing all list of disks to be used for {0}, I.E /tmp/bunchofdisks.txt: ".format(pool_name))
arg_dict["raid"] = zettaknight_utils.query_return_item("Raid level to be used [1-3]. I.E. 4+1, 5+2, 1+1: ")
arg_dict["ashift"] = zettaknight_utils.query_return_item("ashift value for {0}, value is 9 for 512 block disks and 12 for 4096 block disks: ".format(pool_name))
arg_dict["luks"] = zettaknight_utils.query_yes_no("encrypt data disks with LUKS? ")
print("creating zpool: {0}".format(pool_name))
create_output = zettaknight_zpools.create(pool_name, **arg_dict) #from zettaknight_zpools
#ret[pool_name]['Zpool Create'] = create_output[pool_name]['Zpool Create'] #'NoneType' object has no attribute '__getitem__'
else:
print("a pool already exists, moving on")
######################################################
######################################################
################## ssh key creation #################
print("checking for {0}".format(zettaknight_globs.identity_file))
if not os.path.isfile(zettaknight_globs.identity_file):
print(zettaknight_utils.printcolors("zettaknight id file not found, generating.", "WARNING"))
keygen_output = zettaknight_utils.ssh_keygen(zettaknight_globs.identity_file)
ret[pool_name]['Generate SSH Keygen'] = keygen_output[zettaknight_globs.fqdn]['Generate SSH Key']
else:
print("{0} exists, moving on".format(zettaknight_globs.identity_file))
######################################################
############# backup luks headers ####################
print("backing up luks headers for {0}".format(pool_name))
luks_backup_output = zettaknight_utils.backup_luks_headers()
ret[pool_name]['Backup Luks Headers'] = luks_backup_output[zettaknight_globs.fqdn]['Backup Luks Headers']
######################################################
######## create zettaknight configuration file #######
if not os.path.isdir(zettaknight_globs.conf_dir_new): #if new directory does not exists, make it
print("creating {0}".format(zettaknight_globs.conf_dir_new))
os.mkdir(zettaknight_globs.conf_dir_new)
######################################################
########### create zettaknight store #################
######################################################
if not os.path.isdir("/{0}".format(zettaknight_globs.zettaknight_store)):
dataset_check_run = zettaknight_utils.pipe_this2("/sbin/zfs list -H | grep {0}".format(zettaknight_globs.zettaknight_store))
if int(dataset_check_run.returncode) != 0:
print("creating configuration file for {0}".format(zettaknight_globs.zettaknight_store))
zettaknight_utils.create_config(dataset=zettaknight_globs.zettaknight_store)
gerp_run = zettaknight_utils.pipe_this2("/sbin/zfs list -H | /bin/grep {0}".format(zettaknight_globs.zettaknight_store))
if int(gerp_run.returncode) is not 0:
zettaknight_zfs.add_dataset(zettaknight_globs.zettaknight_store)
#backup files to store
files = ["/etc/exports", "/etc/crypttab", "{0}".format(zettaknight_globs.config_file_new)]
for file in files:
if os.path.exists(file):
destination_dir = "/{0}/{1}".format(zettaknight_globs.zettaknight_store, zettaknight_globs.fqdn) #add leading slash, zfs_share defined
filename = file.replace("/", "") #remove illegal characters from file path and save file as the concatenated version
if not os.path.isdir(destination_dir):
os.mkdir(destination_dir)
print("backing up {0} to {1}".format(file, destination_dir))
shutil.copyfile(file, "{0}/{1}".format(destination_dir, filename))
######################################################
######################################################
############ create the first dataset ################
dataset = zettaknight_utils.query_return_list("Datasets to be created on {0}: ".format(pool_name))
for item in dataset:
dset_args = []
dataset_full = "{0}/{1}".format(pool_name, item)
dset_args.append(dataset_full)
dset_args.append("create_config=True")
add_dset_output = zettaknight_zfs.add_dataset(*dset_args) #from zettaknight_utils
ret[pool_name]['Add dataset {0}'.format(dataset_full)] = add_dset_output[dataset_full]['add_dataset']
######################################################
###### create server to server transfer pathing ######
replication_output = zettaknight_zfs.configure_replication() #from zettaknight_zfs
for repl_job, repl_job_output in replication_output[dataset_full].itervalues():
ret[pool_name]['{0}'.format(repl_job)] = repl_job_output
######################################################
except Exception as e:
print(zettaknight_utils.printcolors(e, "FAIL"))
sys.exit(1)
zettaknight_globs.zfs_conf = _get_conf()
return ret
def create_zpool(pool=False,
disk_list=False,
raid=False,
luks=False,
slog=False,
create_config=False,
ldap=False,
recordsize=False,
ashift=False,
keyfile=False):
if zettaknight_globs.help_flag:
ret = """Create Zpool:
See help entry for create function.
zettaknight help create
"""
return ret
ret = {}
ret[pool] = {}
ret[pool]['Create Zpool'] = {}
if not raid:
raid = "12+2"
try:
disks, z_level = raid.split("+")
except Exception as e:
ret[pool]['Create Zpool'] = {
'1': "{0}\nargument raid must be in x+y format, i.e. 2+1".format(e)
}
zettaknight_utils.parse_output(ret)
sys.exit(0)
create_cmd = "bash {0} -d {1} -z {2}".format(
zettaknight_globs.zpool_create_script, disks, z_level)
if disk_list:
create_cmd = "{0} -f '{1}'".format(create_cmd, disk_list)
if pool:
create_cmd = "{0} -p '{1}'".format(create_cmd, pool)
if luks:
create_cmd = "{0} -l".format(create_cmd)
if slog:
create_cmd = "{0} -s '{1}'".format(create_cmd, slog)
if ldap:
create_cmd = "{0} -i".format(create_cmd)
if recordsize:
if any(i in recordsize for i in 'KM'):
create_cmd = "{0} -r {1}".format(create_cmd, recordsize)
else:
print(
zettaknight_utils.printcolors(
"Recordsize must be in number/unit format. ie. 1M, or 512K",
"FAIL"))
sys.exit(0)
if ashift:
create_cmd = "{0} -a {1}".format(create_cmd, ashift)
if keyfile:
create_cmd = "{0} -k {1}".format(create_cmd, keyfile)
try:
ret[pool]['Create Zpool'] = zettaknight_utils.spawn_job(create_cmd)
except Exception as e:
zettaknight_utils.zlog("{0}".format(e), "ERROR")
ret[pool]['Create Zpool']['1'] = e
return ret