def create_snap(dataset, quiet=False):
'''
'''
zettaknight_utils.check_quiet(quiet)
snap = "{0}@{1}".format(dataset, zettaknight_globs.today_date)
gerp_run = zettaknight_utils.pipe_this2("/sbin/zfs list -r -t snapshot -o name -H {0} | /bin/grep {1}".format(dataset, snap))
gerp_out = gerp_run.stdout.read()
if int(gerp_run.returncode) is not 0:
ret = zettaknight_utils.spawn_job("/sbin/zfs snapshot -r {0}".format(snap))
if int(gerp_run.returncode) == 0:
ret = {0: "Snapshot {0} already exists.".format(snap)}
for exit_status, output in ret.iteritems():
if str(exit_status) == "0" and str(output) == "Job succeeded":
ret[exit_status] = "Snapshot created: {0}".format(snap)
if not quiet:
snap_out = {}
snap_out[dataset] = {}
snap_out[dataset][inspect.stack()[0][3]] = ret
zettaknight_utils.parse_output(snap_out)
return ret
def create_snap(dataset, quiet=False):
'''
'''
zettaknight_utils.zlog("create_snap started", "DEBUG")
zettaknight_utils.check_quiet(quiet)
snap = "{0}@{1}".format(dataset, zettaknight_globs.today_date)
snap_list = zettaknight_utils.spawn_job(
"/sbin/zfs list -r -t snapshot -o name -H {0}".format(snap))
if int(0) not in snap_list.iterkeys():
ret = zettaknight_utils.spawn_job(
"/sbin/zfs snapshot -r {0}".format(snap))
else:
ret = {0: "Snapshot {0} already exists.".format(snap)}
zettaknight_utils.zlog("snapshot {0} already exists".format(snap),
"INFO")
for exit_status, output in ret.iteritems():
if str(exit_status) == "0" and str(output) == "Job succeeded":
ret[exit_status] = "Snapshot created: {0}".format(snap)
zettaknight_utils.zlog("snapshot created: {0}".format(snap),
"SUCCESS")
if not quiet:
snap_out = {}
snap_out[dataset] = {}
snap_out[dataset][inspect.stack()[0][3]] = ret
zettaknight_utils.parse_output(snap_out)
zettaknight_utils.zlog("create_snap exiting", "DEBUG")
return ret
def create_snap(dataset, quiet=False):
'''
'''
zettaknight_utils.zlog("create_snap started", "DEBUG")
zettaknight_utils.check_quiet(quiet)
snap = "{0}@{1}".format(dataset, zettaknight_globs.today_date)
snap_list = zettaknight_utils.spawn_job("/sbin/zfs list -r -t snapshot -o name -H {0}".format(snap))
if int(0) not in snap_list.iterkeys():
ret = zettaknight_utils.spawn_job("/sbin/zfs snapshot -r {0}".format(snap))
else:
ret = {0: "Snapshot {0} already exists.".format(snap)}
zettaknight_utils.zlog("snapshot {0} already exists".format(snap), "INFO")
for exit_status, output in ret.iteritems():
if str(exit_status) == "0" and str(output) == "Job succeeded":
ret[exit_status] = "Snapshot created: {0}".format(snap)
zettaknight_utils.zlog("snapshot created: {0}".format(snap), "SUCCESS")
if not quiet:
snap_out = {}
snap_out[dataset] = {}
snap_out[dataset][inspect.stack()[0][3]] = ret
zettaknight_utils.parse_output(snap_out)
zettaknight_utils.zlog("create_snap exiting", "DEBUG")
return ret
def set_reservation(dataset, reservation, quiet=False):
'''
'''
zettaknight_utils.check_quiet(quiet)
#reservation_cmd = "bash {0} -d {1} -r {2}".format(zettaknight_globs.quota_script,dataset,reservation)
reservation_cmd = "/sbin/zfs set reservation={0} {1}".format(reservation, dataset)
ret = {}
ret[dataset] = {}
ret[dataset]['Reservation'] = zettaknight_utils.spawn_job(reservation_cmd)
for exit_status, output in ret[dataset]['Reservation'].iteritems():
if "Job succeeded" in output:
ret[dataset]['Reservation'][exit_status] = "reservation set to {0}".format(reservation)
return ret
def set_reservation(dataset, reservation, quiet=False):
'''
'''
zettaknight_utils.check_quiet(quiet)
#reservation_cmd = "bash {0} -d {1} -r {2}".format(zettaknight_globs.quota_script,dataset,reservation)
reservation_cmd = "/sbin/zfs set reservation={0} {1}".format(
reservation, dataset)
ret = {}
if zettaknight_globs.help_flag:
ret = """Set Reservation:
Sets a ZFS reservation on provided dataset.
Usage:
zettaknight set_reservation <dataset> <reservation>
Required Arguments:
dataset
Specifies the dataset to set a reservation for.
reservation
Specifies the reservation to set. ie. 1T, 100G, etc."""
return ret
ret[dataset] = {}
ret[dataset]['Reservation'] = zettaknight_utils.spawn_job(reservation_cmd)
for exit_status, output in ret[dataset]['Reservation'].iteritems():
if "Job succeeded" in output:
ret[dataset]['Reservation'][
exit_status] = "reservation set to {0}".format(reservation)
return ret
def set_reservation(dataset, reservation, quiet=False):
'''
'''
zettaknight_utils.check_quiet(quiet)
#reservation_cmd = "bash {0} -d {1} -r {2}".format(zettaknight_globs.quota_script,dataset,reservation)
reservation_cmd = "/sbin/zfs set reservation={0} {1}".format(reservation, dataset)
ret = {}
if zettaknight_globs.help_flag:
ret = """Set Reservation:
Sets a ZFS reservation on provided dataset.
Usage:
zettaknight set_reservation <dataset> <reservation>
Required Arguments:
dataset
Specifies the dataset to set a reservation for.
reservation
Specifies the reservation to set. ie. 1T, 100G, etc."""
return ret
ret[dataset] = {}
ret[dataset]['Reservation'] = zettaknight_utils.spawn_job(reservation_cmd)
for exit_status, output in ret[dataset]['Reservation'].iteritems():
if "Job succeeded" in output:
ret[dataset]['Reservation'][exit_status] = "reservation set to {0}".format(reservation)
return ret
def find_versions(dataset, filename, quiet=False):
'''
'''
zettaknight_utils.check_quiet(quiet)
snaps = {}
ret = {}
ret[dataset] = {}
snaplist_cmd = "/sbin/zfs list -r -t snapshot -o name -H {0}".format(dataset)
snaplist_run = subprocess.Popen(shlex.split(snaplist_cmd), stdout = subprocess.PIPE, stderr = subprocess.STDOUT)
snaplist_run.wait()
snaplist_out = snaplist_run.stdout.read()
if not snaplist_out:
try:
out_dict = {}
out_dict[dataset] = {}
job = inspect.stack()[0][3]
if str(inspect.stack()[1][3]) is 'recover':
job = inspect.stack()[1][3]
out_dict[dataset][job] = {}
out_dict[dataset][job]['1'] = "No snapshots found."
raise Exception
except Exception as e:
zettaknight_utils.parse_output(out_dict)
sys.exit(0)
for snap in snaplist_out.split():
if snap.startswith("cannot"):
try:
raise Exception("{0}".format(snaplist_out))
except Exception as e:
print(zettaknight_utils.printcolors(e, "FAIL"))
sys.exit(0)
snapdiff_cmd = "/sbin/zfs diff {0}".format(snap)
gerp_cmd = "/bin/grep {0}".format(filename)
gerp_run = zettaknight_utils.pipe_this(snapdiff_cmd, gerp_cmd)
gerp_out = gerp_run.stdout.read()
gerp_list = []
if gerp_out:
for gerp in gerp_out.split('\n'):
if gerp.startswith("-") or gerp.startswith("M"):
gerp_list.append(gerp)
if gerp_list:
snaps[snap] = gerp_list
gerp_msg = ""
for z in gerp_list:
if gerp_msg:
gerp_msg = "{0}\n{1}".format(gerp_msg, z)
else:
gerp_msg = str(z)
job = "Snapshot: {0}".format(snap)
ret[dataset][job] = {}
job_out = "Path:\n{0}".format(gerp_msg)
ret[dataset][job]['0'] = job_out
if not ret[dataset]:
ret[dataset]['Snapshot'] = {}
ret[dataset]['Snapshot']['1'] = "No modified versions of {0} found.".format(filename, dataset)
if not quiet:
zettaknight_utils.parse_output(ret)
return snaps
def find_versions(dataset, filename, quiet=False):
'''
'''
if zettaknight_globs.help_flag:
ret = """Find Versions:
Usage:
zettaknight find_versions zfs_data/<some dataset> <some filename>
Searches snapshots of provided dataset for previous versions of filename.
Required Arguments:
dataset
Specifies the dataset whose snapshots will be searched.
filename
Defines target file(s) to find previous versions of. This can be a full path to a file (/zfs_data/<some dataset>/<some filename>.<ext>), just a filename with or without an extension (<some filename> or <some filename>.<ext>), or just an extension (.<ext>)"""
return ret
zettaknight_utils.check_quiet(quiet)
snaps = {}
ret = {}
ret[dataset] = {}
snaplist_cmd = "/sbin/zfs list -r -t snapshot -o name -H {0}".format(dataset)
snaplist_run = subprocess.Popen(shlex.split(snaplist_cmd), stdout = subprocess.PIPE, stderr = subprocess.STDOUT)
snaplist_run.wait()
snaplist_out = snaplist_run.stdout.read()
if not snaplist_out:
try:
out_dict = {}
out_dict[dataset] = {}
job = inspect.stack()[0][3]
if str(inspect.stack()[1][3]) is 'recover':
job = inspect.stack()[1][3]
out_dict[dataset][job] = {}
out_dict[dataset][job]['1'] = "No snapshots found."
raise Exception
except Exception as e:
zettaknight_utils.parse_output(out_dict)
sys.exit(0)
for snap in snaplist_out.split():
if snap.startswith("cannot"):
try:
raise Exception("{0}".format(snaplist_out))
except Exception as e:
zettaknight_utils.zlog("{0}".format(e), "ERROR")
sys.exit(0)
snapdiff_cmd = "/sbin/zfs diff {0}".format(snap)
gerp_cmd = "/bin/grep {0}".format(filename)
gerp_run = zettaknight_utils.pipe_this(snapdiff_cmd, gerp_cmd)
gerp_out = gerp_run.stdout.read()
gerp_list = []
if gerp_out:
for gerp in gerp_out.split('\n'):
if gerp.startswith("-") or gerp.startswith("M") or gerp.startswith("R"):
gerp_list.append(gerp)
if gerp_list:
snaps[snap] = gerp_list
gerp_msg = ""
for z in gerp_list:
if gerp_msg:
gerp_msg = "{0}\n{1}".format(gerp_msg, z)
else:
gerp_msg = str(z)
job = "Snapshot: {0}".format(snap)
ret[dataset][job] = {}
job_out = "Path:\n{0}".format(gerp_msg)
ret[dataset][job]['0'] = job_out
if not ret[dataset]:
ret[dataset]['Snapshot'] = {}
ret[dataset]['Snapshot']['1'] = "No modified versions of {0} found.".format(filename, dataset)
if not quiet:
zettaknight_utils.parse_output(ret)
return snaps
def find_versions(dataset, filename, quiet=False):
'''
'''
if zettaknight_globs.help_flag:
ret = """Find Versions:
Usage:
zettaknight find_versions zfs_data/<some dataset> <some filename>
Searches snapshots of provided dataset for previous versions of filename.
Required Arguments:
dataset
Specifies the dataset whose snapshots will be searched.
filename
Defines target file(s) to find previous versions of. This can be a full path to a file (/zfs_data/<some dataset>/<some filename>.<ext>), just a filename with or without an extension (<some filename> or <some filename>.<ext>), or just an extension (.<ext>)"""
return ret
zettaknight_utils.check_quiet(quiet)
snaps = {}
ret = {}
ret[dataset] = {}
snaplist_cmd = "/sbin/zfs list -r -t snapshot -o name -H {0}".format(
dataset)
snaplist_run = subprocess.Popen(shlex.split(snaplist_cmd),
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
snaplist_run.wait()
snaplist_out = snaplist_run.stdout.read()
if not snaplist_out:
try:
out_dict = {}
out_dict[dataset] = {}
job = inspect.stack()[0][3]
if str(inspect.stack()[1][3]) is 'recover':
job = inspect.stack()[1][3]
out_dict[dataset][job] = {}
out_dict[dataset][job]['1'] = "No snapshots found."
raise Exception
except Exception as e:
zettaknight_utils.parse_output(out_dict)
sys.exit(0)
for snap in snaplist_out.split():
if snap.startswith("cannot"):
try:
raise Exception("{0}".format(snaplist_out))
except Exception as e:
zettaknight_utils.zlog("{0}".format(e), "ERROR")
sys.exit(0)
snapdiff_cmd = "/sbin/zfs diff {0}".format(snap)
gerp_cmd = "/bin/grep {0}".format(filename)
gerp_run = zettaknight_utils.pipe_this(snapdiff_cmd, gerp_cmd)
gerp_out = gerp_run.stdout.read()
gerp_list = []
if gerp_out:
for gerp in gerp_out.split('\n'):
if gerp.startswith("-") or gerp.startswith(
"M") or gerp.startswith("R"):
gerp_list.append(gerp)
if gerp_list:
snaps[snap] = gerp_list
gerp_msg = ""
for z in gerp_list:
if gerp_msg:
gerp_msg = "{0}\n{1}".format(gerp_msg, z)
else:
gerp_msg = str(z)
job = "Snapshot: {0}".format(snap)
ret[dataset][job] = {}
job_out = "Path:\n{0}".format(gerp_msg)
ret[dataset][job]['0'] = job_out
if not ret[dataset]:
ret[dataset]['Snapshot'] = {}
ret[dataset]['Snapshot'][
'1'] = "No modified versions of {0} found.".format(
filename, dataset)
if not quiet:
zettaknight_utils.parse_output(ret)
return snaps
def check_usage(dset=False, quiet=False):
'''
'''
ret = {}
zettaknight_utils.check_quiet(quiet)
if dset and str(dset) not in zettaknight_globs.zfs_conf.iterkeys():
ret[dset] = {}
ret[dset]['Check Usage'] = {1: "{0} is not a Zettaknight controlled dataset.".format(dset)}
zettaknight_utils.parse_output(ret)
return ret
for dataset in zettaknight_globs.zfs_conf.iterkeys():
if dset:
if str(dset) != str(dataset):
continue
ret[dataset] = {}
ret[dataset]['Check Usage'] = {}
quota = zettaknight_globs.zfs_conf[dataset]['quota']
reservation = zettaknight_globs.zfs_conf[dataset]['reservation']
#find conversion multiplier to convert reservation to bytes
if 'G' in reservation:
res_x = float(1073741824)
res_unit = "G"
if 'T' in reservation:
res_x = float(1099511627776)
res_unit = "T"
if 'M' in reservation:
res_x = float(1048576)
res_unit = "M"
if str(reservation) == 'none':
res_x = float(1073741824)
res_unit = "G"
contact = zettaknight_globs.zfs_conf[dataset]['contact']
check_usage_cmd = "/sbin/zfs list -ro space {0} -H -p".format(dataset)
check_usage_run = zettaknight_utils.spawn_job(check_usage_cmd)
chk_code, chk_msg = check_usage_run.popitem()
chk_msg = re.sub("\n", "", chk_msg)
chk_msg_split = chk_msg.split("\t")
if not str(chk_code) == "0":
ret[dataset]['Check Usage']['1'] = "{0}Verify correct datasets are defined in configuration file\n".format(re.sub("[\[\"\]]", "", str(chk_msg_split)))
continue
if str(chk_msg_split[0]) == str(dataset):
avail = float(re.sub("[A-Za-z]", "", chk_msg_split[1])) #space available for dataset
used = float(re.sub("[A-Za-z]", "", chk_msg_split[2])) #how much space is used by the dataset
usnap = float(re.sub("[A-Za-z]", "", chk_msg_split[3])) #how much of the used space is consumed by snapshots
uds = float(re.sub("[A-Za-z]", "", chk_msg_split[4])) #how much of the space is consumed by the dataset itself
if str(reservation) != 'none':
res = (float(re.sub("[A-Za-z]", "", reservation)) * res_x)
avail_friendly = "{0:.2f}{1}".format((avail / res_x), res_unit)
used_friendly = "{0:.2f}{1}".format((used / res_x), res_unit)
usnap_friendly = "{0:.2f}{1}".format((usnap / res_x), res_unit)
uds_friendly = "{0:.2f}{1}".format((uds / res_x), res_unit)
if "e-" in str(used_friendly):
used_friendly = "{0}{1}".format(int(0), res_unit)
if "e-" in str(usnap_friendly):
usnap_friendly = "{0}{1}".format(int(0), res_unit)
if "e-" in str(uds_friendly):
uds_friendly = "{0}{1}".format(int(0), res_unit)
if str(reservation) != 'none' and used > res:
a = "{0}: {1}\n\t\t".format(zettaknight_utils.printcolors("Dataset", "OKBLUE"),zettaknight_utils.printcolors(dataset, "FAIL"))
b = "{0}: {1}\n{2}: {3}\n{4}: {5}\n{6}: {7}".format(zettaknight_utils.printcolors("Reservation", "OKBLUE"),zettaknight_utils.printcolors(reservation, "OKGREEN"),zettaknight_utils.printcolors("Total Used", "OKBLUE"),zettaknight_utils.printcolors(used_friendly, "WARNING"),zettaknight_utils.printcolors("Used by Snapshots", "OKBLUE"),zettaknight_utils.printcolors(usnap_friendly, "WARNING"),zettaknight_utils.printcolors("Active Dataset size", "OKBLUE"),zettaknight_utils.printcolors(uds_friendly, "WARNING"))
c = zettaknight_utils.printcolors("\nDataset exceeds space reservation", "WARNING")
msg = "{0}{1}{2}".format(a, b, c)
ret[dataset]['Check Usage']['1'] = "{0}{1}".format(b, c)
else:
a = "{0}: {1}\n\t\t".format(zettaknight_utils.printcolors("Dataset", "OKBLUE"),zettaknight_utils.printcolors(dataset, "OKGREEN"))
b = "{0}: {1}\n{2}: {3}\n{4}: {5}\n{6}: {7}".format(zettaknight_utils.printcolors("Reservation", "OKBLUE"),zettaknight_utils.printcolors(reservation, "OKGREEN"),zettaknight_utils.printcolors("Total Used", "OKBLUE"),zettaknight_utils.printcolors(used_friendly, "OKGREEN"),zettaknight_utils.printcolors("Used by Snapshots", "OKBLUE"),zettaknight_utils.printcolors(usnap_friendly, "OKGREEN"),zettaknight_utils.printcolors("Active Dataset size", "OKBLUE"),zettaknight_utils.printcolors(uds_friendly, "OKGREEN"))
msg = "{0}{1}".format(a, b)
ret[dataset]['Check Usage']['0'] = b
return ret
def check_usage(dset=False, quiet=False):
'''
'''
ret = {}
zettaknight_utils.check_quiet(quiet)
if dset and str(dset) not in zettaknight_globs.zfs_conf.iterkeys():
ret[dset] = {}
ret[dset]['Check Usage'] = {
1: "{0} is not a Zettaknight controlled dataset.".format(dset)
}
zettaknight_utils.parse_output(ret)
return ret
for dataset in zettaknight_globs.zfs_conf.iterkeys():
if dset:
if str(dset) != str(dataset):
continue
ret[dataset] = {}
ret[dataset]['Check Usage'] = {}
quota = zettaknight_globs.zfs_conf[dataset]['quota']
reservation = zettaknight_globs.zfs_conf[dataset]['reservation']
#find conversion multiplier to convert reservation to bytes
if 'G' in reservation:
res_x = float(1073741824)
res_unit = "G"
if 'T' in reservation:
res_x = float(1099511627776)
res_unit = "T"
if 'M' in reservation:
res_x = float(1048576)
res_unit = "M"
if str(reservation) == 'none':
res_x = float(1073741824)
res_unit = "G"
contact = zettaknight_globs.zfs_conf[dataset]['contact']
check_usage_cmd = "/sbin/zfs list -ro space {0} -H -p".format(dataset)
check_usage_run = zettaknight_utils.spawn_job(check_usage_cmd)
chk_code, chk_msg = check_usage_run.popitem()
chk_msg = re.sub("\n", "", chk_msg)
chk_msg_split = chk_msg.split("\t")
if not str(chk_code) == "0":
ret[dataset]['Check Usage'][
'1'] = "{0}Verify correct datasets are defined in configuration file\n".format(
re.sub("[\[\"\]]", "", str(chk_msg_split)))
continue
if str(chk_msg_split[0]) == str(dataset):
avail = float(
re.sub("[A-Za-z]", "",
chk_msg_split[1])) #space available for dataset
used = float(re.sub(
"[A-Za-z]", "",
chk_msg_split[2])) #how much space is used by the dataset
usnap = float(
re.sub("[A-Za-z]", "", chk_msg_split[3]
)) #how much of the used space is consumed by snapshots
uds = float(
re.sub("[A-Za-z]", "", chk_msg_split[4])
) #how much of the space is consumed by the dataset itself
if str(reservation) != 'none':
res = (float(re.sub("[A-Za-z]", "", reservation)) * res_x)
avail_friendly = "{0:.2f}{1}".format((avail / res_x), res_unit)
used_friendly = "{0:.2f}{1}".format((used / res_x), res_unit)
usnap_friendly = "{0:.2f}{1}".format((usnap / res_x), res_unit)
uds_friendly = "{0:.2f}{1}".format((uds / res_x), res_unit)
if "e-" in str(used_friendly):
used_friendly = "{0}{1}".format(int(0), res_unit)
if "e-" in str(usnap_friendly):
usnap_friendly = "{0}{1}".format(int(0), res_unit)
if "e-" in str(uds_friendly):
uds_friendly = "{0}{1}".format(int(0), res_unit)
if str(reservation) != 'none' and used > res:
a = "{0}: {1}\n\t\t".format(
zettaknight_utils.printcolors("Dataset", "OKBLUE"),
zettaknight_utils.printcolors(dataset, "FAIL"))
b = "{0}: {1}\n{2}: {3}\n{4}: {5}\n{6}: {7}".format(
zettaknight_utils.printcolors("Reservation", "OKBLUE"),
zettaknight_utils.printcolors(reservation, "OKGREEN"),
zettaknight_utils.printcolors("Total Used", "OKBLUE"),
zettaknight_utils.printcolors(used_friendly, "WARNING"),
zettaknight_utils.printcolors("Used by Snapshots",
"OKBLUE"),
zettaknight_utils.printcolors(usnap_friendly, "WARNING"),
zettaknight_utils.printcolors("Active Dataset size",
"OKBLUE"),
zettaknight_utils.printcolors(uds_friendly, "WARNING"))
c = zettaknight_utils.printcolors(
"\nDataset exceeds space reservation", "WARNING")
msg = "{0}{1}{2}".format(a, b, c)
ret[dataset]['Check Usage']['1'] = "{0}{1}".format(b, c)
else:
a = "{0}: {1}\n\t\t".format(
zettaknight_utils.printcolors("Dataset", "OKBLUE"),
zettaknight_utils.printcolors(dataset, "OKGREEN"))
b = "{0}: {1}\n{2}: {3}\n{4}: {5}\n{6}: {7}".format(
zettaknight_utils.printcolors("Reservation", "OKBLUE"),
zettaknight_utils.printcolors(reservation, "OKGREEN"),
zettaknight_utils.printcolors("Total Used", "OKBLUE"),
zettaknight_utils.printcolors(used_friendly, "OKGREEN"),
zettaknight_utils.printcolors("Used by Snapshots",
"OKBLUE"),
zettaknight_utils.printcolors(usnap_friendly, "OKGREEN"),
zettaknight_utils.printcolors("Active Dataset size",
"OKBLUE"),
zettaknight_utils.printcolors(uds_friendly, "OKGREEN"))
msg = "{0}{1}".format(a, b)
ret[dataset]['Check Usage']['0'] = b
return ret