def create_snap(dataset, quiet=False):
'''
'''
zettaknight_utils.zlog("create_snap started", "DEBUG")
zettaknight_utils.check_quiet(quiet)
snap = "{0}@{1}".format(dataset, zettaknight_globs.today_date)
snap_list = zettaknight_utils.spawn_job("/sbin/zfs list -r -t snapshot -o name -H {0}".format(snap))
if int(0) not in snap_list.iterkeys():
ret = zettaknight_utils.spawn_job("/sbin/zfs snapshot -r {0}".format(snap))
else:
ret = {0: "Snapshot {0} already exists.".format(snap)}
zettaknight_utils.zlog("snapshot {0} already exists".format(snap), "INFO")
for exit_status, output in ret.iteritems():
if str(exit_status) == "0" and str(output) == "Job succeeded":
ret[exit_status] = "Snapshot created: {0}".format(snap)
zettaknight_utils.zlog("snapshot created: {0}".format(snap), "SUCCESS")
if not quiet:
snap_out = {}
snap_out[dataset] = {}
snap_out[dataset][inspect.stack()[0][3]] = ret
zettaknight_utils.parse_output(snap_out)
zettaknight_utils.zlog("create_snap exiting", "DEBUG")
return ret
def create_snap(dataset, quiet=False):
'''
'''
zettaknight_utils.zlog("create_snap started", "DEBUG")
zettaknight_utils.check_quiet(quiet)
snap = "{0}@{1}".format(dataset, zettaknight_globs.today_date)
snap_list = zettaknight_utils.spawn_job(
"/sbin/zfs list -r -t snapshot -o name -H {0}".format(snap))
if int(0) not in snap_list.iterkeys():
ret = zettaknight_utils.spawn_job(
"/sbin/zfs snapshot -r {0}".format(snap))
else:
ret = {0: "Snapshot {0} already exists.".format(snap)}
zettaknight_utils.zlog("snapshot {0} already exists".format(snap),
"INFO")
for exit_status, output in ret.iteritems():
if str(exit_status) == "0" and str(output) == "Job succeeded":
ret[exit_status] = "Snapshot created: {0}".format(snap)
zettaknight_utils.zlog("snapshot created: {0}".format(snap),
"SUCCESS")
if not quiet:
snap_out = {}
snap_out[dataset] = {}
snap_out[dataset][inspect.stack()[0][3]] = ret
zettaknight_utils.parse_output(snap_out)
zettaknight_utils.zlog("create_snap exiting", "DEBUG")
return ret
def sync(*args, **kwargs):
'''
'''
zettaknight_utils.zlog("kwargs recieved by sync\n\t{0}".format(kwargs),
"DEBUG")
if zettaknight_globs.help_flag:
ret = """Sync:
Syncs snapshots to a remote server.
Usage:
zettaknight sync <dataset> <remote_ssh>
Required Arguments:
dataset
Specifies the dataset to sync.
remote_ssh
Specifies remote server to sync snapshots to."""
return ret
#if 'priority' in zettaknight_globs.zfs_conf[dataset]:
# priority = zettaknight_globs.zfs_conf[dataset]['priority']
# print(priority)
# if isinstance(priority, int):
# print("priority is an integer")
# os.nice(int(priority))
try:
if 'dataset' and 'remote_ssh' in kwargs.iterkeys():
sync_cmd = "bash {0} -d {1} -s {2}".format(
zettaknight_globs.sync_script, kwargs['dataset'],
kwargs['remote_ssh'])
else:
raise Exception(
"dataset and remote_ssh is are required kwargs for sync")
if 'identity_file' in kwargs.iterkeys():
sync_cmd = "{0} -i {1}".format(sync_cmd, kwargs['identity_file'])
if 'pull_snap' in kwargs.iterkeys():
if kwargs['pull_snap']:
sync_cmd = "{0} -p".format(sync_cmd)
if 'priority' in kwargs.iterkeys():
sync_cmd = "{0} -n {1}".format(sync_cmd, kwargs['priority'])
if str(inspect.stack()[1][3]) is 'sync_all':
zettaknight_utils.zlog(
"starting sync job:\n\t{0}".format(sync_cmd), "INFO")
ret = zettaknight_utils.spawn_job(sync_cmd)
else:
ret = {}
ret[dataset] = {}
ret[dataset]['Snapshot sync'] = zettaknight_utils.spawn_job(
sync_cmd)
except Exception as e:
zettaknight_utils.zlog("{0}".format(e), "CRITICAL")
sys.exit(1)
return ret
def build_out_config(force=False):
'''
This function reads in the pool config file and creates the zfs data structures defined in it
'''
ret = {}
ret[zettaknight_globs.fqdn] = {}
ret[zettaknight_globs.fqdn]['Build Config'] = {}
create_config = {}
create_config['create_config'] = False
create_config['nfs'] = False
try:
#create pools defined in zpool conf
if zettaknight_globs.zpool_conf:
for zpool in zettaknight_globs.zpool_conf.iterkeys():
zettaknight_utils.zlog(
"determining if zpool {0} exists:\n\t[build_out_config] --> spawn_job : /sbin/zpool list -H {0}"
.format(zpool), "DEBUG")
d = zettaknight_utils.spawn_job(
"/sbin/zpool list -H '{0}'".format(zpool))
chk_code, chk_msg = d.popitem()
if int(chk_code) is not 0:
ret[zettaknight_globs.fqdn]['Create {0}'.format(
zpool)] = {}
zettaknight_utils.zlog(
"creating {0}:\n\t[build_out_config] --> zettaknight_zpool.create_zpool"
.format(zpool), "DEBUG")
out1 = zettaknight_zpool.create_zpool(
zpool, **zettaknight_globs.zpool_conf[zpool])
ret[zettaknight_globs.fqdn]['Create {0}'.format(
zpool)] = out1[zpool]['Create Zpool']
#print(ret)
#create datasets defined in zfs_conf
for dataset in zettaknight_globs.zfs_conf.iterkeys():
zettaknight_utils.zlog(
"determining if dataset {0} exists:\n\t[build_out_config] --> spawn_job : /sbin/zfs list -H {0}"
.format(dataset), "DEBUG")
d = zettaknight_utils.spawn_job(
"/sbin/zfs list -H '{0}'".format(dataset))
chk_code, chk_msg = d.popitem()
if int(chk_code) is not 0:
zettaknight_utils.zlog(
"creating {0}:\n\t[build_out_config] --> add_dataset".
format(dataset), "DEBUG")
out2 = add_dataset(dataset, **create_config)
ret[zettaknight_globs.fqdn]['Create dataset {0}'.format(
dataset)] = out2
ret[zettaknight_globs.
fqdn]['Build Config']['0'] = "Everything Looks Okay Here"
except Exception as e:
zettaknight_utils.zlog("{0}".format(e), "ERROR")
ret[zettaknight_globs.fqdn]['Build Config']['1'] = e
zettaknight_utils.zlog("ret for [build_out_config]:\n\t{0}".format(ret),
"DEBUG")
return ret
def sync(*args, **kwargs):
'''
'''
zettaknight_utils.zlog("kwargs recieved by sync\n\t{0}".format(kwargs), "DEBUG")
if zettaknight_globs.help_flag:
ret = """Sync:
Syncs snapshots to a remote server.
Usage:
zettaknight sync <dataset> <remote_ssh>
Required Arguments:
dataset
Specifies the dataset to sync.
remote_ssh
Specifies remote server to sync snapshots to."""
return ret
#if 'priority' in zettaknight_globs.zfs_conf[dataset]:
# priority = zettaknight_globs.zfs_conf[dataset]['priority']
# print(priority)
# if isinstance(priority, int):
# print("priority is an integer")
# os.nice(int(priority))
try:
if 'dataset' and 'remote_ssh' in kwargs.iterkeys():
sync_cmd = "bash {0} -d {1} -s {2}".format(zettaknight_globs.sync_script, kwargs['dataset'], kwargs['remote_ssh'])
else:
raise Exception("dataset and remote_ssh is are required kwargs for sync")
if 'identity_file' in kwargs.iterkeys():
sync_cmd = "{0} -i {1}".format(sync_cmd, kwargs['identity_file'])
if 'pull_snap' in kwargs.iterkeys():
if kwargs['pull_snap']:
sync_cmd = "{0} -p".format(sync_cmd)
if 'priority' in kwargs.iterkeys():
sync_cmd = "{0} -n {1}".format(sync_cmd, kwargs['priority'])
if str(inspect.stack()[1][3]) is 'sync_all':
zettaknight_utils.zlog("starting sync job:\n\t{0}".format(sync_cmd),"INFO")
ret = zettaknight_utils.spawn_job(sync_cmd)
else:
ret = {}
ret[dataset] = {}
ret[dataset]['Snapshot sync'] = zettaknight_utils.spawn_job(sync_cmd)
except Exception as e:
zettaknight_utils.zlog("{0}".format(e),"CRITICAL")
sys.exit(1)
return ret
def nuke(pool, force=True):
'''
'''
ret = {}
if zettaknight_globs.help_flag:
ret = """Nuke:
Destroys a zpool and cleans up files associated with the zpool being destroyed.
Usage:
zettaknight nuke <pool_name>
Required Arguments:
pool_name
Specifies the zpool to destroy."""
return ret
ret[pool] = {}
ret[pool]['Nuke Zpool'] = {}
nuke_cmd = "bash {0} -p '{1}'".format(zettaknight_globs.zfs_nuke_script, pool)
if force:
nuke_cmd = "{0} -f".format(nuke_cmd)
ret[pool]['Nuke Zpool'] = zettaknight_utils.spawn_job(nuke_cmd)
#zettaknight_utils.parse_output(ret)
return ret
def set_refquota(dataset, refquota):
'''
'''
refquota_cmd = "/sbin/zfs set refquota={0} {1}".format(refquota, dataset)
ret = {}
if zettaknight_globs.help_flag:
ret = """Set RefQuota:
Sets a ZFS refquota on provided dataset.
Usage:
zettaknight set_refquota <dataset> <refquota>
Required Arguments:
dataset
Specifies the dataset to set a quota for.
refquota
Specifies the quota to set. ie. 1T, 100G, etc."""
return ret
ret[dataset] = {}
ret[dataset]['refquota'] = zettaknight_utils.spawn_job(refquota_cmd)
for exit_status, output in ret[dataset]['refquota'].iteritems():
if "Job succeeded" in output:
ret[dataset]['refquota'][exit_status] = "refquota set to {0}".format(refquota)
return ret
def set_quota(dataset, quota):
'''
'''
#quota_cmd = "bash {0} -d {1} -q {2}".format(zettaknight_globs.quota_script,dataset,quota)
quota_cmd = "/sbin/zfs set quota={0} {1}".format(quota, dataset)
ret = {}
if zettaknight_globs.help_flag:
ret = """Set Quota:
Sets a ZFS quota on provided dataset.
Usage:
zettaknight set_quota <dataset> <quota>
Required Arguments:
dataset
Specifies the dataset to set a quota for.
quota
Specifies the quota to set. ie. 1T, 100G, etc."""
return ret
ret[dataset] = {}
ret[dataset]['Quota'] = zettaknight_utils.spawn_job(quota_cmd)
for exit_status, output in ret[dataset]['Quota'].iteritems():
if "Job succeeded" in output:
ret[dataset]['Quota'][exit_status] = "quota set to {0}".format(quota)
return ret
def scrub(pool):
'''
initiates a scrub of the pool
'''
ret = {}
if zettaknight_globs.help_flag:
ret = """Scrub:
Initiates a scrub of provided pool.
Usage:
zettaknight scrub <pool>
Required Arguments:
pool
Specifies the pool to scrub."""
return ret
ret[pool] = {}
ret[pool]['Scrub'] = {}
scrub_cmd = "/sbin/zpool scrub {0}".format(pool)
scrub_run = zettaknight_utils.spawn_job(scrub_cmd)
for k,v in scrub_run.iteritems():
if str(k) is "0":
ret[pool]['Scrub']['0'] = "Scrub initiated."
else:
ret[pool]['Scrub']['1'] = v
zettaknight_utils.parse_output(ret)
return ret
def set_refreservation(dataset, refreservation):
'''
'''
refreservation_cmd = "/sbin/zfs set refreservation={0} {1}".format(refreservation, dataset)
ret = {}
if zettaknight_globs.help_flag:
ret = """Set Refreservation:
Sets a ZFS Refreservation on provided dataset.
Usage:
zettaknight set_refreservation <dataset> <refreservation>
Required Arguments:
dataset
Specifies the dataset to set a refreservation for.
refreservation
Specifies the refreservation to set. ie. 1T, 100G, etc."""
return ret
ret[dataset] = {}
ret[dataset]['refreservation'] = zettaknight_utils.spawn_job(refreservation_cmd)
for exit_status, output in ret[dataset]['refreservation'].iteritems():
if "Job succeeded" in output:
ret[dataset]['refreservation'][exit_status] = "refreservation set to {0}".format(refreservation)
return ret
def take_snap(dataset,
user,
remote_server,
secure,
nosnap=False,
pull_snap=False):
'''
'''
snap_cmd = "bash {0} -d {1} -i {2} -s {3}@{4}".format(
zettaknight_globs.snap_script, dataset,
zettaknight_globs.identity_file, user, remote_server)
if secure:
snap_cmd = "{0} -e".format(snap_cmd)
if nosnap:
snap_cmd = "{0} -r".format(snap_cmd)
if pull_snap:
snap_cmd = "{0} -p".format(snap_cmd)
print("SNAP_COMMAND: {0}".format(snap_cmd))
ret = zettaknight_utils.spawn_job(snap_cmd)
return ret
def failover(dataset, remote_server=False):
'''
'''
ret = {}
ret[dataset] = {}
ret[dataset]['Failover'] = {}
try:
user = zettaknight_globs.zfs_conf[dataset]['user']
secure = zettaknight_globs.zfs_conf[dataset]['secure']
except Exception as e:
ret[dataset]['Failover']['1'] = "Dataset: {0} not configured in configuration file.".format(dataset)
zettaknight_utils.parse_output(ret)
sys.exit(0)
try:
if not remote_server:
remote_server = zettaknight_globs.zfs_conf[dataset]['snap']['remote_server']
if isinstance(remote_server, list):
if len(remote_server) > 1:
raise Exception("Multiple remote servers defined in configuration.")
except Exception as e:
ret[dataset]['Failover']['1'] = "{0}\nRemote servers defined:\n - {1}\nRe-run with explicit remote server to failover to.".format(e, list(zettaknight_globs.zfs_conf[dataset]['snap']['remote_server']))
zettaknight_utils.parse_output(ret)
sys.exit(0)
snap_cmd = "bash {0} -d {1} -s {2}@{3} -i {4} -f".format(zettaknight_globs.snap_script, dataset, user, remote_server[0], zettaknight_globs.identity_file)
if secure:
snap_cmd = "{0} -e".format(snap_cmd)
ret[dataset]['Failover'] = zettaknight_utils.spawn_job(snap_cmd)
zettaknight_utils.parse_output(ret)
return ret
def nuke(pool, force=True):
'''
'''
ret = {}
if zettaknight_globs.help_flag:
ret = """Nuke:
Destroys a zpool and cleans up files associated with the zpool being destroyed.
Usage:
zettaknight nuke <pool_name>
Required Arguments:
pool_name
Specifies the zpool to destroy."""
return ret
ret[pool] = {}
ret[pool]['Nuke Zpool'] = {}
nuke_cmd = "bash {0} -p '{1}'".format(zettaknight_globs.zfs_nuke_script,
pool)
if force:
nuke_cmd = "{0} -f".format(nuke_cmd)
ret[pool]['Nuke Zpool'] = zettaknight_utils.spawn_job(nuke_cmd)
#zettaknight_utils.parse_output(ret)
return ret
def set_refquota(dataset, refquota):
'''
'''
refquota_cmd = "/sbin/zfs set refquota={0} {1}".format(refquota, dataset)
ret = {}
if zettaknight_globs.help_flag:
ret = """Set RefQuota:
Sets a ZFS refquota on provided dataset.
Usage:
zettaknight set_refquota <dataset> <refquota>
Required Arguments:
dataset
Specifies the dataset to set a quota for.
refquota
Specifies the quota to set. ie. 1T, 100G, etc."""
return ret
ret[dataset] = {}
ret[dataset]['refquota'] = zettaknight_utils.spawn_job(refquota_cmd)
for exit_status, output in ret[dataset]['refquota'].iteritems():
if "Job succeeded" in output:
ret[dataset]['refquota'][
exit_status] = "refquota set to {0}".format(refquota)
return ret
def scrub(pool):
'''
initiates a scrub of the pool
'''
ret = {}
if zettaknight_globs.help_flag:
ret = """Scrub:
Initiates a scrub of provided pool.
Usage:
zettaknight scrub <pool>
Required Arguments:
pool
Specifies the pool to scrub."""
return ret
ret[pool] = {}
ret[pool]['Scrub'] = {}
scrub_cmd = "/sbin/zpool scrub {0}".format(pool)
scrub_run = zettaknight_utils.spawn_job(scrub_cmd)
for k, v in scrub_run.iteritems():
if str(k) is "0":
ret[pool]['Scrub']['0'] = "Scrub initiated."
else:
ret[pool]['Scrub']['1'] = v
zettaknight_utils.parse_output(ret)
return ret
def set_quota(dataset, quota):
'''
'''
#quota_cmd = "bash {0} -d {1} -q {2}".format(zettaknight_globs.quota_script,dataset,quota)
quota_cmd = "/sbin/zfs set quota={0} {1}".format(quota, dataset)
ret = {}
if zettaknight_globs.help_flag:
ret = """Set Quota:
Sets a ZFS quota on provided dataset.
Usage:
zettaknight set_quota <dataset> <quota>
Required Arguments:
dataset
Specifies the dataset to set a quota for.
quota
Specifies the quota to set. ie. 1T, 100G, etc."""
return ret
ret[dataset] = {}
ret[dataset]['Quota'] = zettaknight_utils.spawn_job(quota_cmd)
for exit_status, output in ret[dataset]['Quota'].iteritems():
if "Job succeeded" in output:
ret[dataset]['Quota'][exit_status] = "quota set to {0}".format(
quota)
return ret
def create_snap(dataset, quiet=False):
'''
'''
zettaknight_utils.check_quiet(quiet)
snap = "{0}@{1}".format(dataset, zettaknight_globs.today_date)
gerp_run = zettaknight_utils.pipe_this2("/sbin/zfs list -r -t snapshot -o name -H {0} | /bin/grep {1}".format(dataset, snap))
gerp_out = gerp_run.stdout.read()
if int(gerp_run.returncode) is not 0:
ret = zettaknight_utils.spawn_job("/sbin/zfs snapshot -r {0}".format(snap))
if int(gerp_run.returncode) == 0:
ret = {0: "Snapshot {0} already exists.".format(snap)}
for exit_status, output in ret.iteritems():
if str(exit_status) == "0" and str(output) == "Job succeeded":
ret[exit_status] = "Snapshot created: {0}".format(snap)
if not quiet:
snap_out = {}
snap_out[dataset] = {}
snap_out[dataset][inspect.stack()[0][3]] = ret
zettaknight_utils.parse_output(snap_out)
return ret
def generate_perf_stats(**kwargs):
'''
'''
ret = {}
ret[zettaknight_globs.fqdn] = {}
ret[zettaknight_globs.fqdn]['Perf Stats'] = {}
if zettaknight_globs.help_flag:
help = """Performance Stats:
Function to generate iostat data from defined zpools"""
return help
if not os.path.isdir(zettaknight_globs.zpool_perf_dir):
os.mkdir(zettaknight_globs.zpool_perf_dir)
if zettaknight_globs.zpool_iostat_file:
perf_cmd = "bash {0} -f /{1}".format(
zettaknight_globs.perf_stats_script,
zettaknight_globs.zpool_iostat_file)
ret[zettaknight_globs.
fqdn]['Perf Stats'] = zettaknight_utils.spawn_job(perf_cmd)
else:
ret[zettaknight_globs.fqdn]['Perf Stats'][
'1'] = "{0} is not defined, cannot continue".format(
zettaknight_globs.zpool_iostat_file)
return ret
def generate_perf_stats(**kwargs):
'''
'''
ret = {}
ret[zettaknight_globs.fqdn] = {}
ret[zettaknight_globs.fqdn]['Perf Stats'] = {}
if zettaknight_globs.help_flag:
help = """Performance Stats:
Function to generate iostat data from defined zpools"""
return help
if not os.path.isdir(zettaknight_globs.zpool_perf_dir):
os.mkdir(zettaknight_globs.zpool_perf_dir)
if zettaknight_globs.zpool_iostat_file:
perf_cmd = "bash {0} -f /{1}".format(zettaknight_globs.perf_stats_script, zettaknight_globs.zpool_iostat_file)
ret[zettaknight_globs.fqdn]['Perf Stats'] = zettaknight_utils.spawn_job(perf_cmd)
else:
ret[zettaknight_globs.fqdn]['Perf Stats']['1'] = "{0} is not defined, cannot continue".format(zettaknight_globs.zpool_iostat_file)
return ret
def cleanup_snaps(dataset, retention):
'''
'''
cleanup_cmd = "bash {0} -d {1} -k {2}".format(zettaknight_globs.cleanup_script, dataset, retention)
ret = zettaknight_utils.spawn_job(cleanup_cmd)
return ret
def build_out_config(force=False):
'''
This function reads in the pool config file and creates the zfs data structures defined in it
'''
ret = {}
ret[zettaknight_globs.fqdn] = {}
ret[zettaknight_globs.fqdn]['Build Config'] = {}
create_config = {}
create_config['create_config'] = False
create_config['nfs'] = False
try:
#create pools defined in zpool conf
if zettaknight_globs.zpool_conf:
for zpool in zettaknight_globs.zpool_conf.iterkeys():
zettaknight_utils.zlog("determining if zpool {0} exists:\n\t[build_out_config] --> spawn_job : /sbin/zpool list -H {0}".format(zpool), "DEBUG")
d = zettaknight_utils.spawn_job("/sbin/zpool list -H '{0}'".format(zpool))
chk_code, chk_msg = d.popitem()
if int(chk_code) is not 0:
ret[zettaknight_globs.fqdn]['Create {0}'.format(zpool)] = {}
zettaknight_utils.zlog("creating {0}:\n\t[build_out_config] --> zettaknight_zpool.create_zpool".format(zpool), "DEBUG")
out1 = zettaknight_zpool.create_zpool(zpool, **zettaknight_globs.zpool_conf[zpool])
ret[zettaknight_globs.fqdn]['Create {0}'.format(zpool)] = out1[zpool]['Create Zpool']
#print(ret)
#create datasets defined in zfs_conf
for dataset in zettaknight_globs.zfs_conf.iterkeys():
zettaknight_utils.zlog("determining if dataset {0} exists:\n\t[build_out_config] --> spawn_job : /sbin/zfs list -H {0}".format(dataset), "DEBUG")
d = zettaknight_utils.spawn_job("/sbin/zfs list -H '{0}'".format(dataset))
chk_code, chk_msg = d.popitem()
if int(chk_code) is not 0:
zettaknight_utils.zlog("creating {0}:\n\t[build_out_config] --> add_dataset".format(dataset), "DEBUG")
out2 = add_dataset(dataset, **create_config)
ret[zettaknight_globs.fqdn]['Create dataset {0}'.format(dataset)] = out2
ret[zettaknight_globs.fqdn]['Build Config']['0'] = "Everything Looks Okay Here"
except Exception as e:
zettaknight_utils.zlog("{0}".format(e), "ERROR")
ret[zettaknight_globs.fqdn]['Build Config']['1'] = e
zettaknight_utils.zlog("ret for [build_out_config]:\n\t{0}".format(ret), "DEBUG")
return ret
def get_cifs_quota(**kwargs):
'''
get quota for user or group cifs share.
'''
if zettaknight_globs.help_flag:
ret = """Get CIFS Quota:
Function to get and display quota on a user or group CIFS share. Arguments should be passed as key/value pairs.
Required Arguments:
user or group
Specifies the username or group name tied to the relevant share to set the quota on.
Optional Arguments:
dataset
Specifies the parent dataset to search for the share under. If not specified, the default will be pulled from Zettaknight conf files"""
return ret
ret = {}
user = False
group = False
try:
if kwargs and kwargs['dataset']:
dataset = kwargs['dataset']
except:
dataset = "{0}/{1}".format(zettaknight_globs.pool_name,
zettaknight_globs.cifs_dset_suffix)
pass
if kwargs and "user" in kwargs.iterkeys():
user = kwargs['user']
if kwargs and "group" in kwargs.iterkeys():
group = kwargs['group']
if user and group:
ret[dataset] = {}
try:
raise Exception(
"Arguments user and group must be specified separately. Arguments passed: {0}"
.format(kwargs))
except Exception as e:
ret[dataset]['Get User Quota'] = {1: e}
zettaknight_utils.parse_output(ret)
return ret
if user:
dset = "{0}/{1}".format(dataset, user)
elif group:
dset = "{0}/{1}".format(dataset, group)
ret[dset] = {}
quota_cmd = "/sbin/zfs get refquota -H {0} -o value".format(dset)
ret[dset]['Get User Quota'] = zettaknight_utils.spawn_job(quota_cmd)
return ret
def cleanup_snaps(dataset, retention):
'''
'''
cleanup_cmd = "bash {0} -d {1} -k {2}".format(
zettaknight_globs.cleanup_script, dataset, retention)
ret = zettaknight_utils.spawn_job(cleanup_cmd)
return ret
def create_zpool(pool=False, disk_list=False, raid=False, luks=False, slog=False, create_config=False, ldap=False, recordsize=False, ashift=False, keyfile=False):
ret = {}
ret[pool] = {}
ret[pool]['Create Zpool'] = {}
if not raid:
raid = "12+2"
try:
disks, z_level = raid.split("+")
except Exception as e:
ret[pool]['Create Zpool'] = {'1': "{0}\nargument raid must be in x+y format, i.e. 2+1".format(e)}
zettaknight_utils.parse_output(ret)
sys.exit(0)
create_cmd = "bash {0} -d {1} -z {2}".format(zettaknight_globs.zpool_create_script, disks, z_level)
if disk_list:
create_cmd = "{0} -f '{1}'".format(create_cmd, disk_list)
if pool:
create_cmd = "{0} -p '{1}'".format(create_cmd, pool)
if luks:
create_cmd = "{0} -l".format(create_cmd)
if slog:
create_cmd = "{0} -s '{1}'".format(create_cmd, slog)
if ldap:
create_cmd = "{0} -i".format(create_cmd)
if recordsize:
if any(i in recordsize for i in 'KM'):
create_cmd = "{0} -r {1}".format(create_cmd, recordsize)
else:
print(zettaknight_utils.printcolors("Recordsize must be in number/unit format. ie. 1M, or 512K", "FAIL"))
sys.exit(0)
if ashift:
create_cmd = "{0} -a {1}".format(create_cmd, ashift)
if keyfile:
create_cmd = "{0} -k {1}".format(create_cmd, keyfile)
try:
ret[pool]['Create Zpool'] = zettaknight_utils.spawn_job(create_cmd)
zettaknight_utils.parse_output(ret)
if create_config:
zettaknight_utils.create_config(pool)
except Exception as e:
print(zettaknight_utils.printcolors(e, "FAIL"))
sys.exit(0)
return ret
def get_cifs_quota(**kwargs):
'''
get quota for user or group cifs share.
'''
if zettaknight_globs.help_flag:
ret = """Get CIFS Quota:
Function to get and display quota on a user or group CIFS share. Arguments should be passed as key/value pairs.
Required Arguments:
user or group
Specifies the username or group name tied to the relevant share to set the quota on.
Optional Arguments:
dataset
Specifies the parent dataset to search for the share under. If not specified, the default will be pulled from Zettaknight conf files"""
return ret
ret = {}
user = False
group = False
try:
if kwargs and kwargs['dataset']:
dataset = kwargs['dataset']
except:
dataset = "{0}/{1}".format(zettaknight_globs.pool_name, zettaknight_globs.cifs_dset_suffix)
pass
if kwargs and "user" in kwargs.iterkeys():
user = kwargs['user']
if kwargs and "group" in kwargs.iterkeys():
group = kwargs['group']
if user and group:
ret[dataset] = {}
try:
raise Exception("Arguments user and group must be specified separately. Arguments passed: {0}".format(kwargs))
except Exception as e:
ret[dataset]['Get User Quota'] = {1: e}
zettaknight_utils.parse_output(ret)
return ret
if user:
dset = "{0}/{1}".format(dataset, user)
elif group:
dset = "{0}/{1}".format(dataset, group)
ret[dset] = {}
quota_cmd = "/sbin/zfs get refquota -H {0} -o value".format(dset)
ret[dset]['Get User Quota'] = zettaknight_utils.spawn_job(quota_cmd)
return ret
def sync(dataset, remote_ssh, identity_file=False, pull_snap=False):
'''
'''
sync_cmd = "bash {0} -d {1} -s {2}".format(zettaknight_globs.sync_script, dataset, remote_ssh)
if identity_file:
sync_cmd = "{0} -i {1}".format(sync_cmd, identity_file)
if pull_snap:
sync_cmd = "{0} -p".format(sync_cmd)
if str(inspect.stack()[1][3]) is 'sync_all':
ret = zettaknight_utils.spawn_job(sync_cmd)
else:
ret = {}
ret[dataset] = {}
ret[dataset]['Snapshot sync'] = zettaknight_utils.spawn_job(sync_cmd)
return ret
def failover(dataset, remote_server=False):
'''
'''
ret = {}
if zettaknight_globs.help_flag:
ret = """Failover:
Attempts a controlled failover for provided dataset to remote host defined in configuration files.
Usage:
zettaknight failover <dataset>
Required Arguments:
dataset
Specifies the dataset to fail over.
Optional Arguments:
remote_server
Specifies a remote server to attempt a failover to. By default, this information is pulled from
dataset configuration files."""
return ret
ret[dataset] = {}
ret[dataset]['Failover'] = {}
try:
user = zettaknight_globs.zfs_conf[dataset]['user']
secure = zettaknight_globs.zfs_conf[dataset]['secure']
except Exception as e:
ret[dataset]['Failover']['1'] = "Dataset: {0} not configured in configuration file.".format(dataset)
zettaknight_utils.parse_output(ret)
sys.exit(0)
try:
if not remote_server:
remote_server = zettaknight_globs.zfs_conf[dataset]['snap']['remote_server']
if isinstance(remote_server, list):
if len(remote_server) > 1:
raise Exception("Multiple remote servers defined in configuration.")
except Exception as e:
ret[dataset]['Failover']['1'] = "{0}\nRemote servers defined:\n - {1}\nRe-run with explicit remote server to failover to.".format(e, list(zettaknight_globs.zfs_conf[dataset]['snap']['remote_server']))
zettaknight_utils.parse_output(ret)
sys.exit(0)
snap_cmd = "bash {0} -d {1} -s {2}@{3} -i {4} -f".format(zettaknight_globs.snap_script, dataset, user, remote_server[0], zettaknight_globs.identity_file)
if secure:
snap_cmd = "{0} -e".format(snap_cmd)
ret[dataset]['Failover'] = zettaknight_utils.spawn_job(snap_cmd)
zettaknight_utils.parse_output(ret)
return ret
def add_dataset(dataset, **kwargs):
'''if create_config=True is passed, a configuration file
will be created for the dataset passed in to this function'''
ret = {}
ret[zettaknight_globs.fqdn] = {}
question = False
if 'create_config' in kwargs.iterkeys():
create_config = kwargs['create_config']
else:
create_config = True
if 'nfs' in kwargs.iterkeys():
nfs = kwargs['nfs']
else:
nfs = True
#append dataset to kwargs
if dataset:
kwargs['dataset'] = dataset
gerp_run = zettaknight_utils.pipe_this2(
"zfs list | /bin/grep {0}".format(dataset))
gerp_out = gerp_run.stdout.read()
#print(zettaknight_utils.printcolors("\nWill create {0} after completion of config file\n".format(dataset), "OKGREEN"))
if create_config:
ret[zettaknight_globs.fqdn]['Create Config'] = {}
out1 = zettaknight_utils.create_config(**kwargs)
ret[zettaknight_globs.fqdn]['Create Config'] = out1[
zettaknight_globs.fqdn]['Create Config']
#create dataset after commit of file
if int(gerp_run.returncode) is not 0:
create_cmd = "zfs create -p {0}".format(dataset)
create_run = zettaknight_utils.spawn_job(create_cmd)
ret[zettaknight_globs.fqdn]['add_dataset'] = create_run
else:
ret[zettaknight_globs.fqdn]['add_dataset'] = {
'0': "{0} exists".format(dataset)
}
if nfs:
question = zettaknight_utils.query_yes_no(
"Would you like to create an NFS share for: {0}".format(dataset))
if question:
nfs_share = zettaknight_utils.query_return_item(
"Where would you like to share it? <i.e 10.20.30.1/24 or my.server>"
)
ret[zettaknight_globs.fqdn]['NFS'] = {}
ret[zettaknight_globs.fqdn]['NFS'] = zettaknight_utils.sharenfs(
dataset, nfs_share)
return ret
def add_dataset(*args, **kwargs):
'''if create_config=True is passed, a configuration file
will be created for the dataset passed in to this function'''
#kwargs = {}
arg_list = list(args)
#kwargs = zettaknight_utils.create_kwargs_from_args(arg_list)
#print("This is kwargs : {0}".format(kwargs))
for arg in arg_list[0:]:
if "=" in arg:
k, v = arg.split("=", 1)
kwargs[k] = v
arg_list.remove(arg)
dataset = arg_list[0]
ret = {}
ret[dataset] = {}
gerp_run = zettaknight_utils.pipe_this2("zfs list | /bin/grep {0}".format(dataset))
gerp_out = gerp_run.stdout.read()
if 'create_config' not in kwargs.iterkeys():
kwargs['create_config'] = True
if kwargs['create_config'] == 'False':
kwargs['create_config'] == False
if kwargs['create_config']:
print(zettaknight_utils.printcolors("\nWill create {0} after completion of config file\n".format(dataset), "OKGREEN"))
conf_dict = {'dataset' : dataset}
zettaknight_utils.create_config(**conf_dict)
#create dataset after commit of file
if int(gerp_run.returncode) is not 0:
create_cmd = "zfs create -p {0}".format(dataset)
create_run = zettaknight_utils.spawn_job(create_cmd)
ret[dataset]['add_dataset'] = create_run
else:
ret[dataset]['add_dataset'] = {'0' : "{0} exists".format(dataset)}
question = zettaknight_utils.query_yes_no("Would you like to create an NFS share for: {0}".format(dataset))
if question:
nfs_share = zettaknight_utils.query_return_item("Where would you like to share it? <i.e 10.20.30.1/24 or my.server>")
zettaknight_utils.sharenfs(dataset, nfs_share)
#zettaknight_utils.parse_output(ret)
return ret
def nuke(pool, force=True):
'''
'''
ret = {}
ret[pool] = {}
ret[pool]['Nuke Zpool'] = {}
nuke_cmd = "bash {0} -p '{1}'".format(zettaknight_globs.zfs_nuke_script, pool)
if force:
nuke_cmd = "{0} -f".format(nuke_cmd)
ret[pool]['Nuke Zpool'] = zettaknight_utils.spawn_job(nuke_cmd)
#zettaknight_utils.parse_output(ret)
return ret
def get_cifs_share(**kwargs):
'''
get quota for user or group cifs share.
'''
ret = {}
user = False
group = False
try:
if kwargs and kwargs['dataset']:
dataset = kwargs['dataset']
except:
dataset = "{0}/{1}".format(zettaknight_globs.pool_name, zettaknight_globs.cifs_dset_suffix)
pass
if kwargs and "user" in kwargs.iterkeys():
user = kwargs['user']
if kwargs and "group" in kwargs.iterkeys():
group = kwargs['group']
if user and group:
ret[dataset] = {}
try:
raise Exception("Arguments user and group must be specified separately. Arguments passed: {0}".format(kwargs))
except Exception as e:
ret[dataset]['Get User Quota'] = {1: e}
zettaknight_utils.parse_output(ret)
return ret
if user:
dset = "{0}/{1}".format(dataset, user)
elif group:
dset = "{0}/{1}".format(dataset, group)
ret[dset] = {}
quota_cmd = "/sbin/zfs list -H {0} -o name".format(dset)
#ret[dset]['Get User Share'] = zettaknight_utils.spawn_job(quota_cmd)
job_out = zettaknight_utils.spawn_job(quota_cmd)
print("JOB OUT: {0}".format(job_out))
if 0 in job_out.iterkeys():
job_out[0] = "//{0}/{1}/{2}".format(str(zettaknight_globs.samba_service_name), str(zettaknight_globs.samba_share_suffix), str(user))
print("JOB OUT: {0}".format(job_out))
ret[dset]['Get User Share'] = job_out
return ret
def set_refreservation(dataset, refreservation):
'''
'''
refreservation_cmd = "/sbin/zfs set refreservation={0} {1}".format(refreservation, dataset)
ret = {}
ret[dataset] = {}
ret[dataset]['refreservation'] = zettaknight_utils.spawn_job(refreservation_cmd)
for exit_status, output in ret[dataset]['refreservation'].iteritems():
if "Job succeeded" in output:
ret[dataset]['refreservation'][exit_status] = "refreservation set to {0}".format(refreservation)
return ret
def set_quota(dataset, quota):
'''
'''
#quota_cmd = "bash {0} -d {1} -q {2}".format(zettaknight_globs.quota_script,dataset,quota)
quota_cmd = "/sbin/zfs set quota={0} {1}".format(quota, dataset)
ret = {}
ret[dataset] = {}
ret[dataset]['Quota'] = zettaknight_utils.spawn_job(quota_cmd)
for exit_status, output in ret[dataset]['Quota'].iteritems():
if "Job succeeded" in output:
ret[dataset]['Quota'][exit_status] = "quota set to {0}".format(quota)
return ret
def add_dataset(dataset, **kwargs):
'''if create_config=True is passed, a configuration file
will be created for the dataset passed in to this function'''
ret = {}
ret[zettaknight_globs.fqdn] = {}
question = False
if 'create_config' in kwargs.iterkeys():
create_config = kwargs['create_config']
else:
create_config = True
if 'nfs' in kwargs.iterkeys():
nfs = kwargs['nfs']
else:
nfs = True
#append dataset to kwargs
if dataset:
kwargs['dataset'] = dataset
gerp_run = zettaknight_utils.pipe_this2("zfs list | /bin/grep {0}".format(dataset))
gerp_out = gerp_run.stdout.read()
#print(zettaknight_utils.printcolors("\nWill create {0} after completion of config file\n".format(dataset), "OKGREEN"))
if create_config:
ret[zettaknight_globs.fqdn]['Create Config'] = {}
out1 = zettaknight_utils.create_config(**kwargs)
ret[zettaknight_globs.fqdn]['Create Config'] = out1[zettaknight_globs.fqdn]['Create Config']
#create dataset after commit of file
if int(gerp_run.returncode) is not 0:
create_cmd = "zfs create -p {0}".format(dataset)
create_run = zettaknight_utils.spawn_job(create_cmd)
ret[zettaknight_globs.fqdn]['add_dataset'] = create_run
else:
ret[zettaknight_globs.fqdn]['add_dataset'] = {'0' : "{0} exists".format(dataset)}
if nfs:
question = zettaknight_utils.query_yes_no("Would you like to create an NFS share for: {0}".format(dataset))
if question:
nfs_share = zettaknight_utils.query_return_item("Where would you like to share it? <i.e 10.20.30.1/24 or my.server>")
ret[zettaknight_globs.fqdn]['NFS'] = {}
ret[zettaknight_globs.fqdn]['NFS'] = zettaknight_utils.sharenfs(dataset, nfs_share)
return ret
def zfs_monitor(email_recipients, protocol=False):
ret = {}
ret[zettaknight_globs.fqdn] = {}
protocol = "ssh"
for dataset in zettaknight_globs.zfs_conf.iterkeys():
if str(zettaknight_globs.zfs_conf[dataset]['secure']) == "False":
protocol = "xinetd"
monitor_cmd = "bash {0} -r \"{1}\" -p \"{2}\"".format(zettaknight_globs.zfs_monitor_script, email_recipients, protocol)
ret[zettaknight_globs.fqdn]['Monitor ZFS System Health'] = zettaknight_utils.spawn_job(monitor_cmd)
zettaknight_utils.parse_output(ret)
return ret
def take_snap(dataset, user, remote_server, secure, nosnap=False, pull_snap=False):
'''
'''
snap_cmd = "bash {0} -d {1} -i {2} -s {3}@{4}".format(zettaknight_globs.snap_script, dataset, zettaknight_globs.identity_file, user, remote_server)
if secure:
snap_cmd = "{0} -e".format(snap_cmd)
if nosnap:
snap_cmd = "{0} -r".format(snap_cmd)
if pull_snap:
snap_cmd = "{0} -p".format(snap_cmd)
print("SNAP_COMMAND: {0}".format(snap_cmd))
ret = zettaknight_utils.spawn_job(snap_cmd)
return ret
def set_reservation(dataset, reservation, quiet=False):
'''
'''
zettaknight_utils.check_quiet(quiet)
#reservation_cmd = "bash {0} -d {1} -r {2}".format(zettaknight_globs.quota_script,dataset,reservation)
reservation_cmd = "/sbin/zfs set reservation={0} {1}".format(reservation, dataset)
ret = {}
ret[dataset] = {}
ret[dataset]['Reservation'] = zettaknight_utils.spawn_job(reservation_cmd)
for exit_status, output in ret[dataset]['Reservation'].iteritems():
if "Job succeeded" in output:
ret[dataset]['Reservation'][exit_status] = "reservation set to {0}".format(reservation)
return ret
def scrub(pool):
'''
initiates a scrub of the pool
'''
ret = {}
ret[pool] = {}
ret[pool]['Scrub'] = {}
scrub_cmd = "/sbin/zpool scrub {0}".format(pool)
scrub_run = zettaknight_utils.spawn_job(scrub_cmd)
for k,v in scrub_run.iteritems():
if str(k) is "0":
ret[pool]['Scrub']['0'] = "Scrub initiated."
else:
ret[pool]['Scrub']['1'] = v
zettaknight_utils.parse_output(ret)
return ret
def get_cifs_quota(**kwargs):
'''
get quota for user or group cifs share.
'''
ret = {}
user = False
group = False
try:
if kwargs and kwargs['dataset']:
dataset = kwargs['dataset']
except:
dataset = "{0}/{1}".format(zettaknight_globs.pool_name, zettaknight_globs.cifs_dset_suffix)
pass
if kwargs and "user" in kwargs.iterkeys():
user = kwargs['user']
if kwargs and "group" in kwargs.iterkeys():
group = kwargs['group']
if user and group:
ret[dataset] = {}
try:
raise Exception("Arguments user and group must be specified separately. Arguments passed: {0}".format(kwargs))
except Exception as e:
ret[dataset]['Get User Quota'] = {1: e}
zettaknight_utils.parse_output(ret)
return ret
if user:
dset = "{0}/{1}".format(dataset, user)
elif group:
dset = "{0}/{1}".format(dataset, group)
ret[dset] = {}
quota_cmd = "/sbin/zfs get quota -H {0} -o value".format(dset)
ret[dset]['Get User Quota'] = zettaknight_utils.spawn_job(quota_cmd)
return ret
def zfs_monitor(email_recipients, protocol=False):
ret = {}
ret[zettaknight_globs.fqdn] = {}
zettaknight_globs.mm_flag = True
protocol = "ssh"
for dataset in zettaknight_globs.zfs_conf.iterkeys():
if str(zettaknight_globs.zfs_conf[dataset]['secure']) == "False":
protocol = "xinetd"
monitor_cmd = "bash {0} -r \"{1}\" -p \"{2}\"".format(
zettaknight_globs.zfs_monitor_script, email_recipients, protocol)
ret[zettaknight_globs.fqdn][
'Monitor ZFS System Health'] = zettaknight_utils.spawn_job(monitor_cmd)
#zettaknight_utils.parse_output(ret)
return ret
def set_reservation(dataset, reservation, quiet=False):
'''
'''
zettaknight_utils.check_quiet(quiet)
#reservation_cmd = "bash {0} -d {1} -r {2}".format(zettaknight_globs.quota_script,dataset,reservation)
reservation_cmd = "/sbin/zfs set reservation={0} {1}".format(
reservation, dataset)
ret = {}
if zettaknight_globs.help_flag:
ret = """Set Reservation:
Sets a ZFS reservation on provided dataset.
Usage:
zettaknight set_reservation <dataset> <reservation>
Required Arguments:
dataset
Specifies the dataset to set a reservation for.
reservation
Specifies the reservation to set. ie. 1T, 100G, etc."""
return ret
ret[dataset] = {}
ret[dataset]['Reservation'] = zettaknight_utils.spawn_job(reservation_cmd)
for exit_status, output in ret[dataset]['Reservation'].iteritems():
if "Job succeeded" in output:
ret[dataset]['Reservation'][
exit_status] = "reservation set to {0}".format(reservation)
return ret
def recover(snapshot, filename, relocate=None):
'''
'''
if zettaknight_globs.help_flag:
ret = """Recover:
Usage:
zettaknight recover <snapshot> <filepath> (<directory to recover file to>)*optional
Recovers a previous version of a file or folder from a specified snapshot. Information to use in calling zettaknight recover can be found in the output from the find_versions function. By default, files/folders will be recovered to their original location with .R appended to the end of the name.
Required Arguments:
snapshot
Specifies the snapshot to recover files from.
filename
Defines target file(s) to recover. This should be the full path to a file or folder (/zfs_data/<some dataset>/<some filename>.<ext>)
Optional Arguments:
relocate
Defines an alternate path to recover files/folders to. If used, .R will not be appended, and the recover will overwrite any existing files."""
return ret
file = filename
snap = snapshot
dataset, snap_date = snap.split('@', 1)
find_snap = find_versions(dataset, file, "quiet")
find_snap_keys = find_snap.iterkeys()
snap_basedir = "/{0}/.zfs/snapshot".format(dataset)
active_basedir = "/{0}".format(dataset)
dir_flag = False
ret = {}
ret[dataset] = {}
try:
if str(dataset) in str(file):
file = file.split(dataset, 1)[1]
last_ref = file.rsplit("/", 1)[1]
except Exception as e:
pass
if dataset in file.rsplit("/", 1):
file = file.split(dataset, 1)[1]
last_ref = file.rsplit("/", 1)[1]
if snap not in find_snap_keys:
try:
raise Exception(
"Recoverable versions of {0} not found in dataset {1}".format(
file, dataset))
except Exception as e:
zettaknight_utils.zlog("{0}".format(e), "ERROR")
path_list1 = find_snap[snap]
path_list = []
if os.path.isdir("/{0}{1}".format(dataset, file)):
dir_flag = True
for line in path_list1:
if line.startswith("-") or line.startswith("M") or line.startswith(
"R"):
a, b = line.rsplit("/", 1)
if not dir_flag:
try:
if str(b) == str(last_ref):
path_list.append(line)
except:
if str(b) == str(file):
path_list.append(line)
pass
else:
if a.startswith("-") or line.startswith(
"M") or line.startswith("R"):
a, b = line.rsplit(dataset, 2)
if os.path.isdir("{0}/{1}{2}".format(
snap_basedir, snap_date, b)):
dir_flag = True
path_list.append(line)
if len(path_list) > 1:
exc_out = ""
for i in path_list:
if exc_out:
exc_out = "{0}\n\t{1}".format(exc_out, i)
else:
exc_out = str(i)
err_msg = "Matching files/folders in snapshot:"
print(
zettaknight_utils.printcolors(
"\nAmbiguous recover request. Multiple matches for {0}.".
format(file), "FAIL"))
print("{0} {1}".format(zettaknight_utils.printcolors(err_msg, "FAIL"),
zettaknight_utils.printcolors(snap, "OKBLUE")))
print("\t{0}".format(zettaknight_utils.printcolors(exc_out,
"WARNING")))
print(
zettaknight_utils.printcolors(
"Re-run with explicit path to file.\n", "FAIL"))
try:
raise Exception("Ambiguous file reference.")
except Exception as e:
zettaknight_utils.zlog("{0}".format(e), "ERROR")
sys.exit(0)
if len(path_list) < 1:
try:
raise Exception(
"No restorable files or folders identified. \nRe-run with explicit path to file?"
)
except Exception as e:
zettaknight_utils.zlog("{0}".format(e), "ERROR")
sys.exit(0)
a, p = path_list[0].split(dataset, 1)
path, dirs = p.split(file, 1)
if dir_flag:
a, b = path_list[0].rsplit(dataset, 2)
file_loc = "{0}/{1}{2}".format(snap_basedir, snap_date, b)
#bad, dir_path = path_list[0].split(file, 1)
#list_dirs = dir_path.rsplit("/", 1)
#try:
# out_dir = list_dirs[0]
#except:
out_dir = ""
# pass
#file_loc = "{0}/{1}/{2}{3}/".format(snap_basedir, snap_date, file, dir_path )
#mkdir_cmd = "/bin/mkdir {0}/{1}.R".format(active_basedir, file)
#mkdir_run = zettaknight_utils.spawn_job(mkdir_cmd)
#mkdir_cmd = "/bin/mkdir -p {0}/{1}.R/{2}".format(active_basedir, file, dir_path)
#mkdir_run = zettaknight_utils.spawn_job(mkdir_cmd)
out_path = "{0}{1}{2}.R{3}/".format(active_basedir, str(path), file,
out_dir)
else:
file_loc = "{0}/{1}/{2}".format(snap_basedir, snap_date, p)
out_path = "{0}{1}{2}.R".format(active_basedir, str(path), file)
if relocate:
out_path = relocate
rec_cmd = "/bin/cp " "-r -p " "{0}" " " "{1}".format(file_loc, out_path)
rec_run = zettaknight_utils.spawn_job(rec_cmd)
rec_dict = eval(str(rec_run))
for k, v in rec_dict.iteritems():
if str(k) is "0":
print("Recover operation succeeded.")
print("\tFilename: {0}".format(
zettaknight_utils.printcolors(file, "OKBLUE")))
print("\t\tFile(s) restored to version: {0}".format(
zettaknight_utils.printcolors(snap_date, "OKGREEN")))
print("\t\tFile(s) restored to: {0}".format(
zettaknight_utils.printcolors(out_path, "OKGREEN")))
if str(k) is not "0":
print("Recover operation failed.")
print("\tFilename: {0}".format(
zettaknight_utils.printcolors(file, "OKBLUE")))
print("\t\tRestore failed with error: {0}".format(
zettaknight_utils.printcolors(v, "FAIL")))
return
def set_cifs_quota(**kwargs):
'''
Set quota for user or group cifs share.
'''
ret = {}
user = False
group = False
quota = False
try:
if kwargs and kwargs['dataset']:
dataset = kwargs['dataset']
except:
dataset = "{0}/{1}".format(zettaknight_globs.pool_name, zettaknight_globs.cifs_dset_suffix)
pass
if kwargs and "user" in kwargs.iterkeys():
user = kwargs['user']
if kwargs and "group" in kwargs.iterkeys():
group = kwargs['group']
if kwargs and "quota" in kwargs.iterkeys():
quota = kwargs['quota']
if user and group:
ret[dataset] = {}
try:
raise Exception("Arguments user and group must be specified separately. Arguments passed: {0}".format(kwargs))
except Exception as e:
ret[dataset]['Set User Quota'] = {1: e}
zettaknight_utils.parse_output(ret)
return ret
if not quota:
ret[dataset] = {}
try:
raise Exception("Quota argument must be supplied. Arguments passed: {0}".format(kwargs))
except Exception as e:
ret[dataset]['Set User Quota'] = {1: e}
zettaknight_utils.parse_output(ret)
return ret
if user:
obj = user
dset = "{0}/{1}".format(dataset, user)
elif group:
obj = group
dset = "{0}/{1}".format(dataset, group)
try:
if str(zettaknight_globs.zfs_conf[dataset]['primary']) == str(zettaknight_globs.fqdn):
if zettaknight_globs.zfs_conf[dataset]['snap']['remote_server']:
for remote_server in zettaknight_globs.zfs_conf[dataset]['snap']['remote_server']:
remote_user = zettaknight_globs.zfs_conf[dataset]['user']
remote_ssh = "{0}@{1}".format(remote_user, remote_server)
try:
ssh2 = paramiko.SSHClient()
ssh2.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh2.connect(remote_server, username=remote_user, key_filename=zettaknight_globs.identity_file)
remote_cmd = "zettaknight set_cifs_quota user={0} dataset={1} quota={2}".format(obj, dataset, quota)
ssh2_stdin, ssh2_stdout, ssh2_stderr = ssh2.exec_command(remote_cmd)
ssh2.close()
except Exception as e:
pass
except:
pass
ret[dset] = {}
quota_cmd = "/sbin/zfs set quota={0} {1}".format(quota, dset)
ret[dset]['Set User Quota'] = zettaknight_utils.spawn_job(quota_cmd)
return ret
def first_step():
ret = {}
arg_dict = {}
try:
pool_name = "zfs_data" #default pool name
arg_dict["pool_name"] = pool_name
ret[pool_name] = {}
############### creation of the zpool ################
######################################################
#test if a zpool exists
zpool_list_cmd = "/sbin/zpool list -H"
zpool_list_run = zettaknight_utils.spawn_job(zpool_list_cmd)
chk_code, chk_msg = zpool_list_run.popitem()
if int(chk_code) != 0: #if zpool list -H returns empty
#query_return_item is defined in zettaknight_utils
create_pool_question = zettaknight_utils.query_yes_no(
"No pool detected, would you like to create one?")
if create_pool_question:
arg_dict["disk_list"] = zettaknight_utils.query_return_item(
"File containing all list of disks to be used for {0}, I.E /tmp/bunchofdisks.txt: "
.format(pool_name))
arg_dict["raid"] = zettaknight_utils.query_return_item(
"Raid level to be used [1-3]. I.E. 4+1, 5+2, 1+1: ")
arg_dict["ashift"] = zettaknight_utils.query_return_item(
"ashift value for {0}, value is 9 for 512 block disks and 12 for 4096 block disks: "
.format(pool_name))
arg_dict["luks"] = zettaknight_utils.query_yes_no(
"encrypt data disks with LUKS? ")
print("creating zpool: {0}".format(pool_name))
create_output = zettaknight_zpools.create(
pool_name, **arg_dict) #from zettaknight_zpools
#ret[pool_name]['Zpool Create'] = create_output[pool_name]['Zpool Create'] #'NoneType' object has no attribute '__getitem__'
else:
print("a pool already exists, moving on")
######################################################
######################################################
################## ssh key creation #################
print("checking for {0}".format(zettaknight_globs.identity_file))
if not os.path.isfile(zettaknight_globs.identity_file):
print(
zettaknight_utils.printcolors(
"zettaknight id file not found, generating.", "WARNING"))
keygen_output = zettaknight_utils.ssh_keygen(
zettaknight_globs.identity_file)
ret[pool_name]['Generate SSH Keygen'] = keygen_output[
zettaknight_globs.fqdn]['Generate SSH Key']
else:
print("{0} exists, moving on".format(
zettaknight_globs.identity_file))
######################################################
############# backup luks headers ####################
print("backing up luks headers for {0}".format(pool_name))
luks_backup_output = zettaknight_utils.backup_luks_headers()
ret[pool_name]['Backup Luks Headers'] = luks_backup_output[
zettaknight_globs.fqdn]['Backup Luks Headers']
######################################################
######## create zettaknight configuration file #######
if not os.path.isdir(zettaknight_globs.conf_dir_new
): #if new directory does not exists, make it
print("creating {0}".format(zettaknight_globs.conf_dir_new))
os.mkdir(zettaknight_globs.conf_dir_new)
######################################################
########### create zettaknight store #################
######################################################
if not os.path.isdir("/{0}".format(
zettaknight_globs.zettaknight_store)):
dataset_check_run = zettaknight_utils.pipe_this2(
"/sbin/zfs list -H | grep {0}".format(
zettaknight_globs.zettaknight_store))
if int(dataset_check_run.returncode) != 0:
print("creating configuration file for {0}".format(
zettaknight_globs.zettaknight_store))
zettaknight_utils.create_config(
dataset=zettaknight_globs.zettaknight_store)
gerp_run = zettaknight_utils.pipe_this2(
"/sbin/zfs list -H | /bin/grep {0}".format(
zettaknight_globs.zettaknight_store))
if int(gerp_run.returncode) is not 0:
zettaknight_zfs.add_dataset(zettaknight_globs.zettaknight_store)
#backup files to store
files = [
"/etc/exports", "/etc/crypttab",
"{0}".format(zettaknight_globs.config_file_new)
]
for file in files:
if os.path.exists(file):
destination_dir = "/{0}/{1}".format(
zettaknight_globs.zettaknight_store, zettaknight_globs.fqdn
) #add leading slash, zfs_share defined
filename = file.replace(
"/", ""
) #remove illegal characters from file path and save file as the concatenated version
if not os.path.isdir(destination_dir):
os.mkdir(destination_dir)
print("backing up {0} to {1}".format(file, destination_dir))
shutil.copyfile(file, "{0}/{1}".format(destination_dir,
filename))
######################################################
######################################################
############ create the first dataset ################
dataset = zettaknight_utils.query_return_list(
"Datasets to be created on {0}: ".format(pool_name))
for item in dataset:
dset_args = []
dataset_full = "{0}/{1}".format(pool_name, item)
dset_args.append(dataset_full)
dset_args.append("create_config=True")
add_dset_output = zettaknight_zfs.add_dataset(
*dset_args) #from zettaknight_utils
ret[pool_name]['Add dataset {0}'.format(
dataset_full)] = add_dset_output[dataset_full]['add_dataset']
######################################################
###### create server to server transfer pathing ######
replication_output = zettaknight_zfs.configure_replication(
) #from zettaknight_zfs
for repl_job, repl_job_output in replication_output[
dataset_full].itervalues():
ret[pool_name]['{0}'.format(repl_job)] = repl_job_output
######################################################
except Exception as e:
print(zettaknight_utils.printcolors(e, "FAIL"))
sys.exit(1)
zettaknight_globs.zfs_conf = _get_conf()
return ret
def create_cifs_share(**kwargs):
'''
Creates a zfs dataset for a given user or group.
'''
ret = {}
if zettaknight_globs.help_flag:
ret = """Create CIFS Share:
Function to create a new directory with proper permissions for user/group access over CIFS on primary and secondary ZFS servers. Arguments should be passed as key/value pairs.
Required Arguments:
user or group
Specifies the username or group name to create the share for.
Optional Arguments:
dataset
Specifies the parent dataset to create the share under. If not specified, the default will be pulled from Zettaknight conf files"""
return ret
user = False
group = False
if kwargs:
try:
if kwargs['dataset']:
dataset = kwargs['dataset']
except:
dataset = "{0}/{1}".format(zettaknight_globs.pool_name,
zettaknight_globs.cifs_dset_suffix)
pass
if kwargs and "user" in kwargs.iterkeys():
user = kwargs['user']
if kwargs and "group" in kwargs.iterkeys():
group = kwargs['group']
ret[dataset] = {}
if user and group:
try:
raise Exception(
"Arguments user and group must be specified separately. Arguments passed: {0}"
.format(kwargs))
except Exception as e:
ret[dataset]['Create CIFS Share'] = {1: e}
zettaknight_utils.parse_output(ret)
return ret
if user:
cifs_cmd = "{0} -u {1} -d {2}".format(
zettaknight_globs.cifs_share_script, user, dataset)
if group:
cifs_cmd = "{0} -g {1} -d {2}".format(
zettaknight_globs.cifs_share_script, group, dataset)
if user:
obj = user
elif group:
obj = group
try:
if str(zettaknight_globs.zfs_conf[dataset]['primary']) == str(
zettaknight_globs.fqdn):
if zettaknight_globs.zfs_conf[dataset]['snap']['remote_server']:
for remote_server in zettaknight_globs.zfs_conf[dataset][
'snap']['remote_server']:
remote_user = zettaknight_globs.zfs_conf[dataset]['user']
remote_ssh = "{0}@{1}".format(remote_user, remote_server)
try:
ssh2 = paramiko.SSHClient()
ssh2.set_missing_host_key_policy(
paramiko.AutoAddPolicy())
ssh2.connect(
remote_server,
username=remote_user,
key_filename=zettaknight_globs.identity_file)
remote_cmd = "zettaknight create_cifs_share user={0} dataset={1}".format(
obj, dataset)
ssh2_stdin, ssh2_stdout, ssh2_stderr = ssh2.exec_command(
remote_cmd)
ssh2.close()
except Exception as e:
pass
except:
pass
try:
ret[dataset]['Create CIFS Share'] = zettaknight_utils.spawn_job(
cifs_cmd)
except Exception as e:
ret[dataset]['Create CIFS Share'] = {1: e}
#zettaknight_utils.parse_output(ret)
return ret
def get_cifs_share(**kwargs):
'''
get quota for user or group cifs share.
'''
if zettaknight_globs.help_flag:
ret = """Get CIFS Share:
Returns location UNC path of CIFS share for provided user or group. Arguments should be passed as key/value pairs.
Required Arguments:
user or group
Specifies the username or group name to query.
Optional Arguments:
dataset
Specifies the parent dataset to query for the share under. If not specified, the default will be pulled from Zettaknight conf files"""
return ret
ret = {}
user = False
group = False
try:
if kwargs and kwargs['dataset']:
dataset = kwargs['dataset']
except:
dataset = "{0}/{1}".format(zettaknight_globs.pool_name,
zettaknight_globs.cifs_dset_suffix)
pass
if kwargs and "user" in kwargs.iterkeys():
user = kwargs['user']
if kwargs and "group" in kwargs.iterkeys():
group = kwargs['group']
if user and group:
ret[dataset] = {}
try:
raise Exception(
"Arguments user and group must be specified separately. Arguments passed: {0}"
.format(kwargs))
except Exception as e:
ret[dataset]['Get User Quota'] = {1: e}
zettaknight_utils.parse_output(ret)
return ret
if user:
dset = "{0}/{1}".format(dataset, user)
elif group:
dset = "{0}/{1}".format(dataset, group)
ret[dset] = {}
quota_cmd = "/sbin/zfs list -H {0} -o name".format(dset)
#ret[dset]['Get User Share'] = zettaknight_utils.spawn_job(quota_cmd)
job_out = zettaknight_utils.spawn_job(quota_cmd)
print("JOB OUT: {0}".format(job_out))
if 0 in job_out.iterkeys():
job_out[0] = "//{0}/{1}/{2}".format(
str(zettaknight_globs.samba_service_name),
str(zettaknight_globs.samba_share_suffix), str(user))
print("JOB OUT: {0}".format(job_out))
ret[dset]['Get User Share'] = job_out
return ret
def failover(dataset, remote_server=False):
'''
'''
ret = {}
if zettaknight_globs.help_flag:
ret = """Failover:
Attempts a controlled failover for provided dataset to remote host defined in configuration files.
Usage:
zettaknight failover <dataset>
Required Arguments:
dataset
Specifies the dataset to fail over.
Optional Arguments:
remote_server
Specifies a remote server to attempt a failover to. By default, this information is pulled from
dataset configuration files."""
return ret
ret[dataset] = {}
ret[dataset]['Failover'] = {}
try:
user = zettaknight_globs.zfs_conf[dataset]['user']
secure = zettaknight_globs.zfs_conf[dataset]['secure']
except Exception as e:
ret[dataset]['Failover'][
'1'] = "Dataset: {0} not configured in configuration file.".format(
dataset)
zettaknight_utils.parse_output(ret)
sys.exit(0)
try:
if not remote_server:
remote_server = zettaknight_globs.zfs_conf[dataset]['snap'][
'remote_server']
if isinstance(remote_server, list):
if len(remote_server) > 1:
raise Exception(
"Multiple remote servers defined in configuration.")
except Exception as e:
ret[dataset]['Failover'][
'1'] = "{0}\nRemote servers defined:\n - {1}\nRe-run with explicit remote server to failover to.".format(
e,
list(zettaknight_globs.zfs_conf[dataset]['snap']
['remote_server']))
zettaknight_utils.parse_output(ret)
sys.exit(0)
snap_cmd = "bash {0} -d {1} -s {2}@{3} -i {4} -f".format(
zettaknight_globs.snap_script, dataset, user, remote_server[0],
zettaknight_globs.identity_file)
if secure:
snap_cmd = "{0} -e".format(snap_cmd)
ret[dataset]['Failover'] = zettaknight_utils.spawn_job(snap_cmd)
zettaknight_utils.parse_output(ret)
return ret
def recover(snapshot, filename, relocate=None):
'''
'''
if zettaknight_globs.help_flag:
ret = """Recover:
Usage:
zettaknight recover <snapshot> <filepath> (<directory to recover file to>)*optional
Recovers a previous version of a file or folder from a specified snapshot. Information to use in calling zettaknight recover can be found in the output from the find_versions function. By default, files/folders will be recovered to their original location with .R appended to the end of the name.
Required Arguments:
snapshot
Specifies the snapshot to recover files from.
filename
Defines target file(s) to recover. This should be the full path to a file or folder (/zfs_data/<some dataset>/<some filename>.<ext>)
Optional Arguments:
relocate
Defines an alternate path to recover files/folders to. If used, .R will not be appended, and the recover will overwrite any existing files."""
return ret
file = filename
snap = snapshot
dataset, snap_date = snap.split('@', 1)
find_snap = find_versions(dataset, file, "quiet")
find_snap_keys = find_snap.iterkeys()
snap_basedir = "/{0}/.zfs/snapshot".format(dataset)
active_basedir = "/{0}".format(dataset)
dir_flag = False
ret = {}
ret[dataset] = {}
try:
if str(dataset) in str(file):
file = file.split(dataset, 1)[1]
last_ref = file.rsplit("/", 1)[1]
except Exception as e:
pass
if dataset in file.rsplit("/", 1):
file = file.split(dataset, 1)[1]
last_ref = file.rsplit("/",1)[1]
if snap not in find_snap_keys:
try:
raise Exception("Recoverable versions of {0} not found in dataset {1}".format(file, dataset))
except Exception as e:
zettaknight_utils.zlog("{0}".format(e), "ERROR")
path_list1 = find_snap[snap]
path_list = []
if os.path.isdir("/{0}{1}".format(dataset, file)):
dir_flag = True
for line in path_list1:
if line.startswith("-") or line.startswith("M") or line.startswith("R"):
a, b = line.rsplit("/", 1)
if not dir_flag:
try:
if str(b) == str(last_ref):
path_list.append(line)
except:
if str(b) == str(file):
path_list.append(line)
pass
else:
if a.startswith("-") or line.startswith("M") or line.startswith("R"):
a, b = line.rsplit(dataset, 2)
if os.path.isdir("{0}/{1}{2}".format(snap_basedir, snap_date, b)):
dir_flag = True
path_list.append(line)
if len(path_list) > 1:
exc_out = ""
for i in path_list:
if exc_out:
exc_out = "{0}\n\t{1}".format(exc_out, i)
else:
exc_out = str(i)
err_msg = "Matching files/folders in snapshot:"
print(zettaknight_utils.printcolors("\nAmbiguous recover request. Multiple matches for {0}.".format(file), "FAIL"))
print("{0} {1}".format(zettaknight_utils.printcolors(err_msg, "FAIL"), zettaknight_utils.printcolors(snap, "OKBLUE")))
print("\t{0}".format(zettaknight_utils.printcolors(exc_out, "WARNING")))
print(zettaknight_utils.printcolors("Re-run with explicit path to file.\n", "FAIL"))
try:
raise Exception("Ambiguous file reference.")
except Exception as e:
zettaknight_utils.zlog("{0}".format(e), "ERROR")
sys.exit(0)
if len(path_list) < 1:
try:
raise Exception("No restorable files or folders identified. \nRe-run with explicit path to file?")
except Exception as e:
zettaknight_utils.zlog("{0}".format(e), "ERROR")
sys.exit(0)
a, p = path_list[0].split(dataset, 1)
path, dirs = p.split(file, 1)
if dir_flag:
a, b = path_list[0].rsplit(dataset, 2)
file_loc = "{0}/{1}{2}".format(snap_basedir, snap_date, b)
#bad, dir_path = path_list[0].split(file, 1)
#list_dirs = dir_path.rsplit("/", 1)
#try:
# out_dir = list_dirs[0]
#except:
out_dir = ""
# pass
#file_loc = "{0}/{1}/{2}{3}/".format(snap_basedir, snap_date, file, dir_path )
#mkdir_cmd = "/bin/mkdir {0}/{1}.R".format(active_basedir, file)
#mkdir_run = zettaknight_utils.spawn_job(mkdir_cmd)
#mkdir_cmd = "/bin/mkdir -p {0}/{1}.R/{2}".format(active_basedir, file, dir_path)
#mkdir_run = zettaknight_utils.spawn_job(mkdir_cmd)
out_path = "{0}{1}{2}.R{3}/".format(active_basedir, str(path), file, out_dir)
else:
file_loc = "{0}/{1}/{2}".format(snap_basedir, snap_date, p)
out_path = "{0}{1}{2}.R".format(active_basedir, str(path), file)
if relocate:
out_path = relocate
rec_cmd = "/bin/cp ""-r -p ""{0}"" ""{1}".format(file_loc, out_path)
rec_run = zettaknight_utils.spawn_job(rec_cmd)
rec_dict = eval(str(rec_run))
for k,v in rec_dict.iteritems():
if str(k) is "0":
print("Recover operation succeeded.")
print("\tFilename: {0}".format(zettaknight_utils.printcolors(file, "OKBLUE")))
print("\t\tFile(s) restored to version: {0}".format(zettaknight_utils.printcolors(snap_date, "OKGREEN")))
print("\t\tFile(s) restored to: {0}".format(zettaknight_utils.printcolors(out_path, "OKGREEN")))
if str(k) is not "0":
print("Recover operation failed.")
print("\tFilename: {0}".format(zettaknight_utils.printcolors(file, "OKBLUE")))
print("\t\tRestore failed with error: {0}".format(zettaknight_utils.printcolors(v, "FAIL")))
return
def set_cifs_quota(**kwargs):
'''
Set quota for user or group cifs share.
'''
ret = {}
if zettaknight_globs.help_flag:
ret = """Set CIFS Quota:
Function to set a quota on a user or group CIFS share. Sets the quota on primary and secondary ZFS servers. Arguments should be passed as key/value pairs.
Required Arguments:
user or group
Specifies the username or group name tied to the relevant share to set the quota on.
quota
Quota to be set. Should be specified with a number and unit, ie. 10T, 100G, 10M. 'None' should be specified to remove the quota.
Optional Arguments:
dataset
Specifies the parent dataset to search for the share under. If not specified, the default will be pulled from Zettaknight conf files"""
return ret
user = False
group = False
quota = False
try:
if kwargs and kwargs['dataset']:
dataset = kwargs['dataset']
except:
dataset = "{0}/{1}".format(zettaknight_globs.pool_name,
zettaknight_globs.cifs_dset_suffix)
pass
if kwargs and "user" in kwargs.iterkeys():
user = kwargs['user']
if kwargs and "group" in kwargs.iterkeys():
group = kwargs['group']
if kwargs and "quota" in kwargs.iterkeys():
quota = kwargs['quota']
if user and group:
ret[dataset] = {}
try:
raise Exception(
"Arguments user and group must be specified separately. Arguments passed: {0}"
.format(kwargs))
except Exception as e:
ret[dataset]['Set User Quota'] = {1: e}
zettaknight_utils.parse_output(ret)
return ret
if not quota:
ret[dataset] = {}
try:
raise Exception(
"Quota argument must be supplied. Arguments passed: {0}".
format(kwargs))
except Exception as e:
ret[dataset]['Set User Quota'] = {1: e}
zettaknight_utils.parse_output(ret)
return ret
if user:
obj = user
dset = "{0}/{1}".format(dataset, user)
elif group:
obj = group
dset = "{0}/{1}".format(dataset, group)
try:
if str(zettaknight_globs.zfs_conf[dataset]['primary']) == str(
zettaknight_globs.fqdn):
if zettaknight_globs.zfs_conf[dataset]['snap']['remote_server']:
for remote_server in zettaknight_globs.zfs_conf[dataset][
'snap']['remote_server']:
remote_user = zettaknight_globs.zfs_conf[dataset]['user']
remote_ssh = "{0}@{1}".format(remote_user, remote_server)
try:
ssh2 = paramiko.SSHClient()
ssh2.set_missing_host_key_policy(
paramiko.AutoAddPolicy())
ssh2.connect(
remote_server,
username=remote_user,
key_filename=zettaknight_globs.identity_file)
remote_cmd = "zettaknight set_cifs_quota user={0} dataset={1} quota={2}".format(
obj, dataset, quota)
ssh2_stdin, ssh2_stdout, ssh2_stderr = ssh2.exec_command(
remote_cmd)
ssh2.close()
except Exception as e:
pass
except:
pass
ret[dset] = {}
quota_cmd = "/sbin/zfs set refquota={0} {1}".format(quota, dset)
ret[dset]['Set User Quota'] = zettaknight_utils.spawn_job(quota_cmd)
return ret
def rename_dataset(**kwargs):
'''
'''
import paramiko
try:
conf_file = zettaknight_globs.config_file_new
if not 'keyfile' in kwargs.iterkeys():
keyfile = zettaknight_globs.identity_file
if not 'dataset' in kwargs.iterkeys():
raise ValueError('A very specific bad thing happened')
#required argument, show_help
else:
dataset = kwargs['dataset']
if not 'new_dataset' in kwargs.iterkeys():
#required argument, show_help
raise ValueError('A very specific bad thing happened')
else:
new_dataset = kwargs['new_dataset']
if not 'user' in kwargs.iterkeys():
user = zettaknight_globs.zfs_conf[dataset]['user']
print("user is {0}".format(user))
remote_server = []
for r_server in zettaknight_globs.zfs_conf[dataset]['snap'][
'remote_server']:
remote_server.append(r_server)
print(remote_server)
except Exception as e:
zettaknight_utils.zlog("{0}".format(e), "ERROR")
sys.exit(1)
ret = {}
ret[zettaknight_globs.fqdn] = {}
ret[zettaknight_globs.fqdn]['rename {0}'.format(dataset)] = {}
try:
for r in remote_server:
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect(r, username=user, key_filename=keyfile)
remote_sudo_cmd = "zfs rename {0} {1}".format(dataset, new_dataset)
ssh_stdin, ssh_stdout, ssh_stderr = ssh.exec_command(
remote_sudo_cmd)
print ssh_stdout.read()
#call function replace_string
replace_string(dataset, new_dataset, conf_file)
cmd = "zfs rename {0} {1}".format(dataset, new_dataset)
zettaknight_utils.spawn_job(cmd)
except Exception as e:
zettaknight_utils.zlog("{0}".format(e), "ERROR")
sys.exit(1)
ret[zettaknight_globs.fqdn]['rename {0}'.format(dataset)] = {
0:
"successfully renamed {0} to {1}, all records have been updated".
format(dataset, new_dataset)
}
return ret
def check_usage(dset=False, quiet=False):
'''
'''
ret = {}
zettaknight_utils.check_quiet(quiet)
if dset and str(dset) not in zettaknight_globs.zfs_conf.iterkeys():
ret[dset] = {}
ret[dset]['Check Usage'] = {
1: "{0} is not a Zettaknight controlled dataset.".format(dset)
}
zettaknight_utils.parse_output(ret)
return ret
for dataset in zettaknight_globs.zfs_conf.iterkeys():
if dset:
if str(dset) != str(dataset):
continue
ret[dataset] = {}
ret[dataset]['Check Usage'] = {}
quota = zettaknight_globs.zfs_conf[dataset]['quota']
reservation = zettaknight_globs.zfs_conf[dataset]['reservation']
#find conversion multiplier to convert reservation to bytes
if 'G' in reservation:
res_x = float(1073741824)
res_unit = "G"
if 'T' in reservation:
res_x = float(1099511627776)
res_unit = "T"
if 'M' in reservation:
res_x = float(1048576)
res_unit = "M"
if str(reservation) == 'none':
res_x = float(1073741824)
res_unit = "G"
contact = zettaknight_globs.zfs_conf[dataset]['contact']
check_usage_cmd = "/sbin/zfs list -ro space {0} -H -p".format(dataset)
check_usage_run = zettaknight_utils.spawn_job(check_usage_cmd)
chk_code, chk_msg = check_usage_run.popitem()
chk_msg = re.sub("\n", "", chk_msg)
chk_msg_split = chk_msg.split("\t")
if not str(chk_code) == "0":
ret[dataset]['Check Usage'][
'1'] = "{0}Verify correct datasets are defined in configuration file\n".format(
re.sub("[\[\"\]]", "", str(chk_msg_split)))
continue
if str(chk_msg_split[0]) == str(dataset):
avail = float(
re.sub("[A-Za-z]", "",
chk_msg_split[1])) #space available for dataset
used = float(re.sub(
"[A-Za-z]", "",
chk_msg_split[2])) #how much space is used by the dataset
usnap = float(
re.sub("[A-Za-z]", "", chk_msg_split[3]
)) #how much of the used space is consumed by snapshots
uds = float(
re.sub("[A-Za-z]", "", chk_msg_split[4])
) #how much of the space is consumed by the dataset itself
if str(reservation) != 'none':
res = (float(re.sub("[A-Za-z]", "", reservation)) * res_x)
avail_friendly = "{0:.2f}{1}".format((avail / res_x), res_unit)
used_friendly = "{0:.2f}{1}".format((used / res_x), res_unit)
usnap_friendly = "{0:.2f}{1}".format((usnap / res_x), res_unit)
uds_friendly = "{0:.2f}{1}".format((uds / res_x), res_unit)
if "e-" in str(used_friendly):
used_friendly = "{0}{1}".format(int(0), res_unit)
if "e-" in str(usnap_friendly):
usnap_friendly = "{0}{1}".format(int(0), res_unit)
if "e-" in str(uds_friendly):
uds_friendly = "{0}{1}".format(int(0), res_unit)
if str(reservation) != 'none' and used > res:
a = "{0}: {1}\n\t\t".format(
zettaknight_utils.printcolors("Dataset", "OKBLUE"),
zettaknight_utils.printcolors(dataset, "FAIL"))
b = "{0}: {1}\n{2}: {3}\n{4}: {5}\n{6}: {7}".format(
zettaknight_utils.printcolors("Reservation", "OKBLUE"),
zettaknight_utils.printcolors(reservation, "OKGREEN"),
zettaknight_utils.printcolors("Total Used", "OKBLUE"),
zettaknight_utils.printcolors(used_friendly, "WARNING"),
zettaknight_utils.printcolors("Used by Snapshots",
"OKBLUE"),
zettaknight_utils.printcolors(usnap_friendly, "WARNING"),
zettaknight_utils.printcolors("Active Dataset size",
"OKBLUE"),
zettaknight_utils.printcolors(uds_friendly, "WARNING"))
c = zettaknight_utils.printcolors(
"\nDataset exceeds space reservation", "WARNING")
msg = "{0}{1}{2}".format(a, b, c)
ret[dataset]['Check Usage']['1'] = "{0}{1}".format(b, c)
else:
a = "{0}: {1}\n\t\t".format(
zettaknight_utils.printcolors("Dataset", "OKBLUE"),
zettaknight_utils.printcolors(dataset, "OKGREEN"))
b = "{0}: {1}\n{2}: {3}\n{4}: {5}\n{6}: {7}".format(
zettaknight_utils.printcolors("Reservation", "OKBLUE"),
zettaknight_utils.printcolors(reservation, "OKGREEN"),
zettaknight_utils.printcolors("Total Used", "OKBLUE"),
zettaknight_utils.printcolors(used_friendly, "OKGREEN"),
zettaknight_utils.printcolors("Used by Snapshots",
"OKBLUE"),
zettaknight_utils.printcolors(usnap_friendly, "OKGREEN"),
zettaknight_utils.printcolors("Active Dataset size",
"OKBLUE"),
zettaknight_utils.printcolors(uds_friendly, "OKGREEN"))
msg = "{0}{1}".format(a, b)
ret[dataset]['Check Usage']['0'] = b
return ret
def recover(snapshot, filename, relocate=None):
'''
'''
file = filename
snap = snapshot
dataset, snap_date = snap.split('@', 1)
find_snap = find_versions(dataset, file, "quiet")
find_snap_keys = find_snap.iterkeys()
snap_basedir = "/{0}/.zfs/snapshot".format(dataset)
active_basedir = "/{0}".format(dataset)
dir_flag = False
ret = {}
ret[dataset] = {}
if dataset in file.split("/"):
file = file.split(dataset, 1)[1]
last_ref = file.rsplit("/",1)[1]
if snap not in find_snap_keys:
try:
raise Exception("Recoverable versions of {0} not found in dataset {1}".format(file, dataset))
except Exception as e:
print(zettaknight_utils.printcolors(e, "FAIL"))
path_list1 = find_snap[snap]
path_list = []
if os.path.isdir("/{0}/{1}".format(dataset, file)):
dir_flag = True
for line in path_list1:
if line.startswith("-") or line.startswith("M"):
a, b = line.rsplit("/", 1)
if not dir_flag:
try:
if str(b) == str(last_ref):
path_list.append(line)
except:
if str(b) == str(file):
path_list.append(line)
pass
else:
if a.startswith("-"):
a, b = line.rsplit(dataset, 2)
if os.path.isdir("{0}/{1}{2}".format(snap_basedir, snap_date, b)):
dir_flag = True
path_list.append(line)
if len(path_list) > 1:
exc_out = ""
for i in path_list:
if exc_out:
exc_out = "{0}\n\t{1}".format(exc_out, i)
else:
exc_out = str(i)
err_msg = "Matching files/folders in snapshot:"
print(zettaknight_utils.printcolors("\nAmbiguous recover request. Multiple matches for {0}.".format(file), "FAIL"))
print("{0} {1}".format(zettaknight_utils.printcolors(err_msg, "FAIL"), zettaknight_utils.printcolors(snap, "OKBLUE")))
print("\t{0}".format(zettaknight_utils.printcolors(exc_out, "WARNING")))
print(zettaknight_utils.printcolors("Re-run with explicit path to file.\n", "FAIL"))
try:
raise Exception("Ambiguous file reference.")
except Exception as e:
print(zettaknight_utils.printcolors(e, "FAIL"))
sys.exit(0)
if len(path_list) < 1:
try:
raise Exception("No restorable files or folders identified. \nRe-run with explicit path to file?")
except Exception as e:
print(zettaknight_utils.printcolors(e, "FAIL"))
sys.exit(0)
a, p = path_list[0].split(dataset, 1)
path, dirs = p.split(file, 1)
if dir_flag:
bad, dir_path = path_list[0].split(file, 1)
list_dirs = dir_path.rsplit("/", 1)
try:
out_dir = list_dirs[0]
except:
out_dir = ""
pass
file_loc = "{0}/{1}/{2}{3}/".format(snap_basedir, snap_date, file, dir_path )
mkdir_cmd = "/bin/mkdir {0}/{1}.R".format(active_basedir, file)
mkdir_run = zettaknight_utils.spawn_job(mkdir_cmd)
mkdir_cmd = "/bin/mkdir -p {0}/{1}.R/{2}".format(active_basedir, file, dir_path)
mkdir_run = zettaknight_utils.spawn_job(mkdir_cmd)
out_path = "{0}{1}{2}.R{3}/".format(active_basedir, str(path), file, out_dir)
else:
file_loc = "{0}/{1}/{2}".format(snap_basedir, snap_date, p)
out_path = "{0}{1}{2}.R".format(active_basedir, str(path), file)
if relocate:
out_path = relocate
rec_cmd = "/bin/cp ""-r ""{0}"" ""{1}".format(file_loc, out_path)
rec_run = zettaknight_utils.spawn_job(rec_cmd)
rec_dict = eval(str(rec_run))
for k,v in rec_dict.iteritems():
if str(k) is "0":
print("Recover operation succeeded.")
print("\tFilename: {0}".format(zettaknight_utils.printcolors(file, "OKBLUE")))
print("\t\tFile(s) restored to version: {0}".format(zettaknight_utils.printcolors(snap_date, "OKGREEN")))
print("\t\tFile(s) restored to: {0}".format(zettaknight_utils.printcolors(out_path, "OKGREEN")))
if str(k) is not "0":
print("Recover operation failed.")
print("\tFilename: {0}".format(zettaknight_utils.printcolors(file, "OKBLUE")))
print("\t\tRestore failed with error: {0}".format(zettaknight_utils.printcolors(v, "FAIL")))
return
def create_zpool(pool=False,
disk_list=False,
raid=False,
luks=False,
slog=False,
create_config=False,
ldap=False,
recordsize=False,
ashift=False,
keyfile=False):
if zettaknight_globs.help_flag:
ret = """Create Zpool:
See help entry for create function.
zettaknight help create
"""
return ret
ret = {}
ret[pool] = {}
ret[pool]['Create Zpool'] = {}
if not raid:
raid = "12+2"
try:
disks, z_level = raid.split("+")
except Exception as e:
ret[pool]['Create Zpool'] = {
'1': "{0}\nargument raid must be in x+y format, i.e. 2+1".format(e)
}
zettaknight_utils.parse_output(ret)
sys.exit(0)
create_cmd = "bash {0} -d {1} -z {2}".format(
zettaknight_globs.zpool_create_script, disks, z_level)
if disk_list:
create_cmd = "{0} -f '{1}'".format(create_cmd, disk_list)
if pool:
create_cmd = "{0} -p '{1}'".format(create_cmd, pool)
if luks:
create_cmd = "{0} -l".format(create_cmd)
if slog:
create_cmd = "{0} -s '{1}'".format(create_cmd, slog)
if ldap:
create_cmd = "{0} -i".format(create_cmd)
if recordsize:
if any(i in recordsize for i in 'KM'):
create_cmd = "{0} -r {1}".format(create_cmd, recordsize)
else:
print(
zettaknight_utils.printcolors(
"Recordsize must be in number/unit format. ie. 1M, or 512K",
"FAIL"))
sys.exit(0)
if ashift:
create_cmd = "{0} -a {1}".format(create_cmd, ashift)
if keyfile:
create_cmd = "{0} -k {1}".format(create_cmd, keyfile)
try:
ret[pool]['Create Zpool'] = zettaknight_utils.spawn_job(create_cmd)
except Exception as e:
zettaknight_utils.zlog("{0}".format(e), "ERROR")
ret[pool]['Create Zpool']['1'] = e
return ret
