def create_snap(dataset, quiet=False):
    '''
    '''
 
    zettaknight_utils.check_quiet(quiet)
 
    snap = "{0}@{1}".format(dataset, zettaknight_globs.today_date)
    gerp_run = zettaknight_utils.pipe_this2("/sbin/zfs list -r -t snapshot -o name -H {0} | /bin/grep {1}".format(dataset, snap))
    gerp_out = gerp_run.stdout.read()
 
    if int(gerp_run.returncode) is not 0:
        ret = zettaknight_utils.spawn_job("/sbin/zfs snapshot -r {0}".format(snap))
 
    if int(gerp_run.returncode) == 0:
        ret = {0: "Snapshot {0} already exists.".format(snap)}
 
    for exit_status, output in ret.iteritems():
        if str(exit_status) == "0" and str(output) == "Job succeeded":
            ret[exit_status] = "Snapshot created: {0}".format(snap)
 
    if not quiet:
        snap_out = {}
        snap_out[dataset] = {}
        snap_out[dataset][inspect.stack()[0][3]] = ret
        zettaknight_utils.parse_output(snap_out)
 
    return ret
def check_group_quota(group):
    '''
    '''
    zettaknight_globs.nocolor_flag = True

    ret = {}
    dset = zettaknight_utils.pipe_this2(
        "zfs list -H -o name | grep -v {0}/ | grep {0}".format(group))
    dset = dset.stdout.read().strip()
    if dset:
        if str(group) in dset:
            usage_dict = check_usage(dset, "quiet")
    else:
        usage_dict = {}
        usage_dict[zettaknight_globs.fqdn] = {}
        usage_dict[zettaknight_globs.fqdn]['Check Group Quota'] = {
            1:
            "group: {0} does not have a Zettaknight controlled dataset.".
            format(group)
        }

    if str(dset) in usage_dict.iterkeys():
        if "0" in usage_dict[dset]['Check Usage'].iterkeys():
            usage_info = usage_dict[dset]['Check Usage']["0"]
            usage_info = usage_info.splitlines()
            for line in usage_info:
                if "Reservation" in line:
                    purchased = line.split(":")[1]
                    if str(purchased) == " none":
                        res = float(0.00)
                        res_unit = "G"
                    else:
                        res = float(re.sub("[A-Za-z]", "", purchased))
                        if "T" in line:
                            res_unit = "T"
                        elif "G" in line:
                            res_unit = "G"
                        elif "M" in line:
                            res_unit = "M"

                    #print("Purchased: {0}".format(purchased))
                if "Snapshots" in line:
                    snap_used = line.split(":")[1]
                    #print("Snapshot Data use: {0}".format(snap_used))
                if "Dataset" in line:
                    dset_used = line.split(":")[1]
                    #print("User Data use: {0}".format(dset_used))
                if "Total" in line:
                    tot_used = line.split(":")[1]
                    tot_used = float(re.sub("[A-Za-z]", "", tot_used))
                    free_size = "{0:.2f}{1}".format((res - tot_used), res_unit)
                    #print("Free Space Available: {0}".format(free_size))

            ret = "{0}:\t{4} available of {1} purchased. (Active User Data: {2}, Data in Snapshots: {3})".format(
                group, purchased, dset_used, snap_used, free_size)

    return ret
def add_dataset(dataset, **kwargs):
    '''if create_config=True is passed, a configuration file
    will be created for the dataset passed in to this function'''

    ret = {}
    ret[zettaknight_globs.fqdn] = {}
    question = False
    if 'create_config' in kwargs.iterkeys():
        create_config = kwargs['create_config']
    else:
        create_config = True
    if 'nfs' in kwargs.iterkeys():
        nfs = kwargs['nfs']
    else:
        nfs = True
    #append dataset to kwargs
    if dataset:
        kwargs['dataset'] = dataset

    gerp_run = zettaknight_utils.pipe_this2(
        "zfs list | /bin/grep {0}".format(dataset))
    gerp_out = gerp_run.stdout.read()

    #print(zettaknight_utils.printcolors("\nWill create {0} after completion of config file\n".format(dataset), "OKGREEN"))

    if create_config:
        ret[zettaknight_globs.fqdn]['Create Config'] = {}
        out1 = zettaknight_utils.create_config(**kwargs)
        ret[zettaknight_globs.fqdn]['Create Config'] = out1[
            zettaknight_globs.fqdn]['Create Config']

    #create dataset after commit of file
    if int(gerp_run.returncode) is not 0:
        create_cmd = "zfs create -p {0}".format(dataset)
        create_run = zettaknight_utils.spawn_job(create_cmd)
        ret[zettaknight_globs.fqdn]['add_dataset'] = create_run
    else:
        ret[zettaknight_globs.fqdn]['add_dataset'] = {
            '0': "{0} exists".format(dataset)
        }

    if nfs:
        question = zettaknight_utils.query_yes_no(
            "Would you like to create an NFS share for: {0}".format(dataset))
    if question:
        nfs_share = zettaknight_utils.query_return_item(
            "Where would you like to share it? <i.e 10.20.30.1/24 or my.server>"
        )
        ret[zettaknight_globs.fqdn]['NFS'] = {}
        ret[zettaknight_globs.fqdn]['NFS'] = zettaknight_utils.sharenfs(
            dataset, nfs_share)

    return ret
def add_dataset(*args, **kwargs):
    '''if create_config=True is passed, a configuration file
    will be created for the dataset passed in to this function'''
 
 
    #kwargs = {}
    arg_list = list(args)
 
    #kwargs = zettaknight_utils.create_kwargs_from_args(arg_list)
    #print("This is kwargs : {0}".format(kwargs))
 
    for arg in arg_list[0:]:
        if "=" in arg:
            k, v = arg.split("=", 1)
            kwargs[k] = v
            arg_list.remove(arg)
 
    dataset = arg_list[0]
    ret = {}
    ret[dataset] = {}
    
    gerp_run = zettaknight_utils.pipe_this2("zfs list | /bin/grep {0}".format(dataset))
    gerp_out = gerp_run.stdout.read()
    
    
    if 'create_config' not in kwargs.iterkeys():
        kwargs['create_config'] = True
        
    if kwargs['create_config'] == 'False':
        kwargs['create_config'] == False
        
    if kwargs['create_config']:
        print(zettaknight_utils.printcolors("\nWill create {0} after completion of config file\n".format(dataset), "OKGREEN"))
        conf_dict = {'dataset' : dataset}
        zettaknight_utils.create_config(**conf_dict)
    
    #create dataset after commit of file
    if int(gerp_run.returncode) is not 0:
        create_cmd = "zfs create -p {0}".format(dataset)
        create_run = zettaknight_utils.spawn_job(create_cmd)
        ret[dataset]['add_dataset'] = create_run
    else:
        ret[dataset]['add_dataset'] = {'0' : "{0} exists".format(dataset)}
        
    question = zettaknight_utils.query_yes_no("Would you like to create an NFS share for: {0}".format(dataset))
    if question:
        nfs_share = zettaknight_utils.query_return_item("Where would you like to share it? <i.e 10.20.30.1/24 or my.server>")
        zettaknight_utils.sharenfs(dataset, nfs_share)
 
    #zettaknight_utils.parse_output(ret)
 
    return ret
def check_group_quota(group):
    '''
    '''
    zettaknight_globs.nocolor_flag = True
 
    ret = {}
    dset = zettaknight_utils.pipe_this2("zfs list -H -o name | grep -v {0}/ | grep {0}".format(group))
    dset = dset.stdout.read().strip()
    if dset:
        if str(group) in dset:
            usage_dict = check_usage(dset, "quiet")
    else:
        usage_dict = {}
        usage_dict[zettaknight_globs.fqdn] = {}
        usage_dict[zettaknight_globs.fqdn]['Check Group Quota'] = {1: "group: {0} does not have a Zettaknight controlled dataset.".format(group)}
 
    if str(dset) in usage_dict.iterkeys():
        if "0" in usage_dict[dset]['Check Usage'].iterkeys():
            usage_info = usage_dict[dset]['Check Usage']["0"]
            usage_info = usage_info.splitlines()
            for line in usage_info:
                if "Reservation" in line:
                    purchased = line.split(":")[1]
                    if str(purchased) == " none":
                        res = float(0.00)
                        res_unit = "G"
                    else:
                        res = float(re.sub("[A-Za-z]", "", purchased))
                        if "T" in line:
                            res_unit = "T"
                        elif "G" in line:
                            res_unit = "G"
                        elif "M" in line:
                            res_unit = "M"
 
                    #print("Purchased: {0}".format(purchased))
                if "Snapshots" in line:
                    snap_used = line.split(":")[1]
                    #print("Snapshot Data use: {0}".format(snap_used))
                if "Dataset" in line:
                    dset_used = line.split(":")[1]
                    #print("User Data use: {0}".format(dset_used))
                if "Total" in line:
                    tot_used = line.split(":")[1]
                    tot_used = float(re.sub("[A-Za-z]", "", tot_used))
                    free_size = "{0:.2f}{1}".format((res - tot_used), res_unit)
                    #print("Free Space Available: {0}".format(free_size))
 
            ret = "{0}:\t{4} available of {1} purchased. (Active User Data: {2}, Data in Snapshots: {3})".format(group, purchased, dset_used, snap_used, free_size)
 
    return ret
Example #6
0
def add_dataset(dataset, **kwargs):
    '''if create_config=True is passed, a configuration file
    will be created for the dataset passed in to this function'''
    
    ret = {}
    ret[zettaknight_globs.fqdn] = {}
    question = False
    if 'create_config' in kwargs.iterkeys():
        create_config = kwargs['create_config']
    else:
        create_config = True
    if 'nfs' in kwargs.iterkeys():
        nfs = kwargs['nfs']
    else:
        nfs = True    
    #append dataset to kwargs
    if dataset:
        kwargs['dataset'] = dataset
    
    gerp_run = zettaknight_utils.pipe_this2("zfs list | /bin/grep {0}".format(dataset))
    gerp_out = gerp_run.stdout.read()
        
    #print(zettaknight_utils.printcolors("\nWill create {0} after completion of config file\n".format(dataset), "OKGREEN"))


    if create_config:
        ret[zettaknight_globs.fqdn]['Create Config'] = {}
        out1 = zettaknight_utils.create_config(**kwargs)
        ret[zettaknight_globs.fqdn]['Create Config'] = out1[zettaknight_globs.fqdn]['Create Config']

    #create dataset after commit of file
    if int(gerp_run.returncode) is not 0:
        create_cmd = "zfs create -p {0}".format(dataset)
        create_run = zettaknight_utils.spawn_job(create_cmd)
        ret[zettaknight_globs.fqdn]['add_dataset'] = create_run
    else:
        ret[zettaknight_globs.fqdn]['add_dataset'] = {'0' : "{0} exists".format(dataset)}
    
    if nfs:    
        question = zettaknight_utils.query_yes_no("Would you like to create an NFS share for: {0}".format(dataset))
    if question:
        nfs_share = zettaknight_utils.query_return_item("Where would you like to share it? <i.e 10.20.30.1/24 or my.server>")
        ret[zettaknight_globs.fqdn]['NFS'] = {}
        ret[zettaknight_globs.fqdn]['NFS'] = zettaknight_utils.sharenfs(dataset, nfs_share)
 
    return ret
def audit_last_snap(dataset):
    '''
    '''
    
    snaps = {}
    ret = {}
    ret[dataset] = {}
    job = inspect.stack()[0][3]
    snaplist_run = zettaknight_utils.pipe_this2("/sbin/zfs list -r -t snapshot -o name -H {0} | tail -2".format(dataset))
    snaplist_out = snaplist_run.stdout.read()
    if not snaplist_out:
        try:
            ret[dataset][job] = {}
            ret[dataset][job]['1'] = "No snapshots found."
            raise Exception
        except Exception as e:
            return ret
    
    snap_list = list(snaplist_out.split())
    if snap_list[0].startswith("cannot"):
        try:
            ret[dataset][job] = {}
            ret[dataset][job]['1'] = "{0}".format(snap_list[0])
            raise Exception
        except Exception as e:
            return ret
                
    if len(snap_list) == 2:
        diff_cmd = "/sbin/zfs diff {0} {1} -H | cat".format(snap_list[0], snap_list[1])
        job = "Audit of newly ingested snapshot {0}".format(snap_list[1])
    else:
        diff_cmd = "/sbin/zfs diff {0} -H | cat".format(snap_list[0])
        job = "Audit of newly ingested snapshot {0}".format(snap_list[0])
        
    diff_run = zettaknight_utils.pipe_this2("{0}".format(diff_cmd))
    diff_out = diff_run.stdout.read()
        
    if diff_out:
        mod_count = 0
        mod_list = ""
        add_count = 0
        add_list = ""
        del_count = 0
        del_list = ""
        for diff in diff_out.split('\n'):
            if diff.startswith("M"):
                mod_count += 1
                if mod_list:
                    mod_list = "{0}\n\t{1}".format(mod_list, diff)
                else:
                    mod_list = "\t{0}".format(str(diff))
            
            if diff.startswith("+"):
                add_count += 1
                if add_list:
                    add_list = "{0}\n\t{1}".format(add_list, diff)
                else:
                    add_list = "\t{0}".format(str(diff))
                    
            if diff.startswith("-"):
                del_count += 1
                if del_list:
                    del_list = "{0}\n\t{1}".format(del_list, diff)
                else:
                    del_list = "\t{0}".format(str(diff))
        
    ret[dataset][job] = {}
    job_out = ""

    if mod_list:
        job_out = "\nFile modifications in last ingested snapshot:\n{0}".format(mod_list)
    if add_list:
        job_out = "{0}\n\nNew Files in last ingested snapshot:\n{1}".format(job_out, add_list)
    if del_list:
        job_out = "{0}\n\nFile deletions in last ingested snapshot:\n{1}".format(job_out, del_list)
    
    job_out = "{0}\n\nChange counts in last ingested snapshot:\n\tModified Files: {1}\n\tNew Files: {2}\n\tDeleted Files: {3}".format(job_out, str(mod_count), str(add_count), str(del_count))
    
    ret[dataset][job]['0'] = job_out
    
    return ret
def audit_last_snap(dataset):
    '''
    '''

    if zettaknight_globs.help_flag:
        ret = """Last Snapshot Audit:

	Usage:
		zettaknight audit_last_snap <dataset>
	
    Returns an audit of all differences between the two most recent snapshots of a dataset.

    Required Arguments:
        dataset
            Specifies the dataset whose snapshots will be audited."""
        return ret

    snaps = {}
    ret = {}
    ret[dataset] = {}
    job = inspect.stack()[0][3]
    snaplist_run = zettaknight_utils.pipe_this2(
        "/sbin/zfs list -r -t snapshot -o name -H {0} | tail -2".format(
            dataset))
    snaplist_out = snaplist_run.stdout.read()
    if not snaplist_out:
        try:
            ret[dataset][job] = {}
            ret[dataset][job]['1'] = "No snapshots found."
            raise Exception
        except Exception as e:
            return ret

    snap_list = list(snaplist_out.split())
    if snap_list[0].startswith("cannot"):
        try:
            ret[dataset][job] = {}
            ret[dataset][job]['1'] = "{0}".format(snap_list[0])
            raise Exception
        except Exception as e:
            return ret

    if len(snap_list) == 2:
        diff_cmd = "/sbin/zfs diff {0} {1} -H | cat".format(
            snap_list[0], snap_list[1])
        job = "Audit of newly ingested snapshot {0}".format(snap_list[1])
    else:
        diff_cmd = "/sbin/zfs diff {0} -H | cat".format(snap_list[0])
        job = "Audit of newly ingested snapshot {0}".format(snap_list[0])

    diff_run = zettaknight_utils.pipe_this2("{0}".format(diff_cmd))
    diff_out = diff_run.stdout.read()

    if diff_out:
        mod_count = 0
        mod_list = ""
        add_count = 0
        add_list = ""
        del_count = 0
        del_list = ""
        for diff in diff_out.split('\n'):
            if diff.startswith("M"):
                mod_count += 1
                if mod_list:
                    mod_list = "{0}\n\t{1}".format(mod_list, diff)
                else:
                    mod_list = "\t{0}".format(str(diff))

            if diff.startswith("+"):
                add_count += 1
                if add_list:
                    add_list = "{0}\n\t{1}".format(add_list, diff)
                else:
                    add_list = "\t{0}".format(str(diff))

            if diff.startswith("-"):
                del_count += 1
                if del_list:
                    del_list = "{0}\n\t{1}".format(del_list, diff)
                else:
                    del_list = "\t{0}".format(str(diff))

    ret[dataset][job] = {}
    job_out = ""

    if mod_list:
        job_out = "\nFile modifications in last ingested snapshot:\n{0}".format(
            mod_list)
    if add_list:
        job_out = "{0}\n\nNew Files in last ingested snapshot:\n{1}".format(
            job_out, add_list)
    if del_list:
        job_out = "{0}\n\nFile deletions in last ingested snapshot:\n{1}".format(
            job_out, del_list)

    job_out = "{0}\n\nChange counts in last ingested snapshot:\n\tModified Files: {1}\n\tNew Files: {2}\n\tDeleted Files: {3}".format(
        job_out, str(mod_count), str(add_count), str(del_count))

    ret[dataset][job]['0'] = job_out

    return ret
Example #9
0
def first_step():
    ret = {}
    arg_dict = {}

    try:

        pool_name = "zfs_data"  #default pool name
        arg_dict["pool_name"] = pool_name

        ret[pool_name] = {}

        ############### creation of the zpool ################
        ######################################################

        #test if a zpool exists
        zpool_list_cmd = "/sbin/zpool list -H"
        zpool_list_run = zettaknight_utils.spawn_job(zpool_list_cmd)
        chk_code, chk_msg = zpool_list_run.popitem()
        if int(chk_code) != 0:  #if zpool list -H returns empty
            #query_return_item is defined in zettaknight_utils
            create_pool_question = zettaknight_utils.query_yes_no(
                "No pool detected, would you like to create one?")
            if create_pool_question:
                arg_dict["disk_list"] = zettaknight_utils.query_return_item(
                    "File containing all list of disks to be used for {0}, I.E /tmp/bunchofdisks.txt: "
                    .format(pool_name))
                arg_dict["raid"] = zettaknight_utils.query_return_item(
                    "Raid level to be used [1-3]. I.E. 4+1, 5+2, 1+1: ")
                arg_dict["ashift"] = zettaknight_utils.query_return_item(
                    "ashift value for {0}, value is 9 for 512 block disks and 12 for 4096 block disks: "
                    .format(pool_name))
                arg_dict["luks"] = zettaknight_utils.query_yes_no(
                    "encrypt data disks with LUKS? ")

                print("creating zpool: {0}".format(pool_name))
                create_output = zettaknight_zpools.create(
                    pool_name, **arg_dict)  #from zettaknight_zpools
            #ret[pool_name]['Zpool Create'] = create_output[pool_name]['Zpool Create'] #'NoneType' object has no attribute '__getitem__'
        else:
            print("a pool already exists, moving on")
        ######################################################
        ######################################################

        ################## ssh key creation  #################
        print("checking for {0}".format(zettaknight_globs.identity_file))
        if not os.path.isfile(zettaknight_globs.identity_file):
            print(
                zettaknight_utils.printcolors(
                    "zettaknight id file not found, generating.", "WARNING"))
            keygen_output = zettaknight_utils.ssh_keygen(
                zettaknight_globs.identity_file)
            ret[pool_name]['Generate SSH Keygen'] = keygen_output[
                zettaknight_globs.fqdn]['Generate SSH Key']
        else:
            print("{0} exists, moving on".format(
                zettaknight_globs.identity_file))
        ######################################################

        ############# backup luks headers ####################
        print("backing up luks headers for {0}".format(pool_name))
        luks_backup_output = zettaknight_utils.backup_luks_headers()
        ret[pool_name]['Backup Luks Headers'] = luks_backup_output[
            zettaknight_globs.fqdn]['Backup Luks Headers']
        ######################################################

        ######## create zettaknight configuration file #######
        if not os.path.isdir(zettaknight_globs.conf_dir_new
                             ):  #if new directory does not exists, make it
            print("creating {0}".format(zettaknight_globs.conf_dir_new))
            os.mkdir(zettaknight_globs.conf_dir_new)
        ######################################################

        ########### create zettaknight store #################
        ######################################################
        if not os.path.isdir("/{0}".format(
                zettaknight_globs.zettaknight_store)):
            dataset_check_run = zettaknight_utils.pipe_this2(
                "/sbin/zfs list -H | grep {0}".format(
                    zettaknight_globs.zettaknight_store))
            if int(dataset_check_run.returncode) != 0:
                print("creating configuration file for {0}".format(
                    zettaknight_globs.zettaknight_store))
                zettaknight_utils.create_config(
                    dataset=zettaknight_globs.zettaknight_store)

        gerp_run = zettaknight_utils.pipe_this2(
            "/sbin/zfs list -H | /bin/grep {0}".format(
                zettaknight_globs.zettaknight_store))
        if int(gerp_run.returncode) is not 0:
            zettaknight_zfs.add_dataset(zettaknight_globs.zettaknight_store)

        #backup files to store
        files = [
            "/etc/exports", "/etc/crypttab",
            "{0}".format(zettaknight_globs.config_file_new)
        ]
        for file in files:
            if os.path.exists(file):
                destination_dir = "/{0}/{1}".format(
                    zettaknight_globs.zettaknight_store, zettaknight_globs.fqdn
                )  #add leading slash, zfs_share defined
                filename = file.replace(
                    "/", ""
                )  #remove illegal characters from file path and save file as the concatenated version
                if not os.path.isdir(destination_dir):
                    os.mkdir(destination_dir)
                print("backing up {0} to {1}".format(file, destination_dir))
                shutil.copyfile(file, "{0}/{1}".format(destination_dir,
                                                       filename))
        ######################################################
        ######################################################

        ############ create the first dataset ################
        dataset = zettaknight_utils.query_return_list(
            "Datasets to be created on {0}: ".format(pool_name))
        for item in dataset:
            dset_args = []
            dataset_full = "{0}/{1}".format(pool_name, item)
            dset_args.append(dataset_full)
            dset_args.append("create_config=True")
            add_dset_output = zettaknight_zfs.add_dataset(
                *dset_args)  #from zettaknight_utils
            ret[pool_name]['Add dataset {0}'.format(
                dataset_full)] = add_dset_output[dataset_full]['add_dataset']
        ######################################################

        ###### create server to server transfer pathing ######
        replication_output = zettaknight_zfs.configure_replication(
        )  #from zettaknight_zfs
        for repl_job, repl_job_output in replication_output[
                dataset_full].itervalues():
            ret[pool_name]['{0}'.format(repl_job)] = repl_job_output
        ######################################################

    except Exception as e:
        print(zettaknight_utils.printcolors(e, "FAIL"))
        sys.exit(1)

    zettaknight_globs.zfs_conf = _get_conf()

    return ret
def first_step():
    ret = {}
    arg_dict = {}
    
    try:
        
        pool_name = "zfs_data" #default pool name 
        arg_dict["pool_name"] = pool_name
        
        ret[pool_name] = {}
        
        ############### creation of the zpool ################
        ######################################################
        
        #test if a zpool exists
        zpool_list_cmd = "/sbin/zpool list -H"
        zpool_list_run = zettaknight_utils.spawn_job(zpool_list_cmd)
        chk_code, chk_msg = zpool_list_run.popitem()
        if int(chk_code) != 0: #if zpool list -H returns empty
            #query_return_item is defined in zettaknight_utils
            create_pool_question = zettaknight_utils.query_yes_no("No pool detected, would you like to create one?")
            if create_pool_question:
                arg_dict["disk_list"] = zettaknight_utils.query_return_item("File containing all list of disks to be used for {0}, I.E /tmp/bunchofdisks.txt: ".format(pool_name))
                arg_dict["raid"] = zettaknight_utils.query_return_item("Raid level to be used [1-3]. I.E. 4+1, 5+2, 1+1: ")
                arg_dict["ashift"] = zettaknight_utils.query_return_item("ashift value for {0}, value is 9 for 512 block disks and 12 for 4096 block disks: ".format(pool_name))
                arg_dict["luks"] = zettaknight_utils.query_yes_no("encrypt data disks with LUKS? ")
        
                print("creating zpool: {0}".format(pool_name))
                create_output = zettaknight_zpools.create(pool_name, **arg_dict) #from zettaknight_zpools
            #ret[pool_name]['Zpool Create'] = create_output[pool_name]['Zpool Create'] #'NoneType' object has no attribute '__getitem__'
        else:
            print("a pool already exists, moving on")
        ######################################################
        ######################################################
        
        ################## ssh key creation  #################
        print("checking for {0}".format(zettaknight_globs.identity_file))
        if not os.path.isfile(zettaknight_globs.identity_file):
            print(zettaknight_utils.printcolors("zettaknight id file not found, generating.", "WARNING"))
            keygen_output = zettaknight_utils.ssh_keygen(zettaknight_globs.identity_file)
            ret[pool_name]['Generate SSH Keygen'] = keygen_output[zettaknight_globs.fqdn]['Generate SSH Key']
        else:
            print("{0} exists, moving on".format(zettaknight_globs.identity_file))
        ######################################################
        
        ############# backup luks headers ####################
        print("backing up luks headers for {0}".format(pool_name))
        luks_backup_output = zettaknight_utils.backup_luks_headers()
        ret[pool_name]['Backup Luks Headers'] = luks_backup_output[zettaknight_globs.fqdn]['Backup Luks Headers']
        ######################################################
        
        
        ######## create zettaknight configuration file #######
        if not os.path.isdir(zettaknight_globs.conf_dir_new): #if new directory does not exists, make it
            print("creating {0}".format(zettaknight_globs.conf_dir_new))
            os.mkdir(zettaknight_globs.conf_dir_new)
        ######################################################
        
        ########### create zettaknight store #################
        ######################################################
        if not os.path.isdir("/{0}".format(zettaknight_globs.zettaknight_store)):
            dataset_check_run = zettaknight_utils.pipe_this2("/sbin/zfs list -H | grep {0}".format(zettaknight_globs.zettaknight_store))
            if int(dataset_check_run.returncode) != 0:
                print("creating configuration file for {0}".format(zettaknight_globs.zettaknight_store))
                zettaknight_utils.create_config(dataset=zettaknight_globs.zettaknight_store)
                
        gerp_run = zettaknight_utils.pipe_this2("/sbin/zfs list -H | /bin/grep {0}".format(zettaknight_globs.zettaknight_store))
        if int(gerp_run.returncode) is not 0:
            zettaknight_zfs.add_dataset(zettaknight_globs.zettaknight_store)
        
        
        #backup files to store
        files = ["/etc/exports", "/etc/crypttab", "{0}".format(zettaknight_globs.config_file_new)]
        for file in files:
            if os.path.exists(file):
                destination_dir = "/{0}/{1}".format(zettaknight_globs.zettaknight_store, zettaknight_globs.fqdn) #add leading slash, zfs_share defined
                filename = file.replace("/", "") #remove illegal characters from file path and save file as the concatenated version
                if not os.path.isdir(destination_dir):
                    os.mkdir(destination_dir)
                print("backing up {0} to {1}".format(file, destination_dir))
                shutil.copyfile(file, "{0}/{1}".format(destination_dir, filename))
        ######################################################
        ######################################################
        
        ############ create the first dataset ################
        dataset = zettaknight_utils.query_return_list("Datasets to be created on {0}: ".format(pool_name))
        for item in dataset:
            dset_args = []
            dataset_full = "{0}/{1}".format(pool_name, item)
            dset_args.append(dataset_full)
            dset_args.append("create_config=True")
            add_dset_output = zettaknight_zfs.add_dataset(*dset_args) #from zettaknight_utils
            ret[pool_name]['Add dataset {0}'.format(dataset_full)] = add_dset_output[dataset_full]['add_dataset']
        ######################################################
        
        ###### create server to server transfer pathing ######
        replication_output = zettaknight_zfs.configure_replication() #from zettaknight_zfs
        for repl_job, repl_job_output in replication_output[dataset_full].itervalues():
            ret[pool_name]['{0}'.format(repl_job)] = repl_job_output
        ######################################################
    
    except Exception as e:
        print(zettaknight_utils.printcolors(e, "FAIL"))
        sys.exit(1)
    
    zettaknight_globs.zfs_conf = _get_conf()
    
    return ret