def test_send_raw(self, zpools): """Checks if raw_send works""" fs0, fs1 = zpools fs0.destroy(force=True) fs1.destroy(force=True) raw_send = ['yes'] config = [{'name': fs0.name, 'dest': [fs1.name], 'raw_send': raw_send}] zfs.create('{:s}/sub1'.format(fs0.name), props={'compression': 'gzip'}) zfs.create('{:s}/sub2'.format(fs0.name), props={'compression': 'lz4'}) zfs.create('{:s}/sub3'.format(fs0.name), props={'compression': 'gzip'}) zfs.create('{:s}/sub3/abc'.format(fs0.name)) zfs.create('{:s}/sub3/abc_abc'.format(fs0.name)) zfs.create('{:s}/sub3/efg'.format(fs0.name)) fs0.snapshot('snap', recursive=True) send_config(config) fs0_children = set([ child.name.replace(fs0.name, '') for child in zfs.find(fs0.name, types=['all'])[1:] ]) fs1_children = set([ child.name.replace(fs1.name, '') for child in zfs.find(fs1.name, types=['all'])[1:] ]) assert set(fs0_children) == set(fs1_children)
def test_send_compress(self, zpools): """Checks if send_snap totally replicates a filesystem""" fs1, fs0 = zpools # here fs0 is the remote pool ssh = fs0.ssh fs0.destroy(force=True) fs1.destroy(force=True) fs0.snapshot('snap0') zfs.create('{:s}/sub1'.format(fs0.name), ssh=ssh) fs0.snapshot('snap1', recursive=True) zfs.create('{:s}/sub2'.format(fs0.name), ssh=ssh) fs0.snapshot('snap2', recursive=True) fs0.snapshot('snap3', recursive=True) zfs.create('{:s}/sub2/abc'.format(fs0.name), ssh=ssh) fs0.snapshot('snap4', recursive=True) fs0.snapshot('snap5', recursive=True) for compression in ['none', 'lzop', 'lz4']: fs1.destroy(force=True) config = [{'name': 'ssh:{:d}:{}'.format(PORT, fs0), 'key': KEY, 'dest': [fs1.name], 'compress': [compression]}] send_config(config) fs0_children = [child.name.replace(fs0.name, '') for child in zfs.find(fs0.name, types=['all'], ssh=ssh)[1:]] fs1_children = [child.name.replace(fs1.name, '') for child in zfs.find(fs1.name, types=['all'])[1:]] assert set(fs0_children) == set(fs1_children)
def test_send_exclude(self, zpools): """Checks if exclude rules work""" fs0, fs1 = zpools fs0.destroy(force=True) fs1.destroy(force=True) exclude = ['*/sub1', '*/sub3/abc', '*/sub3/efg'] config = [{'name': fs0.name, 'dest': [fs1.name], 'exclude': [exclude]}] zfs.create('{:s}/sub1'.format(fs0.name)) zfs.create('{:s}/sub2'.format(fs0.name)) zfs.create('{:s}/sub3'.format(fs0.name)) zfs.create('{:s}/sub3/abc'.format(fs0.name)) zfs.create('{:s}/sub3/abc_abc'.format(fs0.name)) zfs.create('{:s}/sub3/efg'.format(fs0.name)) fs0.snapshot('snap', recursive=True) send_config(config) fs0_children = set([ child.name.replace(fs0.name, '') for child in zfs.find(fs0.name, types=['all'])[1:] ]) fs1_children = set([ child.name.replace(fs1.name, '') for child in zfs.find(fs1.name, types=['all'])[1:] ]) # remove unwanted datasets/snapshots for match in exclude: fs0_children -= set(fnmatch.filter(fs0_children, match)) fs0_children -= set(fnmatch.filter(fs0_children, match + '@snap')) assert set(fs0_children) == set(fs1_children)
def test_send_delete_old(self, zpools): fs0, fs1 = zpools ssh = fs1.ssh # Delete old snapshot on source fs0.snapshots()[0].destroy(force=True) fs0.snapshot('snap7', recursive=True) config = [{ 'name': fs0.name, 'dest': ['ssh:{:d}:{}'.format(PORT, fs1)], 'dest_keys': [KEY], 'compress': None }] send_config(config) fs0_children = [ child.name.replace(fs0.name, '') for child in zfs.find(fs0.name, types=['all'])[1:] ] fs1_children = [ child.name.replace(fs1.name, '') for child in zfs.find(fs1.name, types=['all'], ssh=ssh)[1:] ] assert not (set(fs0_children) == set(fs1_children)) # Assert that snap0 was not deleted from fs1 for child in set(fs1_children) - set(fs0_children): assert child.endswith('snap0')
def test_send_exclude(self, zpools): """Checks if send_snap totally replicates a filesystem""" fs1, fs0 = zpools # here fs0 is the remote pool ssh = fs0.ssh fs0.destroy(force=True) fs1.destroy(force=True) exclude = ['*/sub1', '*/sub3/abc', '*/sub3/efg'] config = [{'name': 'ssh:{:d}:{}'.format(PORT, fs0), 'dest': [fs1.name], 'exclude': [exclude]}] zfs.create('{:s}/sub1'.format(fs0.name), ssh=ssh) zfs.create('{:s}/sub2'.format(fs0.name), ssh=ssh) zfs.create('{:s}/sub3'.format(fs0.name), ssh=ssh) zfs.create('{:s}/sub3/abc'.format(fs0.name), ssh=ssh) zfs.create('{:s}/sub3/abc_abc'.format(fs0.name), ssh=ssh) zfs.create('{:s}/sub3/efg'.format(fs0.name), ssh=ssh) fs0.snapshot('snap', recursive=True) send_config(config) fs0_children = set([child.name.replace(fs0.name, '') for child in zfs.find(fs0.name, types=['all'], ssh=ssh)[1:]]) fs1_children = set([child.name.replace(fs1.name, '') for child in zfs.find(fs1.name, types=['all'])[1:]]) # remove unwanted datasets/snapshots for match in exclude: fs0_children -= set(fnmatch.filter(fs0_children, match)) fs0_children -= set(fnmatch.filter(fs0_children, match + '@snap')) assert set(fs0_children) == set(fs1_children)
def test_send_delete_sub(self, zpools): fs0, fs1 = zpools ssh = fs1.ssh # Delete subfilesystems sub3 = fs1.filesystems()[-1] sub3.destroy(force=True) fs0.snapshot('snap6', recursive=True) sub2 = fs1.filesystems()[-1] sub2.destroy(force=True) config = [{ 'name': fs0.name, 'dest': ['ssh:{:d}:{}'.format(PORT, fs1)], 'dest_keys': [KEY], 'compress': None }] send_config(config) fs0_children = [ child.name.replace(fs0.name, '') for child in zfs.find(fs0.name, types=['all'])[1:] ] fs1_children = [ child.name.replace(fs1.name, '') for child in zfs.find(fs1.name, types=['all'], ssh=ssh)[1:] ] assert set(fs0_children) == set(fs1_children)
def test_send_full(self, zpools): """Checks if send_snap totally replicates a filesystem""" fs0, fs1 = zpools fs0.destroy(force=True) fs1.destroy(force=True) config = [{'name': fs0.name, 'dest': [fs1.name]}] fs0.snapshot('snap0') zfs.create('{:s}/sub1'.format(fs0.name)) fs0.snapshot('snap1', recursive=True) zfs.create('{:s}/sub2'.format(fs0.name)) fs0.snapshot('snap2', recursive=True) zfs.create('{:s}/sub3'.format(fs0.name)) fs0.snapshot('snap3', recursive=True) fs0.snapshot('snap4', recursive=True) fs0.snapshot('snap5', recursive=True) zfs.create('{:s}/sub3/abc'.format(fs0.name)) fs0.snapshot('snap6', recursive=True) zfs.create('{:s}/sub3/abc_abc'.format(fs0.name)) fs0.snapshot('snap7', recursive=True) zfs.create('{:s}/sub3/efg'.format(fs0.name)) fs0.snapshot('snap8', recursive=True) fs0.snapshot('snap9', recursive=True) send_config(config) fs0_children = [ child.name.replace(fs0.name, '') for child in zfs.find(fs0.name, types=['all'])[1:] ] fs1_children = [ child.name.replace(fs1.name, '') for child in zfs.find(fs1.name, types=['all'])[1:] ] assert set(fs0_children) == set(fs1_children)
def test_send_delete_snapshot(self, zpools): fs0, fs1 = zpools config = [{'name': fs0.name, 'dest': [fs1.name]}] # Delete recent snapshots on dest fs1.snapshots()[-1].destroy(force=True) fs1.snapshots()[-1].destroy(force=True) send_config(config) fs0_children = [ child.name.replace(fs0.name, '') for child in zfs.find(fs0.name, types=['all'])[1:] ] fs1_children = [ child.name.replace(fs1.name, '') for child in zfs.find(fs1.name, types=['all'])[1:] ] assert set(fs0_children) == set(fs1_children) # Delete recent snapshot on source fs0.snapshot('snap4', recursive=True) send_config(config) fs0.snapshots()[-1].destroy(force=True) fs0.snapshot('snap5', recursive=True) send_config(config) fs0_children = [ child.name.replace(fs0.name, '') for child in zfs.find(fs0.name, types=['all'])[1:] ] fs1_children = [ child.name.replace(fs1.name, '') for child in zfs.find(fs1.name, types=['all'])[1:] ] assert set(fs0_children) == set(fs1_children)
def test_create_new(self, zpools, config_send): """Tests pyznap over 10 years and checks if newly created filesystems are correctly replicated""" fs0, fs1 = zpools ssh = fs1.ssh fs0.destroy(force=True) fs1.destroy(force=True) # have to start at 1969 as faketime only goes from 1969 to 2068 dates = [datetime(1969 + i, 1, 1) for i in range(10)] for n,date in enumerate(dates): # at every step create a new subfilesystem zfs.create('{:s}/sub{:d}'.format(fs0.name, n)) faketime = ['faketime', date.strftime('%y-%m-%d %H:%M:%S')] pyznap_take = faketime + ['pyznap', '--config', config_send, 'snap', '--take'] pyznap_clean = faketime + ['pyznap', '--config', config_send, 'snap', '--clean'] pyznap_send = faketime + ['pyznap', '--config', config_send, 'send'] # take, send & clean snaps every 1y _, _ = Popen(pyznap_take).communicate() _, _ = Popen(pyznap_send).communicate() _, _ = Popen(pyznap_clean).communicate() # get all snapshots on fs0 snapshots = {'frequent': [], 'hourly': [], 'daily': [], 'weekly': [], 'monthly': [], 'yearly': []} for snap in fs0.snapshots(): snap_type = snap.name.split('_')[-1] snapshots[snap_type].append(snap) # check if there are not too many snapshots taken for snap_type, snaps in snapshots.items(): assert len(snaps) <= SNAPSHOTS_REF[snap_type] # check if after N_FREQUENT runs there are N_FREQUENT 'frequent' snapshots if n+1 >= N_FREQUENT: assert len(snapshots['frequent']) == SNAPSHOTS_REF['frequent'] # check if after N-HOURLY runs there are N-HOURLY 'hourly' snapshots if n+1 >= N_HOURLY: assert len(snapshots['hourly']) == SNAPSHOTS_REF['hourly'] # check if after N_DAILY runs there are N_DAILY 'daily' snapshots if n+1 >= N_DAILY: assert len(snapshots['daily']) == SNAPSHOTS_REF['daily'] # check if after N_WEEKLY runs there are N_WEEKLY 'weekly' snapshots if n+1 >= N_WEEKLY: assert len(snapshots['weekly']) == SNAPSHOTS_REF['weekly'] # check if after N_MONTHLY runs there are N_MONTHLY 'monthly' snapshots if n+1 >= N_MONTHLY: assert len(snapshots['monthly']) == SNAPSHOTS_REF['monthly'] # check if after N_YEARLY runs there are N_YEARLY 'yearly' snapshots if n+1 >= N_YEARLY: assert len(snapshots['yearly']) == SNAPSHOTS_REF['yearly'] # check if filesystem is completely replicated on dest fs0_children = [child.name.replace(fs0.name, '') for child in zfs.find(fs0.name, types=['all'])[1:]] fs1_children = [child.name.replace(fs1.name, '') for child in zfs.find(fs1.name, types=['all'], ssh=ssh)[1:]] assert set(fs0_children) == set(fs1_children)
def latest_snapshots(fsname): logger = logging.getLogger(__name__) # logger.debug('Cleaning snapshots on {}...'.format(filesystem)) snapshots = {'frequent': [], 'hourly': [], 'daily': [], 'weekly': [], 'monthly': [], 'yearly': []} # catch exception if dataset was destroyed since pyznap was started #try ssh = None children = zfs.find(path=fsname, types=['filesystem', 'volume'], ssh=ssh) fs_snapshots = children[0].snapshots() #except (DatasetNotFoundError, DatasetBusyError) as err #except as err: # logger.error('Error while opening {}: {}...'.format(filesystem, err)) # return 1 #from pprint import pprint #pprint(fs_snapshots) # Ignore snapshots not taken with pyznap or sanoid for snap in fs_snapshots[:]: # make a copy of fs_snapshots if not snap.name.split('@')[1].startswith(('pyznap', 'autosnap')): fs_snapshots.remove(snap) #pprint(fs_snapshots) latest = str(fs_snapshots[-1]) import subprocess command = str("zfs get -Hpo value creation " + latest) data = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=True) stdout, stderr = data.communicate() return int(stdout)
def take_config(config): """Takes snapshots according to strategy given in config. Parameters: ---------- config : {list of dict} Full config list containing all strategies for different filesystems """ logger = logging.getLogger(__name__) logger.info('Taking snapshots...') for conf in config: if not conf.get('snap', None): continue name = conf['name'] try: _type, fsname, user, host, port = parse_name(name) except ValueError as err: logger.error('Could not parse {:s}: {}...'.format(name, err)) continue if _type == 'ssh': try: ssh = SSH(user, host, port=port, key=conf['key']) except (FileNotFoundError, SSHException): continue name_log = '{:s}@{:s}:{:s}'.format(user, host, fsname) else: ssh = None name_log = fsname try: # Children includes the base filesystem (named 'fsname') children = zfs.find(path=fsname, types=['filesystem', 'volume'], ssh=ssh) except DatasetNotFoundError as err: logger.error('Dataset {:s} does not exist...'.format(name_log)) continue except ValueError as err: logger.error(err) continue except CalledProcessError as err: logger.error('Error while opening {:s}: \'{:s}\'...'.format( name_log, err.stderr.rstrip().decode())) continue else: # Take recursive snapshot of parent filesystem take_filesystem(children[0], conf) # Take snapshot of all children that don't have all snapshots yet for child in children[1:]: take_filesystem(child, conf) finally: if ssh: ssh.close()
def test_send_delete_snapshot(self, zpools): fs0, fs1 = zpools ssh = fs1.ssh # Delete recent snapshots on dest fs1.snapshots()[-1].destroy(force=True) fs1.snapshots()[-1].destroy(force=True) config = [{ 'name': fs0.name, 'dest': ['ssh:{:d}:{}'.format(PORT, fs1)], 'dest_keys': [KEY], 'compress': None }] send_config(config) fs0_children = [ child.name.replace(fs0.name, '') for child in zfs.find(fs0.name, types=['all'])[1:] ] fs1_children = [ child.name.replace(fs1.name, '') for child in zfs.find(fs1.name, types=['all'], ssh=ssh)[1:] ] assert set(fs0_children) == set(fs1_children) # Delete recent snapshot on source fs0.snapshot('snap4', recursive=True) send_config(config) fs0.snapshots()[-1].destroy(force=True) fs0.snapshot('snap5', recursive=True) config = [{ 'name': fs0.name, 'dest': ['ssh:{:d}:{}'.format(PORT, fs1)], 'dest_keys': [KEY], 'compress': None }] send_config(config) fs0_children = [ child.name.replace(fs0.name, '') for child in zfs.find(fs0.name, types=['all'])[1:] ] fs1_children = [ child.name.replace(fs1.name, '') for child in zfs.find(fs1.name, types=['all'], ssh=ssh)[1:] ] assert set(fs0_children) == set(fs1_children)
def test_send_delete_old(self, zpools): fs0, fs1 = zpools config = [{'name': fs0.name, 'dest': [fs1.name]}] # Delete old snapshot on source fs0.snapshots()[0].destroy(force=True) fs0.snapshot('snap7', recursive=True) send_config(config) fs0_children = [ child.name.replace(fs0.name, '') for child in zfs.find(fs0.name, types=['all'])[1:] ] fs1_children = [ child.name.replace(fs1.name, '') for child in zfs.find(fs1.name, types=['all'])[1:] ] assert not (set(fs0_children) == set(fs1_children)) # Assert that snap0 was not deleted from fs1 for child in set(fs1_children) - set(fs0_children): assert child.endswith('snap0')
def test_send_delete_sub(self, zpools): fs0, fs1 = zpools config = [{'name': fs0.name, 'dest': [fs1.name]}] # Delete subfilesystems sub3 = fs1.filesystems()[-1] sub3.destroy(force=True) fs0.snapshot('snap6', recursive=True) sub2 = fs1.filesystems()[-1] sub2.destroy(force=True) send_config(config) fs0_children = [ child.name.replace(fs0.name, '') for child in zfs.find(fs0.name, types=['all'])[1:] ] fs1_children = [ child.name.replace(fs1.name, '') for child in zfs.find(fs1.name, types=['all'])[1:] ] assert set(fs0_children) == set(fs1_children)
def test_send_incremental(self, zpools): fs1, fs0 = zpools # here fs0 is the remote pool ssh = fs0.ssh fs0.destroy(force=True) fs1.destroy(force=True) fs0.snapshot('snap0', recursive=True) zfs.create('{:s}/sub1'.format(fs0.name), ssh=ssh) fs0.snapshot('snap1', recursive=True) config = [{'name': 'ssh:{:d}:{}'.format(PORT, fs0), 'key': KEY, 'dest': [fs1.name], 'compress': None}] send_config(config) fs0_children = [child.name.replace(fs0.name, '') for child in zfs.find(fs0.name, types=['all'], ssh=ssh)[1:]] fs1_children = [child.name.replace(fs1.name, '') for child in zfs.find(fs1.name, types=['all'])[1:]] assert set(fs0_children) == set(fs1_children) zfs.create('{:s}/sub2'.format(fs0.name), ssh=ssh) fs0.snapshot('snap2', recursive=True) config = [{'name': 'ssh:{:d}:{}'.format(PORT, fs0), 'key': KEY, 'dest': [fs1.name], 'compress': None}] send_config(config) fs0_children = [child.name.replace(fs0.name, '') for child in zfs.find(fs0.name, types=['all'], ssh=ssh)[1:]] fs1_children = [child.name.replace(fs1.name, '') for child in zfs.find(fs1.name, types=['all'])[1:]] assert set(fs0_children) == set(fs1_children) zfs.create('{:s}/sub3'.format(fs0.name), ssh=ssh) fs0.snapshot('snap3', recursive=True) config = [{'name': 'ssh:{:d}:{}'.format(PORT, fs0), 'key': KEY, 'dest': [fs1.name], 'compress': None}] send_config(config) fs0_children = [child.name.replace(fs0.name, '') for child in zfs.find(fs0.name, types=['all'], ssh=ssh)[1:]] fs1_children = [child.name.replace(fs1.name, '') for child in zfs.find(fs1.name, types=['all'])[1:]] assert set(fs0_children) == set(fs1_children)
def count_snapshots(fsname,snap_freq): logger = logging.getLogger(__name__) # logger.debug('Cleaning snapshots on {}...'.format(filesystem)) snapshots = {'frequent': [], 'hourly': [], 'daily': [], 'weekly': [], 'monthly': [], 'yearly': []} ssh = None children = zfs.find(path=fsname, types=['filesystem', 'volume'], ssh=ssh) fs_snapshots = children[0].snapshots() # catch exception if dataset was destroyed since pyznap was started try: fs_snapshots = children[0].snapshots() except (DatasetNotFoundError, DatasetBusyError) as err: logger.error('Error while opening {}: {}...'.format(filesystem, err)) return 1 # categorize snapshots for snap in fs_snapshots: # Ignore snapshots not taken with pyznap or sanoid if not snap.name.split('@')[1].startswith(('pyznap', 'autosnap')): continue try: snap_type = snap.name.split('_')[-1] snapshots[snap_type].append(snap) except (ValueError, KeyError): continue # Reverse sort by time taken for snaps in snapshots.values(): snaps.reverse() #print(snaps) # Count all snapshots if snap_freq == "total": total = snapshots.values() total_length = sum(len(row) for row in snapshots.values()) return(total_length) else: # Count snapshots of a specific frequency (frequent, hourly, daily, ...) count = len(snapshots[snap_freq]) return(count) for snap in snapshots[snap_freq]: return(snap)
def test_send_incremental(self, zpools): fs0, fs1 = zpools fs0.destroy(force=True) fs1.destroy(force=True) config = [{'name': fs0.name, 'dest': [fs1.name]}] fs0.snapshot('snap0', recursive=True) zfs.create('{:s}/sub1'.format(fs0.name)) fs0.snapshot('snap1', recursive=True) send_config(config) fs0_children = [ child.name.replace(fs0.name, '') for child in zfs.find(fs0.name, types=['all'])[1:] ] fs1_children = [ child.name.replace(fs1.name, '') for child in zfs.find(fs1.name, types=['all'])[1:] ] assert set(fs0_children) == set(fs1_children) zfs.create('{:s}/sub2'.format(fs0.name)) fs0.snapshot('snap2', recursive=True) send_config(config) fs0_children = [ child.name.replace(fs0.name, '') for child in zfs.find(fs0.name, types=['all'])[1:] ] fs1_children = [ child.name.replace(fs1.name, '') for child in zfs.find(fs1.name, types=['all'])[1:] ] assert set(fs0_children) == set(fs1_children) zfs.create('{:s}/sub3'.format(fs0.name)) fs0.snapshot('snap3', recursive=True) send_config(config) fs0_children = [ child.name.replace(fs0.name, '') for child in zfs.find(fs0.name, types=['all'])[1:] ] fs1_children = [ child.name.replace(fs1.name, '') for child in zfs.find(fs1.name, types=['all'])[1:] ] assert set(fs0_children) == set(fs1_children)
def fill_harddrive(args): files = get_snapshot_files(find_snapshot_files()) blacklist = args.blacklist whitelist = args.whitelist if args.append and (args.whitelist is not None): # Only remove files we will end up writing again files = [file for file in files if True in [acceptedItem in str(file) for acceptedItem in args.whitelist]] for each in files: print("Removing old backup file: {}".format(str(each))) remove(each) source_children = zfs.find(path="tank", types=['filesystem', 'volume']) #Source children is literally just the filesystems in TANK #a child is just one filesystem #Apply whitelist to source_children if whitelist is not None: source_children = [child for child in source_children if True in [acceptedItem in str(child) for acceptedItem in args.whitelist]] #Apply blacklist to source_children if blacklist is not None: source_children = [child for child in source_children if not True in [excludedItem in str(child) for excludedItem in args.blacklist]] """ for child in source_children: #if ("brendon" not in str(child)) and ("vmbackup" not in str(child)): #if str(child) not in blacklist: if not True in (excludedItem in str(child) for excludedItem in blacklist): print("Backing up {}!".format(str(child))) else: print("Not backing up {}...".format(str(child))) """ with open("/mnt/snailback/report/noMatch.txt", "w") as no_match_file: picker = backup_picker() for child in source_children: common_snap_name = get_most_recent_common_snapshot(child) if common_snap_name: # There is a common snapshot for this filesystem, send an incremental stream print("Backup: " + str(child) + "@" + str(common_snap_name)) common_snap = None for snapshot in child.snapshots(): if common_snap_name in snapshot.snapname(): common_snap = snapshot print("Source: " + str(common_snap)) snapshot = child.snapshots()[-1] if args.dryrun: picker.add_backup(snapshot, common_snap) else: filename = '/mnt/snailback/' + snapshot.name.split('@')[0].split('/')[-1] + '.gzip' snapshot.send_to_file(filename, base=common_snap, intermediates=True) else: # There is not common snapshot for this filesystem, send the latest monthly snapshot snapshot = get_most_recent(child.snapshots(), "monthly") if args.dryrun: picker.add_backup(snapshot) else: no_match_file.write(str(snapshot.name) + "\n") filename = '/mnt/snailback/' + snapshot.name.split('@')[0].split('/')[-1] + '.gzip' snapshot.send_to_file(filename) print("Found {} total backups to make".format(str(picker.num_children))) print("Need {} bytes disk space to backup".format(str(picker.size_total))) print("Need {} KB disk space to backup".format(str(picker.size_total/1000))) print("Need {} MB disk space to backup".format(str(picker.size_total/1000000))) print("Need {} GB disk space to backup".format(str(picker.size_total/1000000000))) print("Need {} TB disk space to backup".format(str(picker.size_total/1000000000000))) picker.show_data()
def send_config(config): """Tries to sync all entries in the config to their dest. Finds all children of the filesystem and calls send_filesystem on each of them. Parameters: ---------- config : {list of dict} Full config list containing all strategies for different filesystems """ logger = logging.getLogger(__name__) logger.info('Sending snapshots...') for conf in config: if not conf.get('dest', None): continue backup_source = conf['name'] try: _type, source_name, user, host, port = parse_name(backup_source) except ValueError as err: logger.error('Could not parse {:s}: {}...'.format( backup_source, err)) continue # if source is remote, open ssh connection if _type == 'ssh': key = conf['key'] if conf.get('key', None) else None compress = conf['compress'].pop(0) if conf.get('compress', None) else 'lzop' try: ssh_source = SSH(user, host, port=port, key=key, compress=compress) except (FileNotFoundError, SSHException): continue source_name_log = '{:s}@{:s}:{:s}'.format(user, host, source_name) else: ssh_source = None source_name_log = source_name try: # Children includes the base filesystem (named 'source_name') source_children = zfs.find(path=source_name, types=['filesystem', 'volume'], ssh=ssh_source) except DatasetNotFoundError as err: logger.error( 'Source {:s} does not exist...'.format(source_name_log)) continue except ValueError as err: logger.error(err) continue except CalledProcessError as err: logger.error('Error while opening source {:s}: \'{:s}\'...'.format( source_name_log, err.stderr.rstrip())) continue # Send to every backup destination for backup_dest in conf['dest']: try: _type, dest_name, user, host, port = parse_name(backup_dest) except ValueError as err: logger.error('Could not parse {:s}: {}...'.format( backup_dest, err)) continue # if dest is remote, open ssh connection if _type == 'ssh': dest_key = conf['dest_keys'].pop(0) if conf.get( 'dest_keys', None) else None # if 'ssh_source' is set, then 'compress' is already set and we use same compression for both source and dest # if not then we take the next entry in config if not ssh_source: compress = conf['compress'].pop(0) if conf.get( 'compress', None) else 'lzop' try: ssh_dest = SSH(user, host, port=port, key=dest_key, compress=compress) except (FileNotFoundError, SSHException): continue dest_name_log = '{:s}@{:s}:{:s}'.format(user, host, dest_name) else: ssh_dest = None dest_name_log = dest_name # get exclude rules exclude = conf['exclude'].pop(0) if conf.get('exclude', None) else [] # check if raw send was requested raw = conf['raw_send'].pop(0) if conf.get('raw_send', None) else False # Check if base destination filesystem exists, if not do not send try: zfs.open(dest_name, ssh=ssh_dest) except DatasetNotFoundError: logger.error( 'Destination {:s} does not exist...'.format(dest_name_log)) continue except ValueError as err: logger.error(err) continue except CalledProcessError as err: logger.error( 'Error while opening dest {:s}: \'{:s}\'...'.format( dest_name_log, err.stderr.rstrip())) continue else: # Match children on source to children on dest dest_children_names = [ child.name.replace(source_name, dest_name) for child in source_children ] # Send all children to corresponding children on dest for source_fs, dest_name in zip(source_children, dest_children_names): # exclude filesystems from rules if any( fnmatch(source_fs.name, pattern) for pattern in exclude): logger.debug( 'Matched {} in exclude rules, not sending...'. format(source_fs)) continue # send not excluded filesystems send_filesystem(source_fs, dest_name, ssh_dest=ssh_dest, raw=raw) finally: if ssh_dest: ssh_dest.close() if ssh_source: ssh_source.close()
def clean_config(config): """Deletes old snapshots according to strategies given in config. Goes through each config, opens up ssh connection if necessary and then recursively calls clean_filesystem. Parameters: ---------- config : {list of dict} Full config list containing all strategies for different filesystems """ logger = logging.getLogger(__name__) logger.info('Cleaning snapshots...') for conf in config: if not conf.get('clean', None): continue name = conf['name'] try: _type, fsname, user, host, port = parse_name(name) except ValueError as err: logger.error('Could not parse {:s}: {}...'.format(name, err)) continue if _type == 'ssh': try: ssh = SSH(user, host, port=port, key=conf['key']) except (FileNotFoundError, SSHException): continue name_log = '{:s}@{:s}:{:s}'.format(user, host, fsname) else: ssh = None name_log = fsname try: # Children includes the base filesystem (named 'fsname') children = zfs.find(path=fsname, types=['filesystem', 'volume'], ssh=ssh) except DatasetNotFoundError as err: logger.error('Dataset {:s} does not exist...'.format(name_log)) continue except ValueError as err: logger.error(err) continue except CalledProcessError as err: logger.error('Error while opening {:s}: \'{:s}\'...'.format( name_log, err.stderr.rstrip().decode())) else: # Clean snapshots of parent filesystem clean_filesystem(children[0], conf) # Clean snapshots of all children that don't have a seperate config entry for child in children[1:]: # Check if any of the parents (but child of base filesystem) have a config entry for parent in children[1:]: if ssh: child_name = 'ssh:{:d}:{:s}@{:s}:{:s}'.format( port, user, host, child.name) parent_name = 'ssh:{:d}:{:s}@{:s}:{:s}'.format( port, user, host, parent.name) else: child_name = child.name parent_name = parent.name # Skip if child has an entry or if any parent entry already in config child_parent = '/'.join(child_name.split( '/')[:-1]) # get parent of child filesystem if ((child_name == parent_name or child_parent.startswith(parent_name)) and (parent_name in [entry['name'] for entry in config])): break else: clean_filesystem(child, conf) finally: if ssh: ssh.close()
def send_config(config): """Tries to sync all entries in the config to their dest. Finds all children of the filesystem and calls send_filesystem on each of them. Parameters: ---------- config : {list of dict} Full config list containing all strategies for different filesystems """ logger = logging.getLogger(__name__) logger.info('Sending snapshots...') for conf in config: if not conf.get('dest', None): continue dry_run = conf.get('dry_run', None) dry_msg = '*** DRY RUN ***' if dry_run else '' backup_source = conf['name'] try: _type, source_name, user, host, port = parse_name(backup_source) except ValueError as err: logger.error('Could not parse {:s}: {}...'.format( backup_source, err)) continue # if source is remote, open ssh connection if _type == 'ssh': key = conf['key'] if conf.get('key', None) else None compress = conf['compress'].pop(0) if conf.get('compress', None) else 'lzop' try: ssh_source = SSH(user, host, port=port, key=key, compress=compress) except (FileNotFoundError, SSHException): continue source_name_log = '{:s}@{:s}:{:s}'.format(user, host, source_name) else: ssh_source = None source_name_log = source_name try: # Children includes the base filesystem (named 'source_name') source_children = zfs.find(path=source_name, types=['filesystem', 'volume'], ssh=ssh_source) except DatasetNotFoundError as err: logger.error( 'Source {:s} does not exist...'.format(source_name_log)) continue except ValueError as err: logger.error(err) continue except CalledProcessError as err: logger.error('Error while opening source {:s}: \'{:s}\'...'.format( source_name_log, err.stderr.rstrip())) continue # Send to every backup destination for backup_dest in conf['dest']: # get exclude rules exclude = conf['exclude'].pop(0) if conf.get('exclude', None) else [] # check if raw send was requested raw = conf['raw_send'].pop(0) if conf.get('raw_send', None) else False # check if we need to retry retries = conf['retries'].pop(0) if conf.get('retries', None) else 0 retry_interval = conf['retry_interval'].pop(0) if conf.get( 'retry_interval', None) else 10 # check if resumable send was requested resume = conf['resume'].pop(0) if conf.get('resume', None) else False # check if we should create dataset if it doesn't exist dest_auto_create = conf['dest_auto_create'].pop(0) if conf.get( 'dest_auto_create', None) else False try: _type, dest_name, user, host, port = parse_name(backup_dest) except ValueError as err: logger.error('Could not parse {:s}: {}...'.format( backup_dest, err)) continue # if dest is remote, open ssh connection if _type == 'ssh': dest_key = conf['dest_keys'].pop(0) if conf.get( 'dest_keys', None) else None # if 'ssh_source' is set, then 'compress' is already set and we use same compression for both source and dest # if not then we take the next entry in config if not ssh_source: compress = conf['compress'].pop(0) if conf.get( 'compress', None) else 'lzop' try: ssh_dest = SSH(user, host, port=port, key=dest_key, compress=compress) except (FileNotFoundError, SSHException): continue dest_name_log = '{:s}@{:s}:{:s}'.format(user, host, dest_name) else: ssh_dest = None dest_name_log = dest_name # check if dest exists try: zfs.open(dest_name, ssh=ssh_dest) except DatasetNotFoundError: if dest_auto_create: logger.info( 'Destination {:s} does not exist, will create it... {}' .format(dest_name_log, dry_msg)) if create_dataset(dest_name, dest_name_log, ssh=ssh_dest, dry_run=dry_run): continue else: logger.error( 'Destination {:s} does not exist, manually create it or use "dest-auto-create" option...' .format(dest_name_log)) continue except ValueError as err: logger.error(err) continue except CalledProcessError as err: logger.error( 'Error while opening dest {:s}: \'{:s}\'...'.format( dest_name_log, err.stderr.rstrip())) continue # Match children on source to children on dest dest_children_names = [ child.name.replace(source_name, dest_name) for child in source_children ] # Send all children to corresponding children on dest for source_fs, dest_name in zip(source_children, dest_children_names): # exclude filesystems from rules if any( fnmatch(source_fs.name, pattern) for pattern in exclude): logger.debug( 'Matched {} in exclude rules, not sending...'.format( source_fs)) continue # Check for ZFS user property to bypass filesystem fs_props = source_fs.getprops() exclude_prop = 'pyznap:exclude' ignore_me = fs_props.get(exclude_prop, ('false', 'false'))[0].lower() logger.debug("Property {}={} for {}".format( exclude_prop, ignore_me, source_fs)) if ignore_me == 'true': logger.info('Matched {}={} for {}, not sending...'.format( exclude_prop, ignore_me, source_fs)) continue # Check for max size used_prop = 'used' fs_used_bytes = int(fs_props.get(used_prop, ('0', '0'))[0]) # Bytes fs_used_fmt = bytes_fmt(fs_used_bytes) # MB logger.debug("Property {}={} ({}) for {}".format( used_prop, fs_used_fmt, fs_used_bytes, source_fs)) max_prop = 'pyznap:max_size' fs_max_fmt = fs_props.get(max_prop, ('0', '0'))[0] # String fs_max_bytes = parse_size(fs_max_fmt) # Bytes logger.debug("Property {}={} ({}) for {}".format( max_prop, fs_max_fmt, fs_max_bytes, source_fs)) if fs_max_bytes > 0 and fs_used_bytes > fs_max_bytes: logger.info( 'Filesystem size {} exceeds {}={} for {}, not sending...' .format(fs_used_fmt, max_prop, fs_max_fmt, source_fs)) continue # send not excluded filesystems for retry in range(1, retries + 2): rc = send_filesystem(source_fs, dest_name, ssh_dest=ssh_dest, raw=raw, resume=resume, dry_run=dry_run) if rc == 2 and retry <= retries: logger.info( 'Retrying send in {:d}s (retry {:d} of {:d})...'. format(retry_interval, retry, retries)) sleep(retry_interval) else: break if ssh_dest: ssh_dest.close() if ssh_source: ssh_source.close()
def send_config(config): """Tries to sync all entries in the config to their dest. Finds all children of the filesystem and calls send_snap on each of them. Parameters: ---------- config : {list of dict} Full config list containing all strategies for different filesystems """ logger = logging.getLogger(__name__) logger.info('Sending snapshots...') for conf in config: if not conf.get('dest', None): continue source_name = conf['name'] if source_name.startswith('ssh'): logger.error('Cannot send from remote location ({:s})...'.format( source_name)) continue try: # Children includes the base filesystem (named 'source_fs') source_children = zfs.find(path=source_name, types=['filesystem', 'volume']) except DatasetNotFoundError as err: logger.error('Source {:s} does not exist...'.format(source_name)) continue except ValueError as err: logger.error(err) continue except CalledProcessError as err: logger.error('Error while opening source {:s}: \'{:s}\'...'.format( source_name, err.stderr.rstrip())) continue # Send to every backup destination for backup_dest in conf['dest']: try: _type, dest_name, user, host, port = parse_name(backup_dest) except ValueError as err: logger.error('Could not parse {:s}: {}...'.format( backup_dest, err)) continue if _type == 'ssh': dest_key = conf['dest_keys'].pop( 0) if conf['dest_keys'] else None try: ssh = open_ssh(user, host, port=port, key=dest_key) except (FileNotFoundError, SSHException): continue dest_name_log = '{:s}@{:s}:{:s}'.format(user, host, dest_name) else: ssh = None dest_name_log = dest_name # Check if base destination filesystem exists, if not do not send try: zfs.open(dest_name, ssh=ssh) except DatasetNotFoundError: logger.error( 'Destination {:s} does not exist...'.format(dest_name_log)) continue except ValueError as err: logger.error(err) continue except CalledProcessError as err: logger.error( 'Error while opening dest {:s}: \'{:s}\'...'.format( dest_name_log, err.stderr.rstrip())) continue else: # Match children on source to children on dest dest_children_names = [ child.name.replace(source_name, dest_name) for child in source_children ] # Send all children to corresponding children on dest for source, dest in zip(source_children, dest_children_names): send_snap(source, dest, ssh=ssh) finally: if ssh: ssh.close()
def fix_snapshots(filesystems, format=None, type=None, type_map=None, recurse=False): """Fix snapshots name Parameters: ---------- filesystems : [strings] Filesystems to fix """ logger = logging.getLogger(__name__) if format.startswith('@'): if not type_map and format in MAPS: type_map = MAPS[format] if format in FORMATS: format = FORMATS[format] else: logger.error('Unknown format {}.'.format(format)) sys.exit(1) logger.debug('FORMAT: ' + str(format)) logger.debug('MAP: ' + str(type_map)) rp = re.compile(format) now = datetime.now() cur_century = int(now.year / 100) * 100 # for all specified filesystems for fsname in filesystems: logger.info('Checking snapshots on {}...'.format(fsname)) try: parent = zfs.open(fsname) except DatasetNotFoundError: logger.error('Filesystem not exists {}'.format(fsname)) continue if recurse: # get all child's filesystem fstree = zfs.find(fsname, types=['filesystem', 'volume']) else: # only scan specified filesystem fstree = [parent] for filesystem in fstree: logger.info('Fixing {}...'.format(filesystem.name)) snapshots = filesystem.snapshots() for snapshot in snapshots: snapname = snapshot.snapname() try: r = rp.match(snapname) except: r = False if r: # guess year year = re_get_group_int(r, 'year', default=now.year) if year < 100: year += +cur_century # get type from snap, with optional map or default type if specified snaptype = r.group('type') if type_map: if snaptype in type_map: snaptype = type_map[snaptype] if not snaptype and type: snaptype = type if not snaptype: logger.error( 'Unknown snap type {} for snapshot {}'.format( snaptype, snapname)) continue new_snapname = 'pyznap_' + datetime( year, re_get_group_int(r, 'month', default=now.month), re_get_group_int(r, 'day', default=now.day), hour=re_get_group_int(r, 'hour', default=now.hour), minute=re_get_group_int( r, 'minute', default=now.minute), second=re_get_group_int( r, 'second', default=now.second)).strftime( '%Y-%m-%d_%H:%M:%S') + '_' + snaptype logger.debug('Renaming {} -> {}'.format( snapname, new_snapname)) snapshot.rename(snapshot.fsname() + '@' + new_snapname)
if snapshot != snapshot_to_keep: print( "{} is not the snapshot to keep {}, destroying...".format( str(snapshot), str(snapshot_to_keep))) snapshot.destroy() def prune_syncoid(snapshots): """ prune syncoid snapshots. Call only once they are no longer needed """ for snapshot in snapshots: if "syncoid" in str(snapshot): print("Pruning syncoid snapshot {}...".format(str(snapshot))) snapshot.destroy() source_children = zfs.find(path="superior/tankbackup", types=['filesystem', 'volume']) print("Checking that report directory exists...") if not path.isdir('/mnt/snailback/report') or not path.exists( '/mnt/snailback/report'): raise NotADirectoryError( "/mnt/snailback/report does not exist, verify drive is mounted: mount /dev/sdg1 /mnt/snailback" ) print("Opening /mnt/snailback/report/backups.txt") with open("/mnt/snailback/report/backups.txt", "w") as backup_file: print("Most recent montly snapshots in superior/tankbackup are:") for child in source_children: snapshots = child.snapshots() if len(snapshots) > 0: for period in ['yearly', 'monthly', 'weekly', 'daily', 'hourly']: most_recent = get_most_recent(snapshots, period)