def zpools(): """Creates two temporary zpools to be called from test functions. Yields the two pool names and destroys them after testing.""" zpool = '/sbin/zpool' _word = randomword(8) pool0 = 'pyznap_source_' + _word pool1 = 'pyznap_dest_' + _word # Create temporary files on which the zpools are created with NamedTemporaryFile() as file0, NamedTemporaryFile() as file1: filename0 = file0.name filename1 = file1.name # Fix size to 100Mb file0.seek(100 * 1024**2 - 1) file0.write(b'0') file0.seek(0) file1.seek(100 * 1024**2 - 1) file1.write(b'0') file1.seek(0) # Create temporary test pools for pool, filename in zip([pool0, pool1], [filename0, filename1]): try: sp.check_call([zpool, 'create', pool, filename]) except sp.CalledProcessError as err: logger.error(err) return try: fs0 = zfs.open(pool0) fs1 = zfs.open(pool1) assert fs0.name == pool0 assert fs1.name == pool1 except (DatasetNotFoundError, AssertionError, Exception) as err: logger.error(err) else: yield fs0, fs1 # Destroy temporary test pools for pool in [pool0, pool1]: try: sp.check_call([zpool, 'destroy', pool]) except sp.CalledProcessError as err: logger.error(err)
def zpools(): """Creates two temporary zpools to be called from test functions. Yields the two pool names and destroys them after testing.""" # Create temporary files on which the zpools are created with NamedTemporaryFile() as file0, NamedTemporaryFile() as file1: filename0 = file0.name filename1 = file1.name # Fix size to 100Mb file0.seek(100*1024**2-1) file0.write(b'0') file0.seek(0) file1.seek(100*1024**2-1) file1.write(b'0') file1.seek(0) # Create temporary test pools for pool, filename in zip([POOL0, POOL1], [filename0, filename1]): try: sp.check_call([ZPOOL, 'create', pool, filename]) except sp.CalledProcessError as err: logger.error(err) return try: fs0 = zfs.open(POOL0) fs1 = zfs.open(POOL1) assert fs0.name == POOL0 assert fs1.name == POOL1 except (DatasetNotFoundError, AssertionError, Exception) as err: logger.error(err) else: yield fs0, fs1 # Destroy temporary test pools for pool in [POOL0, POOL1]: try: sp.check_call([ZPOOL, 'destroy', pool]) except sp.CalledProcessError as err: logger.error(err)
def send_config(config, settings={}): """Tries to sync all entries in the config to their dest. Finds all children of the filesystem and calls send_filesystem on each of them. Parameters: ---------- config : {list of dict} Full config list containing all strategies for different filesystems """ logger = logging.getLogger(__name__) logger.info('Sending snapshots...') for conf in config: if not conf.get('dest', None): logger.debug('Ignore config from send {}...'.format(conf['name'])) continue logger.debug('Process config {}...'.format(conf['name'])) backup_source = conf['name'] try: _type, source_name, user, host, port = parse_name(backup_source) except ValueError as err: logger.error('Could not parse {:s}: {}...'.format( backup_source, err)) continue # if source is remote, open ssh connection if _type == 'ssh': key = conf['key'] if conf.get('key', None) else None compress = conf['compress'].pop(0) if conf.get('compress', None) else 'lzop' try: ssh_source = SSH(user, host, port=port, key=key, compress=compress) except (FileNotFoundError, SSHException): continue source_name_log = '{:s}@{:s}:{:s}'.format(user, host, source_name) else: ssh_source = None source_name_log = source_name try: # Children includes the base filesystem (named 'source_name') source_children = zfs.find_exclude(conf, config, ssh=ssh_source, matching=settings['matching']) except DatasetNotFoundError as err: logger.error( 'Source {:s} does not exist...'.format(source_name_log)) continue except ValueError as err: logger.error(err) continue except CalledProcessError as err: logger.error('Error while opening source {:s}: \'{:s}\'...'.format( source_name_log, err.stderr.rstrip())) continue send_exclude_property = conf.get('send_exclude_property') # Send to every backup destination for backup_dest in conf['dest']: # get exclude rules exclude = conf['exclude'].pop(0) if conf.get('exclude', None) else [] # check if raw send was requested raw = conf['raw_send'].pop(0) if conf.get('raw_send', None) else False # check if we need to retry retries = conf['retries'].pop(0) if conf.get('retries', None) else 0 retry_interval = conf['retry_interval'].pop(0) if conf.get( 'retry_interval', None) else 10 # check if resumable send was requested resume = conf['resume'].pop(0) if conf.get('resume', None) else False # check if send_last_snapshot was requested send_last_snapshot = conf['send_last_snapshot'].pop(0) if conf.get( 'send_last_snapshot', None) else False if send_last_snapshot == 'no': send_last_snapshot = False # check if we should create dataset if it doesn't exist dest_auto_create = conf['dest_auto_create'].pop(0) if conf.get( 'dest_auto_create', None) else False try: _type, dest_name, user, host, port = parse_name(backup_dest) except ValueError as err: logger.error('Could not parse {:s}: {}...'.format( backup_dest, err)) continue # if dest is remote, open ssh connection if _type == 'ssh': dest_key = conf['dest_keys'].pop(0) if conf.get( 'dest_keys', None) else None # if 'ssh_source' is set, then 'compress' is already set and we use same compression for both source and dest # if not then we take the next entry in config if not ssh_source: compress = conf['compress'].pop(0) if conf.get( 'compress', None) else 'lzop' try: ssh_dest = SSH(user, host, port=port, key=dest_key, compress=compress) except (FileNotFoundError, SSHException): continue dest_name_log = '{:s}@{:s}:{:s}'.format(user, host, dest_name) else: ssh_dest = None dest_name_log = dest_name # check if dest exists try: zfs.open(dest_name, ssh=ssh_dest) except DatasetNotFoundError: if dest_auto_create: logger.info( 'Destination {:s} does not exist, will create it...'. format(dest_name_log)) if create_dataset(dest_name, dest_name_log, ssh=ssh_dest): continue else: logger.error( 'Destination {:s} does not exist, manually create it or use "dest-auto-create" option...' .format(dest_name_log)) continue except ValueError as err: logger.error(err) continue except CalledProcessError as err: logger.error( 'Error while opening dest {:s}: \'{:s}\'...'.format( dest_name_log, err.stderr.rstrip())) continue # Match children on source to children on dest if source_name == '': dest_children_names = [ dest_name + '/' + child.name for child in source_children ] else: dest_children_names = [ child.name.replace(source_name, dest_name) for child in source_children ] # Send all children to corresponding children on dest for source_fs, dest_name in zip(source_children, dest_children_names): # exclude filesystems from rules if any( fnmatch(source_fs.name, pattern) for pattern in exclude): logger.debug( 'Matched {} in exclude rules, not sending...'.format( source_fs)) continue # check exclude attribute if send_exclude_property and source_fs.ispropval( send_exclude_property, check='false'): logger.debug( 'Not sending {}, have property {:s}=false'.format( source_fs, send_exclude_property)) continue # TODO: create missing skipped filesystem on destination # send not excluded filesystems for retry in range(1, retries + 2): rc = send_filesystem(source_fs, dest_name, ssh_dest=ssh_dest, raw=raw, resume=resume, send_last_snapshot=send_last_snapshot, dest_auto_create=dest_auto_create) if rc == 2 and retry <= retries: logger.info( 'Retrying send in {:d}s (retry {:d} of {:d})...'. format(retry_interval, retry, retries)) sleep(retry_interval) else: break if ssh_dest: ssh_dest.close() if ssh_source: ssh_source.close()
def send_filesystem(source_fs, dest_name, ssh_dest=None, raw=False, resume=False, send_last_snapshot=False, dest_auto_create=False): """Checks for common snapshots between source and dest. If none are found, send the oldest snapshot, then update with the most recent one. If there are common snaps, update destination with the most recent one. Parameters: ---------- source_fs : {ZFSFilesystem} Source zfs filesystem from where to send dest_name : {str} Name of the location to send to ssh_dest : {ssh.SSH}, optional Open ssh connection for remote backup (the default is None, meaning local backup) Returns ------- int 0 if success, 1 if not, 2 for ssh errors """ logger = logging.getLogger(__name__) dest_name_log = '{:s}@{:s}:{:s}'.format( ssh_dest.user, ssh_dest.host, dest_name) if ssh_dest else dest_name logger.debug('Sending {} to {:s}...'.format(source_fs, dest_name_log)) resume_token = None # Check if dest already has a 'zfs receive' ongoing if check_recv(dest_name, ssh=ssh_dest): return 1 # get snapshots on source, catch exception if dataset was destroyed since pyznap was started try: snapshots = source_fs.snapshots()[::-1] except (DatasetNotFoundError, DatasetBusyError) as err: logger.error('Error while opening source {}: {}...'.format( source_fs, err)) return 1 except CalledProcessError as err: message = err.stderr.rstrip() if message.startswith('ssh: '): logger.error( 'Connection issue while opening source {}: \'{:s}\'...'.format( source_fs, message)) return 2 else: logger.error('Error while opening source {}: \'{:s}\'...'.format( source_fs, message)) return 1 snapnames = [snap.name.split('@')[1] for snap in snapshots] try: snapshot = snapshots[0] # Most recent snapshot base = snapshots[-1] # Oldest snapshot except IndexError: logger.error('No snapshots on {}, cannot send...'.format(source_fs)) return 1 try: dest_fs = zfs.open(dest_name, ssh=ssh_dest) except DatasetNotFoundError: if dest_auto_create: logger.info( 'Destination {:s} does not exist, will create it...'.format( dest_name_log)) if create_dataset(dest_name, dest_name_log, ssh=ssh_dest): return 1 else: logger.error( 'Destination {:s} does not exist, manually create it or use "dest-auto-create" option...' .format(dest_name_log)) return 1 dest_snapnames = [] common = set() except CalledProcessError as err: message = err.stderr.rstrip() if message.startswith('ssh: '): logger.error( 'Connection issue while opening dest {:s}: \'{:s}\'...'.format( dest_name_log, message)) return 2 else: logger.error('Error while opening dest {:s}: \'{:s}\'...'.format( dest_name_log, message)) return 1 else: # if dest exists, check for resume token resume_token = dest_fs.getprops().get('receive_resume_token', (None, None))[0] # find common snapshots between source & dest dest_snapnames = [ snap.name.split('@')[1] for snap in dest_fs.snapshots() ] common = set(snapnames) & set(dest_snapnames) # if not resume and resume_token is not None: # if not abort: # logger.error('{:s} contains partially-complete state from "zfs receive -s" (~{:s}), ' # 'but neither resume nor abort option is given...' # .format(dest_name_log, bytes_fmt(base.stream_size(raw=raw, resume_token=resume_token)))) # return 1 # else: # logger.info('{:s} contains partially-complete state from "zfs receive -s" (~{:s}), ' # 'will abort it...' # .format(dest_name_log, bytes_fmt(base.stream_size(raw=raw, resume_token=resume_token)))) # if abort_resume(dest_fs): # return 1 zfs.STATS.add('zfs_send_filesystem_count') was_transfer = False if resume_token is not None: logger.info( 'Found resume token. Resuming last transfer of {:s} (~{:s})...'. format( dest_name_log, bytes_fmt(base.stream_size(raw=raw, resume_token=resume_token)))) was_transfer = True rc = send_snap(base, dest_name, base=None, ssh_dest=ssh_dest, raw=raw, resume=True, resume_token=resume_token) if rc: return rc # we need to update common snapshots after finishing the resumable send dest_snapnames = [ snap.name.split('@')[1] for snap in dest_fs.snapshots() ] common = set(snapnames) & set(dest_snapnames) if not common: if dest_snapnames: logger.error( 'No common snapshots on {:s}, but snapshots exist. Not sending...' .format(dest_name_log)) return 1 else: if send_last_snapshot: base = snapshot for snap in snapshots: if send_last_snapshot in snap.name.split('@')[1]: base = snap break logger.info( 'No common snapshots on {:s}, sending last snapshot {} (~{:s})...' .format(dest_name_log, base, bytes_fmt(base.stream_size(raw=raw)))) else: logger.info( 'No common snapshots on {:s}, sending oldest snapshot {} (~{:s})...' .format(dest_name_log, base, bytes_fmt(base.stream_size(raw=raw)))) was_transfer = True rc = send_snap(base, dest_name, base=None, ssh_dest=ssh_dest, raw=raw, resume=resume) if rc: return rc else: # If there are common snapshots, get the most recent one base = next( filter(lambda x: x.name.split('@')[1] in common, snapshots), None) if base.name != snapshot.name: logger.info( 'Updating {:s} with recent snapshot {} from {} (~{:s})...'.format( dest_name_log, snapshot, base.name.split('@')[1], bytes_fmt(snapshot.stream_size(base, raw=raw)))) was_transfer = True rc = send_snap(snapshot, dest_name, base=base, ssh_dest=ssh_dest, raw=raw, resume=resume) if rc: return rc if was_transfer: zfs.STATS.add('zfs_send_changed_count') else: zfs.STATS.add('zfs_send_unchanged_count') logger.info('{:s} is up to date...'.format(dest_name_log)) return 0
def zpools(): """Creates two temporary zpools to be called from test functions, source is local and dest on remote ssh location. Yields the two pool names and destroys them after testing.""" sftp_filename = '/tmp/' + randomword(10) # ssh arguments for zfs functions ssh = open_ssh(USER, HOST, port=PORT, key=KEY) sftp = ssh.open_sftp() # Create temporary file on which the source zpool is created. Manually create sftp file with NamedTemporaryFile() as file0, sftp.open(sftp_filename, 'w') as file1: filename0 = file0.name filename1 = sftp_filename # Fix size to 100Mb file0.seek(100 * 1024**2 - 1) file0.write(b'0') file0.seek(0) file1.seek(100 * 1024**2 - 1) file1.write(b'0') file1.seek(0) # Create temporary test pools try: sp.check_output([ZPOOL, 'create', POOL0, filename0]) except sp.CalledProcessError as err: logger.error(err) return try: sp.check_output([ZPOOL, 'create', POOL1, filename1], ssh=ssh) except sp.CalledProcessError as err: logger.error(err) return try: fs0 = zfs.open(POOL0) fs1 = zfs.open(POOL1, ssh=ssh) assert fs0.name == POOL0 assert fs1.name == POOL1 except (DatasetNotFoundError, AssertionError, Exception) as err: logger.error(err) else: yield fs0, fs1 # Destroy temporary test pools try: sp.check_output([ZPOOL, 'destroy', POOL0]) except sp.CalledProcessError as err: logger.error(err) try: sp.check_output([ZPOOL, 'destroy', POOL1], ssh=ssh) except sp.CalledProcessError as err: logger.error(err) # Delete tempfile on dest sftp.remove(sftp_filename) sftp.close() ssh.close()
def status_filesystem(filesystem, conf, output='log', show_all=False, main_fs=False, values=None, filter=None, filter_values=None, filter_exclude=None): """Deletes snapshots of a single filesystem according to conf. Parameters: ---------- filesystem : {ZFSFilesystem} Filesystem to status conf : {dict} Config entry with snapshot strategy main_fs: mark configured filesystem, ignore exclude zfs property """ global OUTPUT logger = logging.getLogger(__name__) fs_name = str(filesystem) if filter_exclude: if any(fnmatch(fs_name, pattern) for pattern in filter_exclude): logger.debug('Exclude filesystem {} by --exclude'.format(fs_name)) return logger.debug('Checking snapshots on {}...'.format(fs_name)) zfs.STATS.add('checked_count') snap = conf.get('snap', False) clean = conf.get('clean', False) send = bool(conf.get('dest', False)) excluded = False snap_exclude_property = conf['snap_exclude_property'] if not main_fs and snap_exclude_property and filesystem.ispropval( snap_exclude_property, check='false'): zfs.STATS.add('snap_excluded_count') logger.debug( 'Ignore dataset fron snap {:s}, have property {:s}=false'.format( filesystem.name, snap_exclude_property)) snap = False clean = False send_exclude_property = conf['send_exclude_property'] if not main_fs and send_exclude_property and filesystem.ispropval( send_exclude_property, check='false'): zfs.STATS.add('send_excluded_count') logger.debug( 'Ignore dataset fron send {:s}, have property {:s}=false'.format( filesystem.name, snap_exclude_property)) send = False if not (snap or clean or send): if show_all: zfs.STATS.add('excluded_count') excluded = True else: return # increase stats count and check excludes in send if snap: zfs.STATS.add('snap_count') if clean: zfs.STATS.add('clean_count') if send: dest = conf.get('dest', False) if dest and conf['exclude']: # check excluded sending = [] for exclude, dst in zip(conf['exclude'], dest): if exclude and any( fnmatch(filesystem.name, pattern) for pattern in exclude): zfs.STATS.add('dest_excluded_count') logger.debug('Excluded from send {} -> {}...'.format( filesystem, dst)) sending.append(False) else: sending.append(dst) dest = sending send = send and dest and any([x for x in dest if bool(x)]) else: dest = None if send: zfs.STATS.add('send_count') snapshots = {t: [] for t in SNAPSHOT_TYPES} # catch exception if dataset was destroyed since pyznap was started try: fs_snapshots = filesystem.snapshots() except (DatasetNotFoundError, DatasetBusyError) as err: logger.error('Error while opening {}: {}...'.format(filesystem, err)) return 1 have_snapshots = bool(fs_snapshots) # categorize snapshots for snaps in fs_snapshots: # Ignore snapshots not taken with pyznap if not snaps.name.split('@')[1].startswith('pyznap'): continue try: snap_type = snaps.name.split('_')[-1] snapshots[snap_type].append(snaps) except (ValueError, KeyError): continue # Reverse sort by time taken for snaps in snapshots.values(): snaps.reverse() level = logging.INFO # prepare data for status counts = {} for s in snapshots.keys(): counts[s] = conf.get(s, 0) or 0 if clean else 0 pyznap_snapshots = sum(len(s) for s in snapshots.values()) # TODO: remote uptodate check # TODO: T/F oversnapshot/undesnapshot/othersnapshots/unvantedsnapshot on exluded fs # check needed snapshots count missing_snapshots = any( [len(snapshots[t]) < counts[t] for t in SNAPSHOT_TYPES]) extra_snapshots = any( [len(snapshots[t]) > counts[t] for t in SNAPSHOT_TYPES]) if missing_snapshots: level = logging.WARNING # make status data status = OrderedDict() status['hostname'] = os.uname()[1] status['name'] = fs_name status['conf'] = conf['name'] status['excluded'] = excluded status['do-snap'] = snap status['do-clean'] = clean status['do-send'] = send status['conf-snap_exclude_property'] = snap_exclude_property status['conf-send_exclude_property'] = send_exclude_property status['snapshot-have'] = have_snapshots status['snapshot-missing'] = missing_snapshots status['snapshot-extra'] = extra_snapshots status['snapshot-count-all'] = len(fs_snapshots) status['snapshot-count-pyznap'] = pyznap_snapshots status['snapshot-count-nopyznap'] = len(fs_snapshots) - pyznap_snapshots for stype in SNAPSHOT_TYPES: status['snapshot-types-' + stype] = str(len(snapshots[stype])) + '/' + str(counts[stype]) def bytes_fmt_no_raw(bytes): return bytes if output == 'jsonl' else bytes_fmt(bytes) status['dest'] = dest if dest: i = 0 snapnames = [snap.name.split('@')[1] for snap in fs_snapshots] for d in dest: if d: _prefix = 'dest-' + str(i) + '-' _type, _dest_name, _user, _host, _port = parse_name(d) status[_prefix + 'type'] = _type status[_prefix + 'host'] = _host if conf['name']: dest_name = fs_name.replace(conf['name'], _dest_name) else: dest_name = _dest_name + '/' + fs_name status[_prefix + 'name'] = dest_name # check snapshots on dest common_snapshots = [] ssh_dest = get_ssh_for_dest(d, conf) try: dest_fs = zfs.open(dest_name, ssh=ssh_dest) except DatasetNotFoundError: dest_snapshots = [] dest_snapnames = [] common = set() except CalledProcessError as err: message = err.stderr.rstrip() if message.startswith('ssh: '): logger.error( 'Connection issue while opening dest {:s}: \'{:s}\'...' .format(dest_name, message)) return 2 else: logger.error( 'Error while opening dest {:s}: \'{:s}\'...'. format(dest_name, message)) return 1 else: # find common snapshots between source & dest dest_snapshots = dest_fs.snapshots() dest_snapnames = [ snap.name.split('@')[1] for snap in dest_snapshots ] common = set(snapnames) & set(dest_snapnames) if common: common_snapshots = [ s for s in snapnames if s in common ] status[_prefix + 'snapshot-count'] = len(dest_snapnames) status[_prefix + 'snapshot-count-common'] = len(common_snapshots) if common_snapshots: status[_prefix + 'snapshot-common-first'] = common_snapshots[0] status[_prefix + 'snapshot-common-last'] = common_snapshots[-1] if dest_snapnames: status[_prefix + 'snapshot-dest-first'] = dest_snapnames[0] status[_prefix + 'snapshot-dest-last'] = dest_snapnames[-1] i += 1 def add_snapshot_status(snapshot, label): props = snapshot.getprops() status['snapshot-info-' + label + '-timestamp'] = datetime.fromtimestamp( int(props['creation'][0])).isoformat() status['snapshot-info-' + label + '-referenced'] = bytes_fmt_no_raw( int(props['referenced'][0])) status['snapshot-info-' + label + '-logicalreferenced'] = bytes_fmt_no_raw( int(props['logicalreferenced'][0])) if fs_snapshots: add_snapshot_status(fs_snapshots[0], 'first') add_snapshot_status(fs_snapshots[-1], 'last') props = filesystem.getprops() for p in ZFS_SIZE_PROPERTIES: status['zfs-' + p] = bytes_fmt_no_raw(int(props[p][0])) for p in ZFS_OTHER_PROPERTIES: status['zfs-' + p] = props[p][0] if p in props else '---' if filter_values: for f, v in filter_values.items(): if status[f] != v: return if values: fstatus = {} for v in values: for k in tuple(status.keys()): if fnmatch(k, v): fstatus[k] = status[k] del status[k] status = fstatus if output == 'jsonl': print(json.dumps(status)) elif output == 'html': OUTPUT.append(status) else: logger.log(level, 'STATUS: ' + json.dumps(status))
def zpools(): """Creates two temporary zpools to be called from test functions, source is local and dest on remote ssh location. Yields the two pool names and destroys them after testing.""" zpool = '/sbin/zpool' _word = randomword(8) pool0 = 'pyznap_source_' + _word pool1 = 'pyznap_dest_' + _word sftp_filename = '/tmp/' + randomword(10) # ssh arguments for zfs functions ssh = SSH(USER, HOST, port=PORT, key=KEY) # need paramiko for sftp file sshclient = open_ssh(USER, HOST, port=PORT, key=KEY) sftp = sshclient.open_sftp() # Create temporary file on which the source zpool is created. Manually create sftp file with NamedTemporaryFile() as file0, sftp.open(sftp_filename, 'w') as file1: filename0 = file0.name filename1 = sftp_filename # Fix size to 100Mb file0.seek(100 * 1024**2 - 1) file0.write(b'0') file0.seek(0) file1.seek(100 * 1024**2 - 1) file1.write(b'0') file1.seek(0) # Create temporary test pools try: run([zpool, 'create', pool0, filename0]) except sp.CalledProcessError as err: logger.error(err) return try: run([zpool, 'create', pool1, filename1], ssh=ssh) except sp.CalledProcessError as err: logger.error(err) return try: fs0 = zfs.open(pool0) fs1 = zfs.open(pool1, ssh=ssh) assert fs0.name == pool0 assert fs1.name == pool1 except (DatasetNotFoundError, AssertionError, Exception) as err: logger.error(err) else: yield fs0, fs1 # Destroy temporary test pools try: run([zpool, 'destroy', pool0]) except sp.CalledProcessError as err: logger.error(err) try: run([zpool, 'destroy', pool1], ssh=ssh) except sp.CalledProcessError as err: logger.error(err) # Delete tempfile on dest sftp.remove(sftp_filename) sftp.close() ssh.close()
def fix_snapshots(filesystems, format=None, type=None, type_map=None, recurse=False): """Fix snapshots name Parameters: ---------- filesystems : [strings] Filesystems to fix """ logger = logging.getLogger(__name__) if format.startswith('@'): if not type_map and format in MAPS: type_map = MAPS[format] if format in FORMATS: format = FORMATS[format] else: logger.error('Unknown format {}.'.format(format)) sys.exit(1) logger.debug('FORMAT: ' + str(format)) logger.debug('MAP: ' + str(type_map)) rp = re.compile(format) now = datetime.now() cur_century = int(now.year / 100) * 100 # for all specified filesystems for fsname in filesystems: logger.info('Checking snapshots on {}...'.format(fsname)) try: parent = zfs.open(fsname) except DatasetNotFoundError: logger.error('Filesystem not exists {}'.format(fsname)) continue if recurse: # get all child's filesystem fstree = zfs.find(fsname, types=['filesystem', 'volume']) else: # only scan specified filesystem fstree = [parent] for filesystem in fstree: logger.info('Fixing {}...'.format(filesystem.name)) snapshots = filesystem.snapshots() for snapshot in snapshots: snapname = snapshot.snapname() try: r = rp.match(snapname) except: r = False if r: # guess year year = re_get_group_int(r, 'year', default=now.year) if year < 100: year += +cur_century # get type from snap, with optional map or default type if specified snaptype = r.group('type') if type_map: if snaptype in type_map: snaptype = type_map[snaptype] if not snaptype and type: snaptype = type if not snaptype: logger.error( 'Unknown snap type {} for snapshot {}'.format( snaptype, snapname)) continue new_snapname = 'pyznap_' + datetime( year, re_get_group_int(r, 'month', default=now.month), re_get_group_int(r, 'day', default=now.day), hour=re_get_group_int(r, 'hour', default=now.hour), minute=re_get_group_int( r, 'minute', default=now.minute), second=re_get_group_int( r, 'second', default=now.second)).strftime( '%Y-%m-%d_%H:%M:%S') + '_' + snaptype logger.debug('Renaming {} -> {}'.format( snapname, new_snapname)) snapshot.rename(snapshot.fsname() + '@' + new_snapname)
def send_snap(source_fs, dest_name, ssh=None): """Checks for common snapshots between source and dest. If none are found, send the oldest snapshot, then update with the most recent one. If there are common snaps, update destination with the most recent one. Parameters: ---------- source_fs : {ZFSFilesystem} Source zfs filesystem from where to send dest_name : {str} Name of the location to send to ssh : {paramiko.SSHClient}, optional Open ssh connection for remote backup (the default is None, meaning local backup) Returns ------- int 0 if success, 1 if not """ logger = logging.getLogger(__name__) dest_name_log = '{:s}@{:s}:{:s}'.format(ssh.user, ssh.host, dest_name) if ssh else dest_name logger.debug('Sending {} to {:s}...'.format(source_fs, dest_name_log)) # Check if ssh session still active if ssh and not ssh.get_transport().is_active(): logger.error( 'Error while sending to {:s}: ssh session not active...'.format( dest_name_log)) return 1 # Check if dest already has a 'zfs receive' ongoing if check_recv(dest_name, ssh=ssh): return 1 # Get snapshots on source snapshots = source_fs.snapshots()[::-1] snapnames = [snap.name.split('@')[1] for snap in snapshots] try: snapshot = snapshots[0] # Most recent snapshot base = snapshots[-1] # Oldest snapshot except IndexError: logger.error('No snapshots on {}, cannot send...'.format(source_fs)) return 1 try: dest_fs = zfs.open(dest_name, ssh=ssh) except DatasetNotFoundError: dest_snapnames = [] common = set() except CalledProcessError as err: logger.error('Error while opening dest {:s}: \'{:s}\'...'.format( dest_name_log, err.stderr.rstrip())) return 1 else: dest_snapnames = [ snap.name.split('@')[1] for snap in dest_fs.snapshots() ] # Find common snapshots between source & dest common = set(snapnames) & set(dest_snapnames) if not common: if dest_snapnames: logger.error( 'No common snapshots on {:s}, but snapshots exist. Not sending...' .format(dest_name_log)) return 1 else: logger.info( 'No common snapshots on {:s}, sending oldest snapshot {} (~{:s})...' .format(dest_name_log, base, base.stream_size())) if send_recv(base, dest_name, base=None, ssh=ssh): return 1 else: # If there are common snapshots, get the most recent one base = next( filter(lambda x: x.name.split('@')[1] in common, snapshots), None) if base.name != snapshot.name: logger.info('Updating {:s} with recent snapshot {} (~{:s})...'.format( dest_name_log, snapshot, snapshot.stream_size(base))) if send_recv(snapshot, dest_name, base=base, ssh=ssh): return 1 logger.info('{:s} is up to date...'.format(dest_name_log)) return 0
def send_config(config): """Tries to sync all entries in the config to their dest. Finds all children of the filesystem and calls send_snap on each of them. Parameters: ---------- config : {list of dict} Full config list containing all strategies for different filesystems """ logger = logging.getLogger(__name__) logger.info('Sending snapshots...') for conf in config: if not conf.get('dest', None): continue source_name = conf['name'] if source_name.startswith('ssh'): logger.error('Cannot send from remote location ({:s})...'.format( source_name)) continue try: # Children includes the base filesystem (named 'source_fs') source_children = zfs.find(path=source_name, types=['filesystem', 'volume']) except DatasetNotFoundError as err: logger.error('Source {:s} does not exist...'.format(source_name)) continue except ValueError as err: logger.error(err) continue except CalledProcessError as err: logger.error('Error while opening source {:s}: \'{:s}\'...'.format( source_name, err.stderr.rstrip())) continue # Send to every backup destination for backup_dest in conf['dest']: try: _type, dest_name, user, host, port = parse_name(backup_dest) except ValueError as err: logger.error('Could not parse {:s}: {}...'.format( backup_dest, err)) continue if _type == 'ssh': dest_key = conf['dest_keys'].pop( 0) if conf['dest_keys'] else None try: ssh = open_ssh(user, host, port=port, key=dest_key) except (FileNotFoundError, SSHException): continue dest_name_log = '{:s}@{:s}:{:s}'.format(user, host, dest_name) else: ssh = None dest_name_log = dest_name # Check if base destination filesystem exists, if not do not send try: zfs.open(dest_name, ssh=ssh) except DatasetNotFoundError: logger.error( 'Destination {:s} does not exist...'.format(dest_name_log)) continue except ValueError as err: logger.error(err) continue except CalledProcessError as err: logger.error( 'Error while opening dest {:s}: \'{:s}\'...'.format( dest_name_log, err.stderr.rstrip())) continue else: # Match children on source to children on dest dest_children_names = [ child.name.replace(source_name, dest_name) for child in source_children ] # Send all children to corresponding children on dest for source, dest in zip(source_children, dest_children_names): send_snap(source, dest, ssh=ssh) finally: if ssh: ssh.close()
def send_config(config): """Tries to sync all entries in the config to their dest. Finds all children of the filesystem and calls send_filesystem on each of them. Parameters: ---------- config : {list of dict} Full config list containing all strategies for different filesystems """ logger = logging.getLogger(__name__) logger.info('Sending snapshots...') for conf in config: if not conf.get('dest', None): continue dry_run = conf.get('dry_run', None) dry_msg = '*** DRY RUN ***' if dry_run else '' backup_source = conf['name'] try: _type, source_name, user, host, port = parse_name(backup_source) except ValueError as err: logger.error('Could not parse {:s}: {}...'.format( backup_source, err)) continue # if source is remote, open ssh connection if _type == 'ssh': key = conf['key'] if conf.get('key', None) else None compress = conf['compress'].pop(0) if conf.get('compress', None) else 'lzop' try: ssh_source = SSH(user, host, port=port, key=key, compress=compress) except (FileNotFoundError, SSHException): continue source_name_log = '{:s}@{:s}:{:s}'.format(user, host, source_name) else: ssh_source = None source_name_log = source_name try: # Children includes the base filesystem (named 'source_name') source_children = zfs.find(path=source_name, types=['filesystem', 'volume'], ssh=ssh_source) except DatasetNotFoundError as err: logger.error( 'Source {:s} does not exist...'.format(source_name_log)) continue except ValueError as err: logger.error(err) continue except CalledProcessError as err: logger.error('Error while opening source {:s}: \'{:s}\'...'.format( source_name_log, err.stderr.rstrip())) continue # Send to every backup destination for backup_dest in conf['dest']: # get exclude rules exclude = conf['exclude'].pop(0) if conf.get('exclude', None) else [] # check if raw send was requested raw = conf['raw_send'].pop(0) if conf.get('raw_send', None) else False # check if we need to retry retries = conf['retries'].pop(0) if conf.get('retries', None) else 0 retry_interval = conf['retry_interval'].pop(0) if conf.get( 'retry_interval', None) else 10 # check if resumable send was requested resume = conf['resume'].pop(0) if conf.get('resume', None) else False # check if we should create dataset if it doesn't exist dest_auto_create = conf['dest_auto_create'].pop(0) if conf.get( 'dest_auto_create', None) else False try: _type, dest_name, user, host, port = parse_name(backup_dest) except ValueError as err: logger.error('Could not parse {:s}: {}...'.format( backup_dest, err)) continue # if dest is remote, open ssh connection if _type == 'ssh': dest_key = conf['dest_keys'].pop(0) if conf.get( 'dest_keys', None) else None # if 'ssh_source' is set, then 'compress' is already set and we use same compression for both source and dest # if not then we take the next entry in config if not ssh_source: compress = conf['compress'].pop(0) if conf.get( 'compress', None) else 'lzop' try: ssh_dest = SSH(user, host, port=port, key=dest_key, compress=compress) except (FileNotFoundError, SSHException): continue dest_name_log = '{:s}@{:s}:{:s}'.format(user, host, dest_name) else: ssh_dest = None dest_name_log = dest_name # check if dest exists try: zfs.open(dest_name, ssh=ssh_dest) except DatasetNotFoundError: if dest_auto_create: logger.info( 'Destination {:s} does not exist, will create it... {}' .format(dest_name_log, dry_msg)) if create_dataset(dest_name, dest_name_log, ssh=ssh_dest, dry_run=dry_run): continue else: logger.error( 'Destination {:s} does not exist, manually create it or use "dest-auto-create" option...' .format(dest_name_log)) continue except ValueError as err: logger.error(err) continue except CalledProcessError as err: logger.error( 'Error while opening dest {:s}: \'{:s}\'...'.format( dest_name_log, err.stderr.rstrip())) continue # Match children on source to children on dest dest_children_names = [ child.name.replace(source_name, dest_name) for child in source_children ] # Send all children to corresponding children on dest for source_fs, dest_name in zip(source_children, dest_children_names): # exclude filesystems from rules if any( fnmatch(source_fs.name, pattern) for pattern in exclude): logger.debug( 'Matched {} in exclude rules, not sending...'.format( source_fs)) continue # Check for ZFS user property to bypass filesystem fs_props = source_fs.getprops() exclude_prop = 'pyznap:exclude' ignore_me = fs_props.get(exclude_prop, ('false', 'false'))[0].lower() logger.debug("Property {}={} for {}".format( exclude_prop, ignore_me, source_fs)) if ignore_me == 'true': logger.info('Matched {}={} for {}, not sending...'.format( exclude_prop, ignore_me, source_fs)) continue # Check for max size used_prop = 'used' fs_used_bytes = int(fs_props.get(used_prop, ('0', '0'))[0]) # Bytes fs_used_fmt = bytes_fmt(fs_used_bytes) # MB logger.debug("Property {}={} ({}) for {}".format( used_prop, fs_used_fmt, fs_used_bytes, source_fs)) max_prop = 'pyznap:max_size' fs_max_fmt = fs_props.get(max_prop, ('0', '0'))[0] # String fs_max_bytes = parse_size(fs_max_fmt) # Bytes logger.debug("Property {}={} ({}) for {}".format( max_prop, fs_max_fmt, fs_max_bytes, source_fs)) if fs_max_bytes > 0 and fs_used_bytes > fs_max_bytes: logger.info( 'Filesystem size {} exceeds {}={} for {}, not sending...' .format(fs_used_fmt, max_prop, fs_max_fmt, source_fs)) continue # send not excluded filesystems for retry in range(1, retries + 2): rc = send_filesystem(source_fs, dest_name, ssh_dest=ssh_dest, raw=raw, resume=resume, dry_run=dry_run) if rc == 2 and retry <= retries: logger.info( 'Retrying send in {:d}s (retry {:d} of {:d})...'. format(retry_interval, retry, retries)) sleep(retry_interval) else: break if ssh_dest: ssh_dest.close() if ssh_source: ssh_source.close()
def send_config(config): """Tries to sync all entries in the config to their dest. Finds all children of the filesystem and calls send_filesystem on each of them. Parameters: ---------- config : {list of dict} Full config list containing all strategies for different filesystems """ logger = logging.getLogger(__name__) logger.info('Sending snapshots...') for conf in config: if not conf.get('dest', None): continue backup_source = conf['name'] try: _type, source_name, user, host, port = parse_name(backup_source) except ValueError as err: logger.error('Could not parse {:s}: {}...'.format( backup_source, err)) continue # if source is remote, open ssh connection if _type == 'ssh': key = conf['key'] if conf.get('key', None) else None compress = conf['compress'].pop(0) if conf.get('compress', None) else 'lzop' try: ssh_source = SSH(user, host, port=port, key=key, compress=compress) except (FileNotFoundError, SSHException): continue source_name_log = '{:s}@{:s}:{:s}'.format(user, host, source_name) else: ssh_source = None source_name_log = source_name try: # Children includes the base filesystem (named 'source_name') source_children = zfs.find(path=source_name, types=['filesystem', 'volume'], ssh=ssh_source) except DatasetNotFoundError as err: logger.error( 'Source {:s} does not exist...'.format(source_name_log)) continue except ValueError as err: logger.error(err) continue except CalledProcessError as err: logger.error('Error while opening source {:s}: \'{:s}\'...'.format( source_name_log, err.stderr.rstrip())) continue # Send to every backup destination for backup_dest in conf['dest']: try: _type, dest_name, user, host, port = parse_name(backup_dest) except ValueError as err: logger.error('Could not parse {:s}: {}...'.format( backup_dest, err)) continue # if dest is remote, open ssh connection if _type == 'ssh': dest_key = conf['dest_keys'].pop(0) if conf.get( 'dest_keys', None) else None # if 'ssh_source' is set, then 'compress' is already set and we use same compression for both source and dest # if not then we take the next entry in config if not ssh_source: compress = conf['compress'].pop(0) if conf.get( 'compress', None) else 'lzop' try: ssh_dest = SSH(user, host, port=port, key=dest_key, compress=compress) except (FileNotFoundError, SSHException): continue dest_name_log = '{:s}@{:s}:{:s}'.format(user, host, dest_name) else: ssh_dest = None dest_name_log = dest_name # get exclude rules exclude = conf['exclude'].pop(0) if conf.get('exclude', None) else [] # check if raw send was requested raw = conf['raw_send'].pop(0) if conf.get('raw_send', None) else False # Check if base destination filesystem exists, if not do not send try: zfs.open(dest_name, ssh=ssh_dest) except DatasetNotFoundError: logger.error( 'Destination {:s} does not exist...'.format(dest_name_log)) continue except ValueError as err: logger.error(err) continue except CalledProcessError as err: logger.error( 'Error while opening dest {:s}: \'{:s}\'...'.format( dest_name_log, err.stderr.rstrip())) continue else: # Match children on source to children on dest dest_children_names = [ child.name.replace(source_name, dest_name) for child in source_children ] # Send all children to corresponding children on dest for source_fs, dest_name in zip(source_children, dest_children_names): # exclude filesystems from rules if any( fnmatch(source_fs.name, pattern) for pattern in exclude): logger.debug( 'Matched {} in exclude rules, not sending...'. format(source_fs)) continue # send not excluded filesystems send_filesystem(source_fs, dest_name, ssh_dest=ssh_dest, raw=raw) finally: if ssh_dest: ssh_dest.close() if ssh_source: ssh_source.close()