def test_send_raw(self, zpools):
"""Checks if raw_send works"""
fs0, fs1 = zpools
fs0.destroy(force=True)
fs1.destroy(force=True)
raw_send = ['yes']
config = [{'name': fs0.name, 'dest': [fs1.name], 'raw_send': raw_send}]
zfs.create('{:s}/sub1'.format(fs0.name), props={'compression': 'gzip'})
zfs.create('{:s}/sub2'.format(fs0.name), props={'compression': 'lz4'})
zfs.create('{:s}/sub3'.format(fs0.name), props={'compression': 'gzip'})
zfs.create('{:s}/sub3/abc'.format(fs0.name))
zfs.create('{:s}/sub3/abc_abc'.format(fs0.name))
zfs.create('{:s}/sub3/efg'.format(fs0.name))
fs0.snapshot('snap', recursive=True)
send_config(config)
fs0_children = set([
child.name.replace(fs0.name, '')
for child in zfs.find(fs0.name, types=['all'])[1:]
])
fs1_children = set([
child.name.replace(fs1.name, '')
for child in zfs.find(fs1.name, types=['all'])[1:]
])
assert set(fs0_children) == set(fs1_children)
def test_send_compress(self, zpools):
"""Checks if send_snap totally replicates a filesystem"""
fs1, fs0 = zpools # here fs0 is the remote pool
ssh = fs0.ssh
fs0.destroy(force=True)
fs1.destroy(force=True)
fs0.snapshot('snap0')
zfs.create('{:s}/sub1'.format(fs0.name), ssh=ssh)
fs0.snapshot('snap1', recursive=True)
zfs.create('{:s}/sub2'.format(fs0.name), ssh=ssh)
fs0.snapshot('snap2', recursive=True)
fs0.snapshot('snap3', recursive=True)
zfs.create('{:s}/sub2/abc'.format(fs0.name), ssh=ssh)
fs0.snapshot('snap4', recursive=True)
fs0.snapshot('snap5', recursive=True)
for compression in ['none', 'lzop', 'lz4']:
fs1.destroy(force=True)
config = [{'name': 'ssh:{:d}:{}'.format(PORT, fs0), 'key': KEY, 'dest': [fs1.name], 'compress': [compression]}]
send_config(config)
fs0_children = [child.name.replace(fs0.name, '') for child in zfs.find(fs0.name, types=['all'], ssh=ssh)[1:]]
fs1_children = [child.name.replace(fs1.name, '') for child in zfs.find(fs1.name, types=['all'])[1:]]
assert set(fs0_children) == set(fs1_children)
def test_send_exclude(self, zpools):
"""Checks if exclude rules work"""
fs0, fs1 = zpools
fs0.destroy(force=True)
fs1.destroy(force=True)
exclude = ['*/sub1', '*/sub3/abc', '*/sub3/efg']
config = [{'name': fs0.name, 'dest': [fs1.name], 'exclude': [exclude]}]
zfs.create('{:s}/sub1'.format(fs0.name))
zfs.create('{:s}/sub2'.format(fs0.name))
zfs.create('{:s}/sub3'.format(fs0.name))
zfs.create('{:s}/sub3/abc'.format(fs0.name))
zfs.create('{:s}/sub3/abc_abc'.format(fs0.name))
zfs.create('{:s}/sub3/efg'.format(fs0.name))
fs0.snapshot('snap', recursive=True)
send_config(config)
fs0_children = set([
child.name.replace(fs0.name, '')
for child in zfs.find(fs0.name, types=['all'])[1:]
])
fs1_children = set([
child.name.replace(fs1.name, '')
for child in zfs.find(fs1.name, types=['all'])[1:]
])
# remove unwanted datasets/snapshots
for match in exclude:
fs0_children -= set(fnmatch.filter(fs0_children, match))
fs0_children -= set(fnmatch.filter(fs0_children, match + '@snap'))
assert set(fs0_children) == set(fs1_children)
def test_send_delete_old(self, zpools):
fs0, fs1 = zpools
ssh = fs1.ssh
# Delete old snapshot on source
fs0.snapshots()[0].destroy(force=True)
fs0.snapshot('snap7', recursive=True)
config = [{
'name': fs0.name,
'dest': ['ssh:{:d}:{}'.format(PORT, fs1)],
'dest_keys': [KEY],
'compress': None
}]
send_config(config)
fs0_children = [
child.name.replace(fs0.name, '')
for child in zfs.find(fs0.name, types=['all'])[1:]
]
fs1_children = [
child.name.replace(fs1.name, '')
for child in zfs.find(fs1.name, types=['all'], ssh=ssh)[1:]
]
assert not (set(fs0_children) == set(fs1_children))
# Assert that snap0 was not deleted from fs1
for child in set(fs1_children) - set(fs0_children):
assert child.endswith('snap0')
def test_send_exclude(self, zpools):
"""Checks if send_snap totally replicates a filesystem"""
fs1, fs0 = zpools # here fs0 is the remote pool
ssh = fs0.ssh
fs0.destroy(force=True)
fs1.destroy(force=True)
exclude = ['*/sub1', '*/sub3/abc', '*/sub3/efg']
config = [{'name': 'ssh:{:d}:{}'.format(PORT, fs0), 'dest': [fs1.name], 'exclude': [exclude]}]
zfs.create('{:s}/sub1'.format(fs0.name), ssh=ssh)
zfs.create('{:s}/sub2'.format(fs0.name), ssh=ssh)
zfs.create('{:s}/sub3'.format(fs0.name), ssh=ssh)
zfs.create('{:s}/sub3/abc'.format(fs0.name), ssh=ssh)
zfs.create('{:s}/sub3/abc_abc'.format(fs0.name), ssh=ssh)
zfs.create('{:s}/sub3/efg'.format(fs0.name), ssh=ssh)
fs0.snapshot('snap', recursive=True)
send_config(config)
fs0_children = set([child.name.replace(fs0.name, '') for child in zfs.find(fs0.name, types=['all'], ssh=ssh)[1:]])
fs1_children = set([child.name.replace(fs1.name, '') for child in zfs.find(fs1.name, types=['all'])[1:]])
# remove unwanted datasets/snapshots
for match in exclude:
fs0_children -= set(fnmatch.filter(fs0_children, match))
fs0_children -= set(fnmatch.filter(fs0_children, match + '@snap'))
assert set(fs0_children) == set(fs1_children)
def test_send_delete_sub(self, zpools):
fs0, fs1 = zpools
ssh = fs1.ssh
# Delete subfilesystems
sub3 = fs1.filesystems()[-1]
sub3.destroy(force=True)
fs0.snapshot('snap6', recursive=True)
sub2 = fs1.filesystems()[-1]
sub2.destroy(force=True)
config = [{
'name': fs0.name,
'dest': ['ssh:{:d}:{}'.format(PORT, fs1)],
'dest_keys': [KEY],
'compress': None
}]
send_config(config)
fs0_children = [
child.name.replace(fs0.name, '')
for child in zfs.find(fs0.name, types=['all'])[1:]
]
fs1_children = [
child.name.replace(fs1.name, '')
for child in zfs.find(fs1.name, types=['all'], ssh=ssh)[1:]
]
assert set(fs0_children) == set(fs1_children)
def test_send_full(self, zpools):
"""Checks if send_snap totally replicates a filesystem"""
fs0, fs1 = zpools
fs0.destroy(force=True)
fs1.destroy(force=True)
config = [{'name': fs0.name, 'dest': [fs1.name]}]
fs0.snapshot('snap0')
zfs.create('{:s}/sub1'.format(fs0.name))
fs0.snapshot('snap1', recursive=True)
zfs.create('{:s}/sub2'.format(fs0.name))
fs0.snapshot('snap2', recursive=True)
zfs.create('{:s}/sub3'.format(fs0.name))
fs0.snapshot('snap3', recursive=True)
fs0.snapshot('snap4', recursive=True)
fs0.snapshot('snap5', recursive=True)
zfs.create('{:s}/sub3/abc'.format(fs0.name))
fs0.snapshot('snap6', recursive=True)
zfs.create('{:s}/sub3/abc_abc'.format(fs0.name))
fs0.snapshot('snap7', recursive=True)
zfs.create('{:s}/sub3/efg'.format(fs0.name))
fs0.snapshot('snap8', recursive=True)
fs0.snapshot('snap9', recursive=True)
send_config(config)
fs0_children = [
child.name.replace(fs0.name, '')
for child in zfs.find(fs0.name, types=['all'])[1:]
]
fs1_children = [
child.name.replace(fs1.name, '')
for child in zfs.find(fs1.name, types=['all'])[1:]
]
assert set(fs0_children) == set(fs1_children)
def test_send_delete_snapshot(self, zpools):
fs0, fs1 = zpools
config = [{'name': fs0.name, 'dest': [fs1.name]}]
# Delete recent snapshots on dest
fs1.snapshots()[-1].destroy(force=True)
fs1.snapshots()[-1].destroy(force=True)
send_config(config)
fs0_children = [
child.name.replace(fs0.name, '')
for child in zfs.find(fs0.name, types=['all'])[1:]
]
fs1_children = [
child.name.replace(fs1.name, '')
for child in zfs.find(fs1.name, types=['all'])[1:]
]
assert set(fs0_children) == set(fs1_children)
# Delete recent snapshot on source
fs0.snapshot('snap4', recursive=True)
send_config(config)
fs0.snapshots()[-1].destroy(force=True)
fs0.snapshot('snap5', recursive=True)
send_config(config)
fs0_children = [
child.name.replace(fs0.name, '')
for child in zfs.find(fs0.name, types=['all'])[1:]
]
fs1_children = [
child.name.replace(fs1.name, '')
for child in zfs.find(fs1.name, types=['all'])[1:]
]
assert set(fs0_children) == set(fs1_children)
def test_create_new(self, zpools, config_send):
"""Tests pyznap over 10 years and checks if newly created filesystems are correctly
replicated"""
fs0, fs1 = zpools
ssh = fs1.ssh
fs0.destroy(force=True)
fs1.destroy(force=True)
# have to start at 1969 as faketime only goes from 1969 to 2068
dates = [datetime(1969 + i, 1, 1) for i in range(10)]
for n,date in enumerate(dates):
# at every step create a new subfilesystem
zfs.create('{:s}/sub{:d}'.format(fs0.name, n))
faketime = ['faketime', date.strftime('%y-%m-%d %H:%M:%S')]
pyznap_take = faketime + ['pyznap', '--config', config_send, 'snap', '--take']
pyznap_clean = faketime + ['pyznap', '--config', config_send, 'snap', '--clean']
pyznap_send = faketime + ['pyznap', '--config', config_send, 'send']
# take, send & clean snaps every 1y
_, _ = Popen(pyznap_take).communicate()
_, _ = Popen(pyznap_send).communicate()
_, _ = Popen(pyznap_clean).communicate()
# get all snapshots on fs0
snapshots = {'frequent': [], 'hourly': [], 'daily': [], 'weekly': [], 'monthly': [], 'yearly': []}
for snap in fs0.snapshots():
snap_type = snap.name.split('_')[-1]
snapshots[snap_type].append(snap)
# check if there are not too many snapshots taken
for snap_type, snaps in snapshots.items():
assert len(snaps) <= SNAPSHOTS_REF[snap_type]
# check if after N_FREQUENT runs there are N_FREQUENT 'frequent' snapshots
if n+1 >= N_FREQUENT:
assert len(snapshots['frequent']) == SNAPSHOTS_REF['frequent']
# check if after N-HOURLY runs there are N-HOURLY 'hourly' snapshots
if n+1 >= N_HOURLY:
assert len(snapshots['hourly']) == SNAPSHOTS_REF['hourly']
# check if after N_DAILY runs there are N_DAILY 'daily' snapshots
if n+1 >= N_DAILY:
assert len(snapshots['daily']) == SNAPSHOTS_REF['daily']
# check if after N_WEEKLY runs there are N_WEEKLY 'weekly' snapshots
if n+1 >= N_WEEKLY:
assert len(snapshots['weekly']) == SNAPSHOTS_REF['weekly']
# check if after N_MONTHLY runs there are N_MONTHLY 'monthly' snapshots
if n+1 >= N_MONTHLY:
assert len(snapshots['monthly']) == SNAPSHOTS_REF['monthly']
# check if after N_YEARLY runs there are N_YEARLY 'yearly' snapshots
if n+1 >= N_YEARLY:
assert len(snapshots['yearly']) == SNAPSHOTS_REF['yearly']
# check if filesystem is completely replicated on dest
fs0_children = [child.name.replace(fs0.name, '') for child in zfs.find(fs0.name, types=['all'])[1:]]
fs1_children = [child.name.replace(fs1.name, '') for child in zfs.find(fs1.name, types=['all'], ssh=ssh)[1:]]
assert set(fs0_children) == set(fs1_children)
def latest_snapshots(fsname):
logger = logging.getLogger(__name__)
# logger.debug('Cleaning snapshots on {}...'.format(filesystem))
snapshots = {'frequent': [], 'hourly': [], 'daily': [], 'weekly': [], 'monthly': [], 'yearly': []}
# catch exception if dataset was destroyed since pyznap was started
#try
ssh = None
children = zfs.find(path=fsname, types=['filesystem', 'volume'], ssh=ssh)
fs_snapshots = children[0].snapshots()
#except (DatasetNotFoundError, DatasetBusyError) as err
#except as err:
# logger.error('Error while opening {}: {}...'.format(filesystem, err))
# return 1
#from pprint import pprint
#pprint(fs_snapshots)
# Ignore snapshots not taken with pyznap or sanoid
for snap in fs_snapshots[:]: # make a copy of fs_snapshots
if not snap.name.split('@')[1].startswith(('pyznap', 'autosnap')):
fs_snapshots.remove(snap)
#pprint(fs_snapshots)
latest = str(fs_snapshots[-1])
import subprocess
command = str("zfs get -Hpo value creation " + latest)
data = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=True)
stdout, stderr = data.communicate()
return int(stdout)
def take_config(config):
"""Takes snapshots according to strategy given in config.
Parameters:
----------
config : {list of dict}
Full config list containing all strategies for different filesystems
"""
logger = logging.getLogger(__name__)
logger.info('Taking snapshots...')
for conf in config:
if not conf.get('snap', None):
continue
name = conf['name']
try:
_type, fsname, user, host, port = parse_name(name)
except ValueError as err:
logger.error('Could not parse {:s}: {}...'.format(name, err))
continue
if _type == 'ssh':
try:
ssh = SSH(user, host, port=port, key=conf['key'])
except (FileNotFoundError, SSHException):
continue
name_log = '{:s}@{:s}:{:s}'.format(user, host, fsname)
else:
ssh = None
name_log = fsname
try:
# Children includes the base filesystem (named 'fsname')
children = zfs.find(path=fsname,
types=['filesystem', 'volume'],
ssh=ssh)
except DatasetNotFoundError as err:
logger.error('Dataset {:s} does not exist...'.format(name_log))
continue
except ValueError as err:
logger.error(err)
continue
except CalledProcessError as err:
logger.error('Error while opening {:s}: \'{:s}\'...'.format(
name_log,
err.stderr.rstrip().decode()))
continue
else:
# Take recursive snapshot of parent filesystem
take_filesystem(children[0], conf)
# Take snapshot of all children that don't have all snapshots yet
for child in children[1:]:
take_filesystem(child, conf)
finally:
if ssh:
ssh.close()
def test_send_delete_snapshot(self, zpools):
fs0, fs1 = zpools
ssh = fs1.ssh
# Delete recent snapshots on dest
fs1.snapshots()[-1].destroy(force=True)
fs1.snapshots()[-1].destroy(force=True)
config = [{
'name': fs0.name,
'dest': ['ssh:{:d}:{}'.format(PORT, fs1)],
'dest_keys': [KEY],
'compress': None
}]
send_config(config)
fs0_children = [
child.name.replace(fs0.name, '')
for child in zfs.find(fs0.name, types=['all'])[1:]
]
fs1_children = [
child.name.replace(fs1.name, '')
for child in zfs.find(fs1.name, types=['all'], ssh=ssh)[1:]
]
assert set(fs0_children) == set(fs1_children)
# Delete recent snapshot on source
fs0.snapshot('snap4', recursive=True)
send_config(config)
fs0.snapshots()[-1].destroy(force=True)
fs0.snapshot('snap5', recursive=True)
config = [{
'name': fs0.name,
'dest': ['ssh:{:d}:{}'.format(PORT, fs1)],
'dest_keys': [KEY],
'compress': None
}]
send_config(config)
fs0_children = [
child.name.replace(fs0.name, '')
for child in zfs.find(fs0.name, types=['all'])[1:]
]
fs1_children = [
child.name.replace(fs1.name, '')
for child in zfs.find(fs1.name, types=['all'], ssh=ssh)[1:]
]
assert set(fs0_children) == set(fs1_children)
def test_send_delete_old(self, zpools):
fs0, fs1 = zpools
config = [{'name': fs0.name, 'dest': [fs1.name]}]
# Delete old snapshot on source
fs0.snapshots()[0].destroy(force=True)
fs0.snapshot('snap7', recursive=True)
send_config(config)
fs0_children = [
child.name.replace(fs0.name, '')
for child in zfs.find(fs0.name, types=['all'])[1:]
]
fs1_children = [
child.name.replace(fs1.name, '')
for child in zfs.find(fs1.name, types=['all'])[1:]
]
assert not (set(fs0_children) == set(fs1_children))
# Assert that snap0 was not deleted from fs1
for child in set(fs1_children) - set(fs0_children):
assert child.endswith('snap0')
def test_send_delete_sub(self, zpools):
fs0, fs1 = zpools
config = [{'name': fs0.name, 'dest': [fs1.name]}]
# Delete subfilesystems
sub3 = fs1.filesystems()[-1]
sub3.destroy(force=True)
fs0.snapshot('snap6', recursive=True)
sub2 = fs1.filesystems()[-1]
sub2.destroy(force=True)
send_config(config)
fs0_children = [
child.name.replace(fs0.name, '')
for child in zfs.find(fs0.name, types=['all'])[1:]
]
fs1_children = [
child.name.replace(fs1.name, '')
for child in zfs.find(fs1.name, types=['all'])[1:]
]
assert set(fs0_children) == set(fs1_children)
def test_send_incremental(self, zpools):
fs1, fs0 = zpools # here fs0 is the remote pool
ssh = fs0.ssh
fs0.destroy(force=True)
fs1.destroy(force=True)
fs0.snapshot('snap0', recursive=True)
zfs.create('{:s}/sub1'.format(fs0.name), ssh=ssh)
fs0.snapshot('snap1', recursive=True)
config = [{'name': 'ssh:{:d}:{}'.format(PORT, fs0), 'key': KEY, 'dest': [fs1.name], 'compress': None}]
send_config(config)
fs0_children = [child.name.replace(fs0.name, '') for child in zfs.find(fs0.name, types=['all'], ssh=ssh)[1:]]
fs1_children = [child.name.replace(fs1.name, '') for child in zfs.find(fs1.name, types=['all'])[1:]]
assert set(fs0_children) == set(fs1_children)
zfs.create('{:s}/sub2'.format(fs0.name), ssh=ssh)
fs0.snapshot('snap2', recursive=True)
config = [{'name': 'ssh:{:d}:{}'.format(PORT, fs0), 'key': KEY, 'dest': [fs1.name], 'compress': None}]
send_config(config)
fs0_children = [child.name.replace(fs0.name, '') for child in zfs.find(fs0.name, types=['all'], ssh=ssh)[1:]]
fs1_children = [child.name.replace(fs1.name, '') for child in zfs.find(fs1.name, types=['all'])[1:]]
assert set(fs0_children) == set(fs1_children)
zfs.create('{:s}/sub3'.format(fs0.name), ssh=ssh)
fs0.snapshot('snap3', recursive=True)
config = [{'name': 'ssh:{:d}:{}'.format(PORT, fs0), 'key': KEY, 'dest': [fs1.name], 'compress': None}]
send_config(config)
fs0_children = [child.name.replace(fs0.name, '') for child in zfs.find(fs0.name, types=['all'], ssh=ssh)[1:]]
fs1_children = [child.name.replace(fs1.name, '') for child in zfs.find(fs1.name, types=['all'])[1:]]
assert set(fs0_children) == set(fs1_children)
def count_snapshots(fsname,snap_freq):
logger = logging.getLogger(__name__)
# logger.debug('Cleaning snapshots on {}...'.format(filesystem))
snapshots = {'frequent': [], 'hourly': [], 'daily': [], 'weekly': [], 'monthly': [], 'yearly': []}
ssh = None
children = zfs.find(path=fsname, types=['filesystem', 'volume'], ssh=ssh)
fs_snapshots = children[0].snapshots()
# catch exception if dataset was destroyed since pyznap was started
try:
fs_snapshots = children[0].snapshots()
except (DatasetNotFoundError, DatasetBusyError) as err:
logger.error('Error while opening {}: {}...'.format(filesystem, err))
return 1
# categorize snapshots
for snap in fs_snapshots:
# Ignore snapshots not taken with pyznap or sanoid
if not snap.name.split('@')[1].startswith(('pyznap', 'autosnap')):
continue
try:
snap_type = snap.name.split('_')[-1]
snapshots[snap_type].append(snap)
except (ValueError, KeyError):
continue
# Reverse sort by time taken
for snaps in snapshots.values():
snaps.reverse()
#print(snaps)
# Count all snapshots
if snap_freq == "total":
total = snapshots.values()
total_length = sum(len(row) for row in snapshots.values())
return(total_length)
else:
# Count snapshots of a specific frequency (frequent, hourly, daily, ...)
count = len(snapshots[snap_freq])
return(count)
for snap in snapshots[snap_freq]:
return(snap)
def test_send_incremental(self, zpools):
fs0, fs1 = zpools
fs0.destroy(force=True)
fs1.destroy(force=True)
config = [{'name': fs0.name, 'dest': [fs1.name]}]
fs0.snapshot('snap0', recursive=True)
zfs.create('{:s}/sub1'.format(fs0.name))
fs0.snapshot('snap1', recursive=True)
send_config(config)
fs0_children = [
child.name.replace(fs0.name, '')
for child in zfs.find(fs0.name, types=['all'])[1:]
]
fs1_children = [
child.name.replace(fs1.name, '')
for child in zfs.find(fs1.name, types=['all'])[1:]
]
assert set(fs0_children) == set(fs1_children)
zfs.create('{:s}/sub2'.format(fs0.name))
fs0.snapshot('snap2', recursive=True)
send_config(config)
fs0_children = [
child.name.replace(fs0.name, '')
for child in zfs.find(fs0.name, types=['all'])[1:]
]
fs1_children = [
child.name.replace(fs1.name, '')
for child in zfs.find(fs1.name, types=['all'])[1:]
]
assert set(fs0_children) == set(fs1_children)
zfs.create('{:s}/sub3'.format(fs0.name))
fs0.snapshot('snap3', recursive=True)
send_config(config)
fs0_children = [
child.name.replace(fs0.name, '')
for child in zfs.find(fs0.name, types=['all'])[1:]
]
fs1_children = [
child.name.replace(fs1.name, '')
for child in zfs.find(fs1.name, types=['all'])[1:]
]
assert set(fs0_children) == set(fs1_children)
def fill_harddrive(args):
files = get_snapshot_files(find_snapshot_files())
blacklist = args.blacklist
whitelist = args.whitelist
if args.append and (args.whitelist is not None):
# Only remove files we will end up writing again
files = [file for file in files if True in [acceptedItem in str(file) for acceptedItem in args.whitelist]]
for each in files:
print("Removing old backup file: {}".format(str(each)))
remove(each)
source_children = zfs.find(path="tank", types=['filesystem', 'volume'])
#Source children is literally just the filesystems in TANK
#a child is just one filesystem
#Apply whitelist to source_children
if whitelist is not None:
source_children = [child for child in source_children if True in [acceptedItem in str(child) for acceptedItem in args.whitelist]]
#Apply blacklist to source_children
if blacklist is not None:
source_children = [child for child in source_children if not True in [excludedItem in str(child) for excludedItem in args.blacklist]]
"""
for child in source_children:
#if ("brendon" not in str(child)) and ("vmbackup" not in str(child)):
#if str(child) not in blacklist:
if not True in (excludedItem in str(child) for excludedItem in blacklist):
print("Backing up {}!".format(str(child)))
else:
print("Not backing up {}...".format(str(child)))
"""
with open("/mnt/snailback/report/noMatch.txt", "w") as no_match_file:
picker = backup_picker()
for child in source_children:
common_snap_name = get_most_recent_common_snapshot(child)
if common_snap_name:
# There is a common snapshot for this filesystem, send an incremental stream
print("Backup: " + str(child) + "@" + str(common_snap_name))
common_snap = None
for snapshot in child.snapshots():
if common_snap_name in snapshot.snapname():
common_snap = snapshot
print("Source: " + str(common_snap))
snapshot = child.snapshots()[-1]
if args.dryrun:
picker.add_backup(snapshot, common_snap)
else:
filename = '/mnt/snailback/' + snapshot.name.split('@')[0].split('/')[-1] + '.gzip'
snapshot.send_to_file(filename, base=common_snap, intermediates=True)
else:
# There is not common snapshot for this filesystem, send the latest monthly snapshot
snapshot = get_most_recent(child.snapshots(), "monthly")
if args.dryrun:
picker.add_backup(snapshot)
else:
no_match_file.write(str(snapshot.name) + "\n")
filename = '/mnt/snailback/' + snapshot.name.split('@')[0].split('/')[-1] + '.gzip'
snapshot.send_to_file(filename)
print("Found {} total backups to make".format(str(picker.num_children)))
print("Need {} bytes disk space to backup".format(str(picker.size_total)))
print("Need {} KB disk space to backup".format(str(picker.size_total/1000)))
print("Need {} MB disk space to backup".format(str(picker.size_total/1000000)))
print("Need {} GB disk space to backup".format(str(picker.size_total/1000000000)))
print("Need {} TB disk space to backup".format(str(picker.size_total/1000000000000)))
picker.show_data()
def send_config(config):
"""Tries to sync all entries in the config to their dest. Finds all children of the filesystem
and calls send_filesystem on each of them.
Parameters:
----------
config : {list of dict}
Full config list containing all strategies for different filesystems
"""
logger = logging.getLogger(__name__)
logger.info('Sending snapshots...')
for conf in config:
if not conf.get('dest', None):
continue
backup_source = conf['name']
try:
_type, source_name, user, host, port = parse_name(backup_source)
except ValueError as err:
logger.error('Could not parse {:s}: {}...'.format(
backup_source, err))
continue
# if source is remote, open ssh connection
if _type == 'ssh':
key = conf['key'] if conf.get('key', None) else None
compress = conf['compress'].pop(0) if conf.get('compress',
None) else 'lzop'
try:
ssh_source = SSH(user,
host,
port=port,
key=key,
compress=compress)
except (FileNotFoundError, SSHException):
continue
source_name_log = '{:s}@{:s}:{:s}'.format(user, host, source_name)
else:
ssh_source = None
source_name_log = source_name
try:
# Children includes the base filesystem (named 'source_name')
source_children = zfs.find(path=source_name,
types=['filesystem', 'volume'],
ssh=ssh_source)
except DatasetNotFoundError as err:
logger.error(
'Source {:s} does not exist...'.format(source_name_log))
continue
except ValueError as err:
logger.error(err)
continue
except CalledProcessError as err:
logger.error('Error while opening source {:s}: \'{:s}\'...'.format(
source_name_log, err.stderr.rstrip()))
continue
# Send to every backup destination
for backup_dest in conf['dest']:
try:
_type, dest_name, user, host, port = parse_name(backup_dest)
except ValueError as err:
logger.error('Could not parse {:s}: {}...'.format(
backup_dest, err))
continue
# if dest is remote, open ssh connection
if _type == 'ssh':
dest_key = conf['dest_keys'].pop(0) if conf.get(
'dest_keys', None) else None
# if 'ssh_source' is set, then 'compress' is already set and we use same compression for both source and dest
# if not then we take the next entry in config
if not ssh_source:
compress = conf['compress'].pop(0) if conf.get(
'compress', None) else 'lzop'
try:
ssh_dest = SSH(user,
host,
port=port,
key=dest_key,
compress=compress)
except (FileNotFoundError, SSHException):
continue
dest_name_log = '{:s}@{:s}:{:s}'.format(user, host, dest_name)
else:
ssh_dest = None
dest_name_log = dest_name
# get exclude rules
exclude = conf['exclude'].pop(0) if conf.get('exclude',
None) else []
# check if raw send was requested
raw = conf['raw_send'].pop(0) if conf.get('raw_send',
None) else False
# Check if base destination filesystem exists, if not do not send
try:
zfs.open(dest_name, ssh=ssh_dest)
except DatasetNotFoundError:
logger.error(
'Destination {:s} does not exist...'.format(dest_name_log))
continue
except ValueError as err:
logger.error(err)
continue
except CalledProcessError as err:
logger.error(
'Error while opening dest {:s}: \'{:s}\'...'.format(
dest_name_log, err.stderr.rstrip()))
continue
else:
# Match children on source to children on dest
dest_children_names = [
child.name.replace(source_name, dest_name)
for child in source_children
]
# Send all children to corresponding children on dest
for source_fs, dest_name in zip(source_children,
dest_children_names):
# exclude filesystems from rules
if any(
fnmatch(source_fs.name, pattern)
for pattern in exclude):
logger.debug(
'Matched {} in exclude rules, not sending...'.
format(source_fs))
continue
# send not excluded filesystems
send_filesystem(source_fs,
dest_name,
ssh_dest=ssh_dest,
raw=raw)
finally:
if ssh_dest:
ssh_dest.close()
if ssh_source:
ssh_source.close()
def clean_config(config):
"""Deletes old snapshots according to strategies given in config. Goes through each config,
opens up ssh connection if necessary and then recursively calls clean_filesystem.
Parameters:
----------
config : {list of dict}
Full config list containing all strategies for different filesystems
"""
logger = logging.getLogger(__name__)
logger.info('Cleaning snapshots...')
for conf in config:
if not conf.get('clean', None):
continue
name = conf['name']
try:
_type, fsname, user, host, port = parse_name(name)
except ValueError as err:
logger.error('Could not parse {:s}: {}...'.format(name, err))
continue
if _type == 'ssh':
try:
ssh = SSH(user, host, port=port, key=conf['key'])
except (FileNotFoundError, SSHException):
continue
name_log = '{:s}@{:s}:{:s}'.format(user, host, fsname)
else:
ssh = None
name_log = fsname
try:
# Children includes the base filesystem (named 'fsname')
children = zfs.find(path=fsname,
types=['filesystem', 'volume'],
ssh=ssh)
except DatasetNotFoundError as err:
logger.error('Dataset {:s} does not exist...'.format(name_log))
continue
except ValueError as err:
logger.error(err)
continue
except CalledProcessError as err:
logger.error('Error while opening {:s}: \'{:s}\'...'.format(
name_log,
err.stderr.rstrip().decode()))
else:
# Clean snapshots of parent filesystem
clean_filesystem(children[0], conf)
# Clean snapshots of all children that don't have a seperate config entry
for child in children[1:]:
# Check if any of the parents (but child of base filesystem) have a config entry
for parent in children[1:]:
if ssh:
child_name = 'ssh:{:d}:{:s}@{:s}:{:s}'.format(
port, user, host, child.name)
parent_name = 'ssh:{:d}:{:s}@{:s}:{:s}'.format(
port, user, host, parent.name)
else:
child_name = child.name
parent_name = parent.name
# Skip if child has an entry or if any parent entry already in config
child_parent = '/'.join(child_name.split(
'/')[:-1]) # get parent of child filesystem
if ((child_name == parent_name
or child_parent.startswith(parent_name)) and
(parent_name in [entry['name'] for entry in config])):
break
else:
clean_filesystem(child, conf)
finally:
if ssh:
ssh.close()
def send_config(config):
"""Tries to sync all entries in the config to their dest. Finds all children of the filesystem
and calls send_filesystem on each of them.
Parameters:
----------
config : {list of dict}
Full config list containing all strategies for different filesystems
"""
logger = logging.getLogger(__name__)
logger.info('Sending snapshots...')
for conf in config:
if not conf.get('dest', None):
continue
dry_run = conf.get('dry_run', None)
dry_msg = '*** DRY RUN ***' if dry_run else ''
backup_source = conf['name']
try:
_type, source_name, user, host, port = parse_name(backup_source)
except ValueError as err:
logger.error('Could not parse {:s}: {}...'.format(
backup_source, err))
continue
# if source is remote, open ssh connection
if _type == 'ssh':
key = conf['key'] if conf.get('key', None) else None
compress = conf['compress'].pop(0) if conf.get('compress',
None) else 'lzop'
try:
ssh_source = SSH(user,
host,
port=port,
key=key,
compress=compress)
except (FileNotFoundError, SSHException):
continue
source_name_log = '{:s}@{:s}:{:s}'.format(user, host, source_name)
else:
ssh_source = None
source_name_log = source_name
try:
# Children includes the base filesystem (named 'source_name')
source_children = zfs.find(path=source_name,
types=['filesystem', 'volume'],
ssh=ssh_source)
except DatasetNotFoundError as err:
logger.error(
'Source {:s} does not exist...'.format(source_name_log))
continue
except ValueError as err:
logger.error(err)
continue
except CalledProcessError as err:
logger.error('Error while opening source {:s}: \'{:s}\'...'.format(
source_name_log, err.stderr.rstrip()))
continue
# Send to every backup destination
for backup_dest in conf['dest']:
# get exclude rules
exclude = conf['exclude'].pop(0) if conf.get('exclude',
None) else []
# check if raw send was requested
raw = conf['raw_send'].pop(0) if conf.get('raw_send',
None) else False
# check if we need to retry
retries = conf['retries'].pop(0) if conf.get('retries',
None) else 0
retry_interval = conf['retry_interval'].pop(0) if conf.get(
'retry_interval', None) else 10
# check if resumable send was requested
resume = conf['resume'].pop(0) if conf.get('resume',
None) else False
# check if we should create dataset if it doesn't exist
dest_auto_create = conf['dest_auto_create'].pop(0) if conf.get(
'dest_auto_create', None) else False
try:
_type, dest_name, user, host, port = parse_name(backup_dest)
except ValueError as err:
logger.error('Could not parse {:s}: {}...'.format(
backup_dest, err))
continue
# if dest is remote, open ssh connection
if _type == 'ssh':
dest_key = conf['dest_keys'].pop(0) if conf.get(
'dest_keys', None) else None
# if 'ssh_source' is set, then 'compress' is already set and we use same compression for both source and dest
# if not then we take the next entry in config
if not ssh_source:
compress = conf['compress'].pop(0) if conf.get(
'compress', None) else 'lzop'
try:
ssh_dest = SSH(user,
host,
port=port,
key=dest_key,
compress=compress)
except (FileNotFoundError, SSHException):
continue
dest_name_log = '{:s}@{:s}:{:s}'.format(user, host, dest_name)
else:
ssh_dest = None
dest_name_log = dest_name
# check if dest exists
try:
zfs.open(dest_name, ssh=ssh_dest)
except DatasetNotFoundError:
if dest_auto_create:
logger.info(
'Destination {:s} does not exist, will create it... {}'
.format(dest_name_log, dry_msg))
if create_dataset(dest_name,
dest_name_log,
ssh=ssh_dest,
dry_run=dry_run):
continue
else:
logger.error(
'Destination {:s} does not exist, manually create it or use "dest-auto-create" option...'
.format(dest_name_log))
continue
except ValueError as err:
logger.error(err)
continue
except CalledProcessError as err:
logger.error(
'Error while opening dest {:s}: \'{:s}\'...'.format(
dest_name_log, err.stderr.rstrip()))
continue
# Match children on source to children on dest
dest_children_names = [
child.name.replace(source_name, dest_name)
for child in source_children
]
# Send all children to corresponding children on dest
for source_fs, dest_name in zip(source_children,
dest_children_names):
# exclude filesystems from rules
if any(
fnmatch(source_fs.name, pattern)
for pattern in exclude):
logger.debug(
'Matched {} in exclude rules, not sending...'.format(
source_fs))
continue
# Check for ZFS user property to bypass filesystem
fs_props = source_fs.getprops()
exclude_prop = 'pyznap:exclude'
ignore_me = fs_props.get(exclude_prop,
('false', 'false'))[0].lower()
logger.debug("Property {}={} for {}".format(
exclude_prop, ignore_me, source_fs))
if ignore_me == 'true':
logger.info('Matched {}={} for {}, not sending...'.format(
exclude_prop, ignore_me, source_fs))
continue
# Check for max size
used_prop = 'used'
fs_used_bytes = int(fs_props.get(used_prop,
('0', '0'))[0]) # Bytes
fs_used_fmt = bytes_fmt(fs_used_bytes) # MB
logger.debug("Property {}={} ({}) for {}".format(
used_prop, fs_used_fmt, fs_used_bytes, source_fs))
max_prop = 'pyznap:max_size'
fs_max_fmt = fs_props.get(max_prop, ('0', '0'))[0] # String
fs_max_bytes = parse_size(fs_max_fmt) # Bytes
logger.debug("Property {}={} ({}) for {}".format(
max_prop, fs_max_fmt, fs_max_bytes, source_fs))
if fs_max_bytes > 0 and fs_used_bytes > fs_max_bytes:
logger.info(
'Filesystem size {} exceeds {}={} for {}, not sending...'
.format(fs_used_fmt, max_prop, fs_max_fmt, source_fs))
continue
# send not excluded filesystems
for retry in range(1, retries + 2):
rc = send_filesystem(source_fs,
dest_name,
ssh_dest=ssh_dest,
raw=raw,
resume=resume,
dry_run=dry_run)
if rc == 2 and retry <= retries:
logger.info(
'Retrying send in {:d}s (retry {:d} of {:d})...'.
format(retry_interval, retry, retries))
sleep(retry_interval)
else:
break
if ssh_dest:
ssh_dest.close()
if ssh_source:
ssh_source.close()
def send_config(config):
"""Tries to sync all entries in the config to their dest. Finds all children of the filesystem
and calls send_snap on each of them.
Parameters:
----------
config : {list of dict}
Full config list containing all strategies for different filesystems
"""
logger = logging.getLogger(__name__)
logger.info('Sending snapshots...')
for conf in config:
if not conf.get('dest', None):
continue
source_name = conf['name']
if source_name.startswith('ssh'):
logger.error('Cannot send from remote location ({:s})...'.format(
source_name))
continue
try:
# Children includes the base filesystem (named 'source_fs')
source_children = zfs.find(path=source_name,
types=['filesystem', 'volume'])
except DatasetNotFoundError as err:
logger.error('Source {:s} does not exist...'.format(source_name))
continue
except ValueError as err:
logger.error(err)
continue
except CalledProcessError as err:
logger.error('Error while opening source {:s}: \'{:s}\'...'.format(
source_name, err.stderr.rstrip()))
continue
# Send to every backup destination
for backup_dest in conf['dest']:
try:
_type, dest_name, user, host, port = parse_name(backup_dest)
except ValueError as err:
logger.error('Could not parse {:s}: {}...'.format(
backup_dest, err))
continue
if _type == 'ssh':
dest_key = conf['dest_keys'].pop(
0) if conf['dest_keys'] else None
try:
ssh = open_ssh(user, host, port=port, key=dest_key)
except (FileNotFoundError, SSHException):
continue
dest_name_log = '{:s}@{:s}:{:s}'.format(user, host, dest_name)
else:
ssh = None
dest_name_log = dest_name
# Check if base destination filesystem exists, if not do not send
try:
zfs.open(dest_name, ssh=ssh)
except DatasetNotFoundError:
logger.error(
'Destination {:s} does not exist...'.format(dest_name_log))
continue
except ValueError as err:
logger.error(err)
continue
except CalledProcessError as err:
logger.error(
'Error while opening dest {:s}: \'{:s}\'...'.format(
dest_name_log, err.stderr.rstrip()))
continue
else:
# Match children on source to children on dest
dest_children_names = [
child.name.replace(source_name, dest_name)
for child in source_children
]
# Send all children to corresponding children on dest
for source, dest in zip(source_children, dest_children_names):
send_snap(source, dest, ssh=ssh)
finally:
if ssh:
ssh.close()
def fix_snapshots(filesystems,
format=None,
type=None,
type_map=None,
recurse=False):
"""Fix snapshots name
Parameters:
----------
filesystems : [strings]
Filesystems to fix
"""
logger = logging.getLogger(__name__)
if format.startswith('@'):
if not type_map and format in MAPS:
type_map = MAPS[format]
if format in FORMATS:
format = FORMATS[format]
else:
logger.error('Unknown format {}.'.format(format))
sys.exit(1)
logger.debug('FORMAT: ' + str(format))
logger.debug('MAP: ' + str(type_map))
rp = re.compile(format)
now = datetime.now()
cur_century = int(now.year / 100) * 100
# for all specified filesystems
for fsname in filesystems:
logger.info('Checking snapshots on {}...'.format(fsname))
try:
parent = zfs.open(fsname)
except DatasetNotFoundError:
logger.error('Filesystem not exists {}'.format(fsname))
continue
if recurse:
# get all child's filesystem
fstree = zfs.find(fsname, types=['filesystem', 'volume'])
else:
# only scan specified filesystem
fstree = [parent]
for filesystem in fstree:
logger.info('Fixing {}...'.format(filesystem.name))
snapshots = filesystem.snapshots()
for snapshot in snapshots:
snapname = snapshot.snapname()
try:
r = rp.match(snapname)
except:
r = False
if r:
# guess year
year = re_get_group_int(r, 'year', default=now.year)
if year < 100:
year += +cur_century
# get type from snap, with optional map or default type if specified
snaptype = r.group('type')
if type_map:
if snaptype in type_map:
snaptype = type_map[snaptype]
if not snaptype and type:
snaptype = type
if not snaptype:
logger.error(
'Unknown snap type {} for snapshot {}'.format(
snaptype, snapname))
continue
new_snapname = 'pyznap_' + datetime(
year,
re_get_group_int(r, 'month', default=now.month),
re_get_group_int(r, 'day', default=now.day),
hour=re_get_group_int(r, 'hour', default=now.hour),
minute=re_get_group_int(
r, 'minute', default=now.minute),
second=re_get_group_int(
r, 'second', default=now.second)).strftime(
'%Y-%m-%d_%H:%M:%S') + '_' + snaptype
logger.debug('Renaming {} -> {}'.format(
snapname, new_snapname))
snapshot.rename(snapshot.fsname() + '@' + new_snapname)
if snapshot != snapshot_to_keep:
print(
"{} is not the snapshot to keep {}, destroying...".format(
str(snapshot), str(snapshot_to_keep)))
snapshot.destroy()
def prune_syncoid(snapshots):
""" prune syncoid snapshots. Call only once they are no longer needed """
for snapshot in snapshots:
if "syncoid" in str(snapshot):
print("Pruning syncoid snapshot {}...".format(str(snapshot)))
snapshot.destroy()
source_children = zfs.find(path="superior/tankbackup",
types=['filesystem', 'volume'])
print("Checking that report directory exists...")
if not path.isdir('/mnt/snailback/report') or not path.exists(
'/mnt/snailback/report'):
raise NotADirectoryError(
"/mnt/snailback/report does not exist, verify drive is mounted: mount /dev/sdg1 /mnt/snailback"
)
print("Opening /mnt/snailback/report/backups.txt")
with open("/mnt/snailback/report/backups.txt", "w") as backup_file:
print("Most recent montly snapshots in superior/tankbackup are:")
for child in source_children:
snapshots = child.snapshots()
if len(snapshots) > 0:
for period in ['yearly', 'monthly', 'weekly', 'daily', 'hourly']:
most_recent = get_most_recent(snapshots, period)
