def test_send_full(self, zpools):
"""Checks if send_snap totally replicates a filesystem"""
fs0, fs1 = zpools
fs0.destroy(force=True)
fs1.destroy(force=True)
config = [{'name': fs0.name, 'dest': [fs1.name]}]
fs0.snapshot('snap0')
zfs.create('{:s}/sub1'.format(fs0.name))
fs0.snapshot('snap1', recursive=True)
zfs.create('{:s}/sub2'.format(fs0.name))
fs0.snapshot('snap2', recursive=True)
zfs.create('{:s}/sub3'.format(fs0.name))
fs0.snapshot('snap3', recursive=True)
fs0.snapshot('snap4', recursive=True)
fs0.snapshot('snap5', recursive=True)
zfs.create('{:s}/sub3/abc'.format(fs0.name))
fs0.snapshot('snap6', recursive=True)
zfs.create('{:s}/sub3/efg'.format(fs0.name))
fs0.snapshot('snap7', recursive=True)
fs0.snapshot('snap8', recursive=True)
send_config(config)
fs0_children = [
child.name.replace(fs0.name, '')
for child in zfs.find(fs0.name, types=['all'])[1:]
]
fs1_children = [
child.name.replace(fs1.name, '')
for child in zfs.find(fs1.name, types=['all'])[1:]
]
assert set(fs0_children) == set(fs1_children)
def test_send_delete_snapshot(self, zpools):
fs0, fs1 = zpools
config = [{'name': fs0.name, 'dest': [fs1.name]}]
# Delete recent snapshots on dest
fs1.snapshots()[-1].destroy(force=True)
fs1.snapshots()[-1].destroy(force=True)
send_config(config)
fs0_children = [
child.name.replace(fs0.name, '')
for child in zfs.find(fs0.name, types=['all'])[1:]
]
fs1_children = [
child.name.replace(fs1.name, '')
for child in zfs.find(fs1.name, types=['all'])[1:]
]
assert set(fs0_children) == set(fs1_children)
# Delete recent snapshot on source
fs0.snapshot('snap4', recursive=True)
send_config(config)
fs0.snapshots()[-1].destroy(force=True)
fs0.snapshot('snap5', recursive=True)
send_config(config)
fs0_children = [
child.name.replace(fs0.name, '')
for child in zfs.find(fs0.name, types=['all'])[1:]
]
fs1_children = [
child.name.replace(fs1.name, '')
for child in zfs.find(fs1.name, types=['all'])[1:]
]
assert set(fs0_children) == set(fs1_children)
def test_send_delete_old(self, zpools):
fs0, fs1 = zpools
ssh = fs1.ssh
config = [{
'name': fs0.name,
'dest': ['ssh:{:d}:{}'.format(PORT, fs1)],
'dest_keys': [KEY]
}]
# Delete old snapshot on source
fs0.snapshots()[0].destroy(force=True)
fs0.snapshot('snap7', recursive=True)
send_config(config)
fs0_children = [
child.name.replace(fs0.name, '')
for child in zfs.find(fs0.name, types=['all'])[1:]
]
fs1_children = [
child.name.replace(fs1.name, '')
for child in zfs.find(fs1.name, types=['all'], ssh=ssh)[1:]
]
assert not (set(fs0_children) == set(fs1_children))
# Assert that snap0 was not deleted from fs1
for child in set(fs1_children) - set(fs0_children):
assert child.endswith('snap0')
def test_send_delete_sub(self, zpools):
fs0, fs1 = zpools
ssh = fs1.ssh
config = [{
'name': fs0.name,
'dest': ['ssh:{:d}:{}'.format(PORT, fs1)],
'dest_keys': [KEY]
}]
# Delete subfilesystems
sub3 = fs1.filesystems()[-1]
sub3.destroy(force=True)
fs0.snapshot('snap6', recursive=True)
sub2 = fs1.filesystems()[-1]
sub2.destroy(force=True)
send_config(config)
fs0_children = [
child.name.replace(fs0.name, '')
for child in zfs.find(fs0.name, types=['all'])[1:]
]
fs1_children = [
child.name.replace(fs1.name, '')
for child in zfs.find(fs1.name, types=['all'], ssh=ssh)[1:]
]
assert set(fs0_children) == set(fs1_children)
def clean_config(config):
"""Deletes old snapshots according to strategy given in config"""
logtime = lambda: datetime.now().strftime('%b %d %H:%M:%S')
print('{:s} INFO: Cleaning snapshots...'.format(logtime()))
for conf in config:
if not conf.get('clean', None):
continue
name = conf['name']
try:
_type, fsname, user, host, port = parse_name(name)
except ValueError as err:
print('{:s} ERROR: Could not parse {:s}: {}...'.format(
logtime(), name, err))
continue
if _type == 'ssh':
try:
ssh = open_ssh(user, host, port=port, key=conf['key'])
except (FileNotFoundError, SSHException):
continue
else:
ssh = None
try:
# Children includes the base filesystem (filesystem)
children = zfs.find(path=fsname,
types=['filesystem', 'volume'],
ssh=ssh)
except (ValueError, DatasetNotFoundError, CalledProcessError) as err:
print('{:s} ERROR: {}'.format(logtime(), err))
continue
# Clean snapshots of parent filesystem
clean_snap(children[0], conf)
# Clean snapshots of all children that don't have a seperate config entry
for child in children[1:]:
# Check if any of the parents (but child of base filesystem) have a config entry
for parent in children[1:]:
if ssh:
parent_name = 'ssh:{:d}:{:s}@{:s}:{:s}'.format(
port, user, host, parent.name)
else:
parent_name = parent.name
# Skip if any parent entry already in config
if (child.name.startswith(parent.name)
and parent_name in [entry['name']
for entry in config]):
break
else:
clean_snap(child, conf)
if ssh:
ssh.close()
def test_send_delete_sub(self, zpools):
fs0, fs1 = zpools
config = [{'name': fs0.name, 'dest': [fs1.name]}]
# Delete subfilesystems
sub3 = fs1.filesystems()[-1]
sub3.destroy(force=True)
fs0.snapshot('snap6', recursive=True)
sub2 = fs1.filesystems()[-1]
sub2.destroy(force=True)
send_config(config)
fs0_children = [
child.name.replace(fs0.name, '')
for child in zfs.find(fs0.name, types=['all'])[1:]
]
fs1_children = [
child.name.replace(fs1.name, '')
for child in zfs.find(fs1.name, types=['all'])[1:]
]
assert set(fs0_children) == set(fs1_children)
def test_send_delete_snapshot(self, zpools):
fs0, fs1 = zpools
ssh = fs1.ssh
config = [{
'name': fs0.name,
'dest': ['ssh:{:d}:{}'.format(PORT, fs1)],
'dest_keys': [KEY]
}]
# Delete recent snapshots on dest
fs1.snapshots()[-1].destroy(force=True)
fs1.snapshots()[-1].destroy(force=True)
send_config(config)
fs0_children = [
child.name.replace(fs0.name, '')
for child in zfs.find(fs0.name, types=['all'])[1:]
]
fs1_children = [
child.name.replace(fs1.name, '')
for child in zfs.find(fs1.name, types=['all'], ssh=ssh)[1:]
]
assert set(fs0_children) == set(fs1_children)
# Delete recent snapshot on source
fs0.snapshot('snap4', recursive=True)
send_config(config)
fs0.snapshots()[-1].destroy(force=True)
fs0.snapshot('snap5', recursive=True)
send_config(config)
fs0_children = [
child.name.replace(fs0.name, '')
for child in zfs.find(fs0.name, types=['all'])[1:]
]
fs1_children = [
child.name.replace(fs1.name, '')
for child in zfs.find(fs1.name, types=['all'], ssh=ssh)[1:]
]
assert set(fs0_children) == set(fs1_children)
def test_send_incremental(self, zpools):
fs0, fs1 = zpools
ssh = fs1.ssh
fs0.destroy(force=True)
fs1.destroy(force=True)
config = [{
'name': fs0.name,
'dest': ['ssh:{:d}:{}'.format(PORT, fs1)],
'dest_keys': [KEY]
}]
fs0.snapshot('snap0', recursive=True)
zfs.create('{:s}/sub1'.format(fs0.name))
fs0.snapshot('snap1', recursive=True)
send_config(config)
fs0_children = [
child.name.replace(fs0.name, '')
for child in zfs.find(fs0.name, types=['all'])[1:]
]
fs1_children = [
child.name.replace(fs1.name, '')
for child in zfs.find(fs1.name, types=['all'], ssh=ssh)[1:]
]
assert set(fs0_children) == set(fs1_children)
zfs.create('{:s}/sub2'.format(fs0.name))
fs0.snapshot('snap2', recursive=True)
send_config(config)
fs0_children = [
child.name.replace(fs0.name, '')
for child in zfs.find(fs0.name, types=['all'])[1:]
]
fs1_children = [
child.name.replace(fs1.name, '')
for child in zfs.find(fs1.name, types=['all'], ssh=ssh)[1:]
]
assert set(fs0_children) == set(fs1_children)
zfs.create('{:s}/sub3'.format(fs0.name))
fs0.snapshot('snap3', recursive=True)
send_config(config)
fs0_children = [
child.name.replace(fs0.name, '')
for child in zfs.find(fs0.name, types=['all'])[1:]
]
fs1_children = [
child.name.replace(fs1.name, '')
for child in zfs.find(fs1.name, types=['all'], ssh=ssh)[1:]
]
assert set(fs0_children) == set(fs1_children)
def send_config(config):
"""Tries to sync all entries in the config to their dest. Finds all children of the filesystem
and calls send_snap on each of them."""
logtime = lambda: datetime.now().strftime('%b %d %H:%M:%S')
print('{:s} INFO: Sending snapshots...'.format(logtime()))
for conf in config:
if not conf.get('dest', None):
continue
source_fs_name = conf['name']
if source_fs_name.startswith('ssh'):
print('{:s} ERROR: Cannot send from remote location...'.format(logtime()))
continue
try:
# Children includes the base filesystem (source_fs)
source_children = zfs.find(path=source_fs_name, types=['filesystem', 'volume'])
except (ValueError, DatasetNotFoundError, CalledProcessError) as err:
print('{:s} ERROR: {}'.format(logtime(), err))
continue
for backup_dest in conf['dest']:
try:
_type, dest_name, user, host, port = parse_name(backup_dest)
except ValueError as err:
print('{:s} ERROR: Could not parse {:s}: {}...'
.format(logtime(), backup_dest, err))
continue
if _type == 'ssh':
dest_key = conf['dest_keys'].pop(0) if conf['dest_keys'] else None
try:
ssh = open_ssh(user, host, port=port, key=dest_key)
except (FileNotFoundError, SSHException):
continue
dest_name_log = '{:s}@{:s}:{:s}'.format(user, host, dest_name)
else:
ssh = None
dest_name_log = dest_name
# Check if base destination filesystem exists
try:
zfs.open(dest_name, ssh=ssh)
except DatasetNotFoundError:
print('{:s} ERROR: Destination {:s} does not exist...'
.format(logtime(), dest_name_log))
continue
except (ValueError, CalledProcessError) as err:
print('{:s} ERROR: {}'.format(logtime(), err))
continue
# Match children on source to children on dest
dest_children_names = [child.name.replace(source_fs_name, dest_name) for
child in source_children]
# Send all children to corresponding children on dest
for source, dest in zip(source_children, dest_children_names):
send_snap(source, dest, ssh=ssh)
if ssh:
ssh.close()