def test_create_new(self, zpools, config_send):
"""Tests pyznap over 10 years and checks if newly created filesystems are correctly
replicated"""
fs0, fs1 = zpools
ssh = fs1.ssh
fs0.destroy(force=True)
fs1.destroy(force=True)
# have to start at 1969 as faketime only goes from 1969 to 2068
dates = [datetime(1969 + i, 1, 1) for i in range(10)]
for n,date in enumerate(dates):
# at every step create a new subfilesystem
zfs.create('{:s}/sub{:d}'.format(fs0.name, n))
faketime = ['faketime', date.strftime('%y-%m-%d %H:%M:%S')]
pyznap_take = faketime + ['pyznap', '--config', config_send, 'snap', '--take']
pyznap_clean = faketime + ['pyznap', '--config', config_send, 'snap', '--clean']
pyznap_send = faketime + ['pyznap', '--config', config_send, 'send']
# take, send & clean snaps every 1y
_, _ = Popen(pyznap_take).communicate()
_, _ = Popen(pyznap_send).communicate()
_, _ = Popen(pyznap_clean).communicate()
# get all snapshots on fs0
snapshots = {'frequent': [], 'hourly': [], 'daily': [], 'weekly': [], 'monthly': [], 'yearly': []}
for snap in fs0.snapshots():
snap_type = snap.name.split('_')[-1]
snapshots[snap_type].append(snap)
# check if there are not too many snapshots taken
for snap_type, snaps in snapshots.items():
assert len(snaps) <= SNAPSHOTS_REF[snap_type]
# check if after N_FREQUENT runs there are N_FREQUENT 'frequent' snapshots
if n+1 >= N_FREQUENT:
assert len(snapshots['frequent']) == SNAPSHOTS_REF['frequent']
# check if after N-HOURLY runs there are N-HOURLY 'hourly' snapshots
if n+1 >= N_HOURLY:
assert len(snapshots['hourly']) == SNAPSHOTS_REF['hourly']
# check if after N_DAILY runs there are N_DAILY 'daily' snapshots
if n+1 >= N_DAILY:
assert len(snapshots['daily']) == SNAPSHOTS_REF['daily']
# check if after N_WEEKLY runs there are N_WEEKLY 'weekly' snapshots
if n+1 >= N_WEEKLY:
assert len(snapshots['weekly']) == SNAPSHOTS_REF['weekly']
# check if after N_MONTHLY runs there are N_MONTHLY 'monthly' snapshots
if n+1 >= N_MONTHLY:
assert len(snapshots['monthly']) == SNAPSHOTS_REF['monthly']
# check if after N_YEARLY runs there are N_YEARLY 'yearly' snapshots
if n+1 >= N_YEARLY:
assert len(snapshots['yearly']) == SNAPSHOTS_REF['yearly']
# check if filesystem is completely replicated on dest
fs0_children = [child.name.replace(fs0.name, '') for child in zfs.find(fs0.name, types=['all'])[1:]]
fs1_children = [child.name.replace(fs1.name, '') for child in zfs.find(fs1.name, types=['all'], ssh=ssh)[1:]]
assert set(fs0_children) == set(fs1_children)
def test_take_snapshot_recursive(self, zpools):
_, fs = zpools
ssh = fs.ssh
fs.destroy(force=True)
config = [{'name': 'ssh:{:d}:{}'.format(PORT, fs), 'key': KEY, 'frequent': 1, 'hourly': 1,
'daily': 1, 'weekly': 1, 'monthly': 1, 'yearly': 1, 'snap': True}]
take_config(config)
fs.snapshots()[-1].destroy(force=True)
fs.snapshots()[-1].destroy(force=True)
sub1 = zfs.create('{:s}/sub1'.format(fs.name), ssh=ssh)
abc = zfs.create('{:s}/sub1/abc'.format(fs.name), ssh=ssh)
sub1_abc = zfs.create('{:s}/sub1_abc'.format(fs.name), ssh=ssh)
config += [{'name': 'ssh:{:d}:{}/sub1'.format(PORT, fs), 'key': KEY, 'frequent': 1, 'hourly': 1,
'daily': 1, 'weekly': 1, 'monthly': 1, 'yearly': 1, 'snap': False}]
take_config(config)
# Check fs
snapshots = {'frequent': [], 'hourly': [], 'daily': [], 'weekly': [], 'monthly': [], 'yearly': []}
for snap in fs.snapshots():
snap_type = snap.name.split('_')[-1]
snapshots[snap_type].append(snap)
for snap_type, snaps in snapshots.items():
assert len(snaps) == config[0][snap_type]
# Check sub1
snapshots = {'frequent': [], 'hourly': [], 'daily': [], 'weekly': [], 'monthly': [], 'yearly': []}
for snap in sub1.snapshots():
snap_type = snap.name.split('_')[-1]
snapshots[snap_type].append(snap)
for snap_type, snaps in snapshots.items():
assert len(snaps) == config[0][snap_type]
# Check abc
snapshots = {'frequent': [], 'hourly': [], 'daily': [], 'weekly': [], 'monthly': [], 'yearly': []}
for snap in abc.snapshots():
snap_type = snap.name.split('_')[-1]
snapshots[snap_type].append(snap)
for snap_type, snaps in snapshots.items():
assert len(snaps) == config[0][snap_type]
# Check sub1_abc
snapshots = {'frequent': [], 'hourly': [], 'daily': [], 'weekly': [], 'monthly': [], 'yearly': []}
for snap in sub1_abc.snapshots():
snap_type = snap.name.split('_')[-1]
snapshots[snap_type].append(snap)
for snap_type, snaps in snapshots.items():
assert len(snaps) == config[0][snap_type]
def test_send_compress(self, zpools):
"""Checks if send_snap totally replicates a filesystem"""
fs1, fs0 = zpools # here fs0 is the remote pool
ssh = fs0.ssh
fs0.destroy(force=True)
fs1.destroy(force=True)
fs0.snapshot('snap0')
zfs.create('{:s}/sub1'.format(fs0.name), ssh=ssh)
fs0.snapshot('snap1', recursive=True)
zfs.create('{:s}/sub2'.format(fs0.name), ssh=ssh)
fs0.snapshot('snap2', recursive=True)
fs0.snapshot('snap3', recursive=True)
zfs.create('{:s}/sub2/abc'.format(fs0.name), ssh=ssh)
fs0.snapshot('snap4', recursive=True)
fs0.snapshot('snap5', recursive=True)
for compression in ['none', 'lzop', 'lz4']:
fs1.destroy(force=True)
config = [{'name': 'ssh:{:d}:{}'.format(PORT, fs0), 'key': KEY, 'dest': [fs1.name], 'compress': [compression]}]
send_config(config)
fs0_children = [child.name.replace(fs0.name, '') for child in zfs.find(fs0.name, types=['all'], ssh=ssh)[1:]]
fs1_children = [child.name.replace(fs1.name, '') for child in zfs.find(fs1.name, types=['all'])[1:]]
assert set(fs0_children) == set(fs1_children)
def test_send_incremental(self, zpools):
fs1, fs0 = zpools # here fs0 is the remote pool
ssh = fs0.ssh
fs0.destroy(force=True)
fs1.destroy(force=True)
fs0.snapshot('snap0', recursive=True)
zfs.create('{:s}/sub1'.format(fs0.name), ssh=ssh)
fs0.snapshot('snap1', recursive=True)
config = [{'name': 'ssh:{:d}:{}'.format(PORT, fs0), 'key': KEY, 'dest': [fs1.name], 'compress': None}]
send_config(config)
fs0_children = [child.name.replace(fs0.name, '') for child in zfs.find(fs0.name, types=['all'], ssh=ssh)[1:]]
fs1_children = [child.name.replace(fs1.name, '') for child in zfs.find(fs1.name, types=['all'])[1:]]
assert set(fs0_children) == set(fs1_children)
zfs.create('{:s}/sub2'.format(fs0.name), ssh=ssh)
fs0.snapshot('snap2', recursive=True)
config = [{'name': 'ssh:{:d}:{}'.format(PORT, fs0), 'key': KEY, 'dest': [fs1.name], 'compress': None}]
send_config(config)
fs0_children = [child.name.replace(fs0.name, '') for child in zfs.find(fs0.name, types=['all'], ssh=ssh)[1:]]
fs1_children = [child.name.replace(fs1.name, '') for child in zfs.find(fs1.name, types=['all'])[1:]]
assert set(fs0_children) == set(fs1_children)
zfs.create('{:s}/sub3'.format(fs0.name), ssh=ssh)
fs0.snapshot('snap3', recursive=True)
config = [{'name': 'ssh:{:d}:{}'.format(PORT, fs0), 'key': KEY, 'dest': [fs1.name], 'compress': None}]
send_config(config)
fs0_children = [child.name.replace(fs0.name, '') for child in zfs.find(fs0.name, types=['all'], ssh=ssh)[1:]]
fs1_children = [child.name.replace(fs1.name, '') for child in zfs.find(fs1.name, types=['all'])[1:]]
assert set(fs0_children) == set(fs1_children)
def create_dataset(name, name_log, ssh=None, dry_run=False):
"""Creates a dataset and logs success/fail
Parameters
----------
name : {str}
Name of the dataset to be created
name_log : {str}
Name used for logging
ssh : {SSH}, optional
Open ssh connection, by default None
dry_run : {boolean}, optional
Dry run, don't change ZFS pool
Returns
-------
int
0 if success, 1 if not
"""
logger = logging.getLogger(__name__)
try:
zfs.create(name, ssh=ssh, force=True, dry_run=dry_run)
except CalledProcessError as err:
message = err.stderr.rstrip()
if message == "filesystem successfully created, but it may only be mounted by root":
logger.info(
'Successfully created {:s}, but cannot mount as non-root...'.
format(name_log))
return 0
else:
logger.info('Error while creating {}: \'{:s}\'...'.format(
name_log, message))
return 1
except Exception as err:
logger.error('Error while creating {:s}: {}...'.format(name_log, err))
return 1
else:
logger.info('Successfully created {:s}...'.format(name_log))
return 0
def test_send_incremental(self, zpools):
fs0, fs1 = zpools
fs0.destroy(force=True)
fs1.destroy(force=True)
config = [{'name': fs0.name, 'dest': [fs1.name]}]
fs0.snapshot('snap0', recursive=True)
zfs.create('{:s}/sub1'.format(fs0.name))
fs0.snapshot('snap1', recursive=True)
send_config(config)
fs0_children = [
child.name.replace(fs0.name, '')
for child in zfs.find(fs0.name, types=['all'])[1:]
]
fs1_children = [
child.name.replace(fs1.name, '')
for child in zfs.find(fs1.name, types=['all'])[1:]
]
assert set(fs0_children) == set(fs1_children)
zfs.create('{:s}/sub2'.format(fs0.name))
fs0.snapshot('snap2', recursive=True)
send_config(config)
fs0_children = [
child.name.replace(fs0.name, '')
for child in zfs.find(fs0.name, types=['all'])[1:]
]
fs1_children = [
child.name.replace(fs1.name, '')
for child in zfs.find(fs1.name, types=['all'])[1:]
]
assert set(fs0_children) == set(fs1_children)
zfs.create('{:s}/sub3'.format(fs0.name))
fs0.snapshot('snap3', recursive=True)
send_config(config)
fs0_children = [
child.name.replace(fs0.name, '')
for child in zfs.find(fs0.name, types=['all'])[1:]
]
fs1_children = [
child.name.replace(fs1.name, '')
for child in zfs.find(fs1.name, types=['all'])[1:]
]
assert set(fs0_children) == set(fs1_children)
def test_send_full(self, zpools):
"""Checks if send_snap totally replicates a filesystem"""
fs0, fs1 = zpools
fs0.destroy(force=True)
fs1.destroy(force=True)
config = [{'name': fs0.name, 'dest': [fs1.name]}]
fs0.snapshot('snap0')
zfs.create('{:s}/sub1'.format(fs0.name))
fs0.snapshot('snap1', recursive=True)
zfs.create('{:s}/sub2'.format(fs0.name))
fs0.snapshot('snap2', recursive=True)
zfs.create('{:s}/sub3'.format(fs0.name))
fs0.snapshot('snap3', recursive=True)
fs0.snapshot('snap4', recursive=True)
fs0.snapshot('snap5', recursive=True)
zfs.create('{:s}/sub3/abc'.format(fs0.name))
fs0.snapshot('snap6', recursive=True)
zfs.create('{:s}/sub3/abc_abc'.format(fs0.name))
fs0.snapshot('snap7', recursive=True)
zfs.create('{:s}/sub3/efg'.format(fs0.name))
fs0.snapshot('snap8', recursive=True)
fs0.snapshot('snap9', recursive=True)
send_config(config)
fs0_children = [
child.name.replace(fs0.name, '')
for child in zfs.find(fs0.name, types=['all'])[1:]
]
fs1_children = [
child.name.replace(fs1.name, '')
for child in zfs.find(fs1.name, types=['all'])[1:]
]
assert set(fs0_children) == set(fs1_children)
def test_clean_recursive(self, zpools):
fs, _ = zpools
fs.destroy(force=True)
sub1 = zfs.create('{:s}/sub1'.format(fs.name))
abc = zfs.create('{:s}/sub1/abc'.format(fs.name))
abc_efg = zfs.create('{:s}/sub1/abc_efg'.format(fs.name))
sub2 = zfs.create('{:s}/sub2'.format(fs.name))
efg = zfs.create('{:s}/sub2/efg'.format(fs.name))
hij = zfs.create('{:s}/sub2/efg/hij'.format(fs.name))
klm = zfs.create('{:s}/sub2/efg/hij/klm'.format(fs.name))
sub3 = zfs.create('{:s}/sub3'.format(fs.name))
config = [{
'name': fs.name,
'frequent': 1,
'hourly': 1,
'daily': 1,
'weekly': 1,
'monthly': 1,
'yearly': 1,
'snap': True
}]
take_config(config)
config = [{
'name': fs.name,
'frequent': 1,
'hourly': 0,
'daily': 1,
'weekly': 0,
'monthly': 0,
'yearly': 0,
'clean': True
}, {
'name': '{}/sub2'.format(fs),
'frequent': 0,
'hourly': 1,
'daily': 0,
'weekly': 1,
'monthly': 0,
'yearly': 1,
'clean': True
}, {
'name': '{}/sub3'.format(fs),
'frequent': 1,
'hourly': 0,
'daily': 1,
'weekly': 0,
'monthly': 1,
'yearly': 0,
'clean': False
}, {
'name': '{}/sub1/abc'.format(fs),
'frequent': 0,
'hourly': 0,
'daily': 0,
'weekly': 1,
'monthly': 1,
'yearly': 1,
'clean': True
}, {
'name': '{}/sub2/efg/hij'.format(fs),
'frequent': 0,
'hourly': 0,
'daily': 0,
'weekly': 0,
'monthly': 0,
'yearly': 0,
'clean': True
}]
clean_config(config)
# Check parent filesystem
snapshots = {
'frequent': [],
'hourly': [],
'daily': [],
'weekly': [],
'monthly': [],
'yearly': []
}
for snap in fs.snapshots():
snap_type = snap.name.split('_')[-1]
snapshots[snap_type].append(snap)
for snap_type, snaps in snapshots.items():
assert len(snaps) == config[0][snap_type]
# Check sub1
snapshots = {
'frequent': [],
'hourly': [],
'daily': [],
'weekly': [],
'monthly': [],
'yearly': []
}
for snap in sub1.snapshots():
snap_type = snap.name.split('_')[-1]
snapshots[snap_type].append(snap)
for snap_type, snaps in snapshots.items():
assert len(snaps) == config[0][snap_type]
# Check sub1/abc
snapshots = {
'frequent': [],
'hourly': [],
'daily': [],
'weekly': [],
'monthly': [],
'yearly': []
}
for snap in abc.snapshots():
snap_type = snap.name.split('_')[-1]
snapshots[snap_type].append(snap)
for snap_type, snaps in snapshots.items():
assert len(snaps) == config[3][snap_type]
# Check sub1/abc_efg
snapshots = {
'frequent': [],
'hourly': [],
'daily': [],
'weekly': [],
'monthly': [],
'yearly': []
}
for snap in abc_efg.snapshots():
snap_type = snap.name.split('_')[-1]
snapshots[snap_type].append(snap)
for snap_type, snaps in snapshots.items():
assert len(snaps) == config[0][snap_type]
# Check sub2
snapshots = {
'frequent': [],
'hourly': [],
'daily': [],
'weekly': [],
'monthly': [],
'yearly': []
}
for snap in sub2.snapshots():
snap_type = snap.name.split('_')[-1]
snapshots[snap_type].append(snap)
for snap_type, snaps in snapshots.items():
assert len(snaps) == config[1][snap_type]
# Check sub2/efg
snapshots = {
'frequent': [],
'hourly': [],
'daily': [],
'weekly': [],
'monthly': [],
'yearly': []
}
for snap in efg.snapshots():
snap_type = snap.name.split('_')[-1]
snapshots[snap_type].append(snap)
for snap_type, snaps in snapshots.items():
assert len(snaps) == config[1][snap_type]
# Check sub2/efg/hij
snapshots = {
'frequent': [],
'hourly': [],
'daily': [],
'weekly': [],
'monthly': [],
'yearly': []
}
for snap in hij.snapshots():
snap_type = snap.name.split('_')[-1]
snapshots[snap_type].append(snap)
for snap_type, snaps in snapshots.items():
assert len(snaps) == config[4][snap_type]
# Check sub2/efg/hij/klm
snapshots = {
'frequent': [],
'hourly': [],
'daily': [],
'weekly': [],
'monthly': [],
'yearly': []
}
for snap in klm.snapshots():
snap_type = snap.name.split('_')[-1]
snapshots[snap_type].append(snap)
for snap_type, snaps in snapshots.items():
assert len(snaps) == config[4][snap_type]
# Check sub3
snapshots = {
'frequent': [],
'hourly': [],
'daily': [],
'weekly': [],
'monthly': [],
'yearly': []
}
for snap in sub3.snapshots():
snap_type = snap.name.split('_')[-1]
snapshots[snap_type].append(snap)
for snap_type, snaps in snapshots.items():
assert len(snaps) == 1
def test_send_raw(self, zpools):
"""Checks if raw_send works"""
fs0, fs1 = zpools
fs0.destroy(force=True)
fs1.destroy(force=True)
raw_send = ['yes']
config = [{'name': fs0.name, 'dest': [fs1.name], 'raw_send': raw_send}]
zfs.create('{:s}/sub1'.format(fs0.name), props={'compression': 'gzip'})
zfs.create('{:s}/sub2'.format(fs0.name), props={'compression': 'lz4'})
zfs.create('{:s}/sub3'.format(fs0.name), props={'compression': 'gzip'})
zfs.create('{:s}/sub3/abc'.format(fs0.name))
zfs.create('{:s}/sub3/abc_abc'.format(fs0.name))
zfs.create('{:s}/sub3/efg'.format(fs0.name))
fs0.snapshot('snap', recursive=True)
send_config(config)
fs0_children = set([
child.name.replace(fs0.name, '')
for child in zfs.find(fs0.name, types=['all'])[1:]
])
fs1_children = set([
child.name.replace(fs1.name, '')
for child in zfs.find(fs1.name, types=['all'])[1:]
])
assert set(fs0_children) == set(fs1_children)
def test_send_exclude(self, zpools):
"""Checks if exclude rules work"""
fs0, fs1 = zpools
fs0.destroy(force=True)
fs1.destroy(force=True)
exclude = ['*/sub1', '*/sub3/abc', '*/sub3/efg']
config = [{'name': fs0.name, 'dest': [fs1.name], 'exclude': [exclude]}]
zfs.create('{:s}/sub1'.format(fs0.name))
zfs.create('{:s}/sub2'.format(fs0.name))
zfs.create('{:s}/sub3'.format(fs0.name))
zfs.create('{:s}/sub3/abc'.format(fs0.name))
zfs.create('{:s}/sub3/abc_abc'.format(fs0.name))
zfs.create('{:s}/sub3/efg'.format(fs0.name))
fs0.snapshot('snap', recursive=True)
send_config(config)
fs0_children = set([
child.name.replace(fs0.name, '')
for child in zfs.find(fs0.name, types=['all'])[1:]
])
fs1_children = set([
child.name.replace(fs1.name, '')
for child in zfs.find(fs1.name, types=['all'])[1:]
])
# remove unwanted datasets/snapshots
for match in exclude:
fs0_children -= set(fnmatch.filter(fs0_children, match))
fs0_children -= set(fnmatch.filter(fs0_children, match + '@snap'))
assert set(fs0_children) == set(fs1_children)
def test_send_exclude(self, zpools):
"""Checks if send_snap totally replicates a filesystem"""
fs1, fs0 = zpools # here fs0 is the remote pool
ssh = fs0.ssh
fs0.destroy(force=True)
fs1.destroy(force=True)
exclude = ['*/sub1', '*/sub3/abc', '*/sub3/efg']
config = [{'name': 'ssh:{:d}:{}'.format(PORT, fs0), 'dest': [fs1.name], 'exclude': [exclude]}]
zfs.create('{:s}/sub1'.format(fs0.name), ssh=ssh)
zfs.create('{:s}/sub2'.format(fs0.name), ssh=ssh)
zfs.create('{:s}/sub3'.format(fs0.name), ssh=ssh)
zfs.create('{:s}/sub3/abc'.format(fs0.name), ssh=ssh)
zfs.create('{:s}/sub3/abc_abc'.format(fs0.name), ssh=ssh)
zfs.create('{:s}/sub3/efg'.format(fs0.name), ssh=ssh)
fs0.snapshot('snap', recursive=True)
send_config(config)
fs0_children = set([child.name.replace(fs0.name, '') for child in zfs.find(fs0.name, types=['all'], ssh=ssh)[1:]])
fs1_children = set([child.name.replace(fs1.name, '') for child in zfs.find(fs1.name, types=['all'])[1:]])
# remove unwanted datasets/snapshots
for match in exclude:
fs0_children -= set(fnmatch.filter(fs0_children, match))
fs0_children -= set(fnmatch.filter(fs0_children, match + '@snap'))
assert set(fs0_children) == set(fs1_children)