def test_backup_push_fetch(tmpdir, small_push_dir, monkeypatch, config,
noop_pg_backup_statements):
import wal_e.tar_partition
# check that _fsync_files() is called with the right
# arguments. There's a separate unit test in test_tar_hacks.py
# that it actually fsyncs the right files.
fsynced_files = []
monkeypatch.setattr(wal_e.tar_partition, '_fsync_files',
lambda filenames: fsynced_files.extend(filenames))
config.main('backup-push', str(small_push_dir))
fetch_dir = tmpdir.join('fetch-to').ensure(dir=True)
# Spin around backup-fetch LATEST for a while to paper over race
# conditions whereby a backup may not be visible to backup-fetch
# immediately.
from boto import exception
start = datetime.datetime.now()
deadline = start + datetime.timedelta(seconds=15)
while True:
try:
config.main('backup-fetch', str(fetch_dir), 'LATEST')
except exception.S3ResponseError:
if datetime.datetime.now() > deadline:
raise
else:
continue
else:
break
assert fetch_dir.join('arbitrary-file').read() == \
small_push_dir.join('arbitrary-file').read()
for filename in fetch_dir.listdir():
if filename.check(link=0):
assert str(filename) in fsynced_files
elif filename.check(link=1):
assert str(filename) not in fsynced_files
def test_backup_push_fetch(tmpdir, small_push_dir, monkeypatch, config,
noop_pg_backup_statements):
import wal_e.tar_partition
# check that _fsync_files() is called with the right
# arguments. There's a separate unit test in test_tar_hacks.py
# that it actually fsyncs the right files.
fsynced_files = []
monkeypatch.setattr(wal_e.tar_partition, '_fsync_files',
lambda filenames: fsynced_files.extend(filenames))
config.main('backup-push', unicode(small_push_dir))
fetch_dir = tmpdir.join('fetch-to').ensure(dir=True)
# Spin around backup-fetch LATEST for a while to paper over race
# conditions whereby a backup may not be visible to backup-fetch
# immediately.
from boto import exception
start = datetime.datetime.now()
deadline = start + datetime.timedelta(seconds=15)
while True:
try:
config.main('backup-fetch', unicode(fetch_dir), 'LATEST')
except exception.S3ResponseError:
if datetime.datetime.now() > deadline:
raise
else:
continue
else:
break
assert fetch_dir.join('arbitrary-file').read() == \
small_push_dir.join('arbitrary-file').read()
for filename in fetch_dir.listdir():
if filename.check(link=0):
assert unicode(filename) in fsynced_files
elif filename.check(link=1):
assert unicode(filename) not in fsynced_files
def test_backup_push_fetch(tmpdir, small_push_dir, monkeypatch, config,
noop_pg_backup_statements):
import wal_e.tar_partition
# check that _fsync_files() is called with the right
# arguments. There's a separate unit test in test_tar_hacks.py
# that it actually fsyncs the right files.
fsynced_files = []
monkeypatch.setattr(wal_e.tar_partition, '_fsync_files',
lambda filenames: fsynced_files.extend(filenames))
config.main('backup-push', unicode(small_push_dir))
fetch_dir = tmpdir.join('fetch-to').ensure(dir=True)
config.main('backup-fetch', unicode(fetch_dir), 'LATEST')
assert fetch_dir.join('arbitrary-file').read() == \
small_push_dir.join('arbitrary-file').read()
for filename in fetch_dir.listdir():
if filename.check(link=0):
assert unicode(filename) in fsynced_files
elif filename.check(link=1):
assert unicode(filename) not in fsynced_files
def test_backup_push_fetch(tmpdir, small_push_dir, monkeypatch, config,
noop_pg_backup_statements):
import wal_e.tar_partition
# check that _fsync_files() is called with the right
# arguments. There's a separate unit test in test_tar_hacks.py
# that it actually fsyncs the right files.
fsynced_files = []
monkeypatch.setattr(wal_e.tar_partition, '_fsync_files',
lambda filenames: fsynced_files.extend(filenames))
config.main('backup-push', unicode(small_push_dir))
fetch_dir = tmpdir.join('fetch-to').ensure(dir=True)
# Spin around backup-fetch LATEST for a while to paper over race
# conditions whereby a backup may not be visible to backup-fetch
# immediately.
from boto import exception
start = datetime.datetime.now()
deadline = start + datetime.timedelta(seconds=15)
while True:
try:
config.main('backup-fetch', unicode(fetch_dir), 'LATEST')
except exception.S3ResponseError:
if datetime.datetime.now() > deadline:
raise
else:
continue
else:
break
assert fetch_dir.join('arbitrary-file').read() == \
small_push_dir.join('arbitrary-file').read()
for filename in fetch_dir.visit(lambda f: not f.fnmatch(".wal-e"),
lambda f: not f.fnmatch(".wal-e")):
if filename.check(link=0):
assert unicode(filename) in fsynced_files
elif filename.check(link=1):
assert unicode(filename) not in fsynced_files
# verification should be successful
config.main('backup-verify', unicode(fetch_dir))
# But not if a file is missing
with pytest.raises(SystemExit):
verify_dir = tmpdir.join('missing-file')
fetch_dir.copy(verify_dir, True)
victim = random.choice(
list(
verify_dir.visit(lambda f: stat.S_ISREG(f.lstat().mode),
lambda f: not f.fnmatch(".wal-e"))))
print "Removing victim file {0}".format(unicode(victim))
os.unlink(unicode(victim))
config.main('backup-verify', unicode(verify_dir))
# Or if a file is the wrong length
with pytest.raises(SystemExit):
verify_dir = tmpdir.join('resized-file')
fetch_dir.copy(verify_dir, True)
victim = random.choice(
list(
verify_dir.visit(lambda f: stat.S_ISREG(f.lstat().mode),
lambda f: not f.fnmatch(".wal-e"))))
print "Appending to victim file {0}".format(unicode(victim))
with open(unicode(victim), 'ab') as fileobj:
fileobj.write('xyzzy')
config.main('backup-verify', unicode(verify_dir))
# By default checksums aren't being checked
verify_dir = tmpdir.join('checksum-mismatch')
fetch_dir.copy(verify_dir, True)
victim = random.choice(
list(
verify_dir.visit(
lambda f:
(stat.S_ISREG(f.lstat().mode) and f.size() > len('xyzzy')),
lambda f: not f.fnmatch(".wal-e"))))
print "Overwriting victim file {0} (size {1})".format(
unicode(victim), victim.size())
with open(unicode(victim), 'r+b') as fileobj:
# hopefully this string is not in our existing files
fileobj.seek(0, os.SEEK_SET)
fileobj.write('xyzzy')
config.main('backup-verify', unicode(verify_dir))
# But with --verify-checksums they are
with pytest.raises(SystemExit):
config.main('backup-verify', '--verify-checksums', unicode(verify_dir))