def test_backup_push_fetch(tmpdir, small_push_dir, monkeypatch, config, noop_pg_backup_statements): import wal_e.tar_partition # check that _fsync_files() is called with the right # arguments. There's a separate unit test in test_tar_hacks.py # that it actually fsyncs the right files. fsynced_files = [] monkeypatch.setattr(wal_e.tar_partition, '_fsync_files', lambda filenames: fsynced_files.extend(filenames)) config.main('backup-push', str(small_push_dir)) fetch_dir = tmpdir.join('fetch-to').ensure(dir=True) # Spin around backup-fetch LATEST for a while to paper over race # conditions whereby a backup may not be visible to backup-fetch # immediately. from boto import exception start = datetime.datetime.now() deadline = start + datetime.timedelta(seconds=15) while True: try: config.main('backup-fetch', str(fetch_dir), 'LATEST') except exception.S3ResponseError: if datetime.datetime.now() > deadline: raise else: continue else: break assert fetch_dir.join('arbitrary-file').read() == \ small_push_dir.join('arbitrary-file').read() for filename in fetch_dir.listdir(): if filename.check(link=0): assert str(filename) in fsynced_files elif filename.check(link=1): assert str(filename) not in fsynced_files
def test_backup_push_fetch(tmpdir, small_push_dir, monkeypatch, config, noop_pg_backup_statements): import wal_e.tar_partition # check that _fsync_files() is called with the right # arguments. There's a separate unit test in test_tar_hacks.py # that it actually fsyncs the right files. fsynced_files = [] monkeypatch.setattr(wal_e.tar_partition, '_fsync_files', lambda filenames: fsynced_files.extend(filenames)) config.main('backup-push', unicode(small_push_dir)) fetch_dir = tmpdir.join('fetch-to').ensure(dir=True) # Spin around backup-fetch LATEST for a while to paper over race # conditions whereby a backup may not be visible to backup-fetch # immediately. from boto import exception start = datetime.datetime.now() deadline = start + datetime.timedelta(seconds=15) while True: try: config.main('backup-fetch', unicode(fetch_dir), 'LATEST') except exception.S3ResponseError: if datetime.datetime.now() > deadline: raise else: continue else: break assert fetch_dir.join('arbitrary-file').read() == \ small_push_dir.join('arbitrary-file').read() for filename in fetch_dir.listdir(): if filename.check(link=0): assert unicode(filename) in fsynced_files elif filename.check(link=1): assert unicode(filename) not in fsynced_files
def test_backup_push_fetch(tmpdir, small_push_dir, monkeypatch, config, noop_pg_backup_statements): import wal_e.tar_partition # check that _fsync_files() is called with the right # arguments. There's a separate unit test in test_tar_hacks.py # that it actually fsyncs the right files. fsynced_files = [] monkeypatch.setattr(wal_e.tar_partition, '_fsync_files', lambda filenames: fsynced_files.extend(filenames)) config.main('backup-push', unicode(small_push_dir)) fetch_dir = tmpdir.join('fetch-to').ensure(dir=True) config.main('backup-fetch', unicode(fetch_dir), 'LATEST') assert fetch_dir.join('arbitrary-file').read() == \ small_push_dir.join('arbitrary-file').read() for filename in fetch_dir.listdir(): if filename.check(link=0): assert unicode(filename) in fsynced_files elif filename.check(link=1): assert unicode(filename) not in fsynced_files
def test_backup_push_fetch(tmpdir, small_push_dir, monkeypatch, config, noop_pg_backup_statements): import wal_e.tar_partition # check that _fsync_files() is called with the right # arguments. There's a separate unit test in test_tar_hacks.py # that it actually fsyncs the right files. fsynced_files = [] monkeypatch.setattr(wal_e.tar_partition, '_fsync_files', lambda filenames: fsynced_files.extend(filenames)) config.main('backup-push', unicode(small_push_dir)) fetch_dir = tmpdir.join('fetch-to').ensure(dir=True) # Spin around backup-fetch LATEST for a while to paper over race # conditions whereby a backup may not be visible to backup-fetch # immediately. from boto import exception start = datetime.datetime.now() deadline = start + datetime.timedelta(seconds=15) while True: try: config.main('backup-fetch', unicode(fetch_dir), 'LATEST') except exception.S3ResponseError: if datetime.datetime.now() > deadline: raise else: continue else: break assert fetch_dir.join('arbitrary-file').read() == \ small_push_dir.join('arbitrary-file').read() for filename in fetch_dir.visit(lambda f: not f.fnmatch(".wal-e"), lambda f: not f.fnmatch(".wal-e")): if filename.check(link=0): assert unicode(filename) in fsynced_files elif filename.check(link=1): assert unicode(filename) not in fsynced_files # verification should be successful config.main('backup-verify', unicode(fetch_dir)) # But not if a file is missing with pytest.raises(SystemExit): verify_dir = tmpdir.join('missing-file') fetch_dir.copy(verify_dir, True) victim = random.choice( list( verify_dir.visit(lambda f: stat.S_ISREG(f.lstat().mode), lambda f: not f.fnmatch(".wal-e")))) print "Removing victim file {0}".format(unicode(victim)) os.unlink(unicode(victim)) config.main('backup-verify', unicode(verify_dir)) # Or if a file is the wrong length with pytest.raises(SystemExit): verify_dir = tmpdir.join('resized-file') fetch_dir.copy(verify_dir, True) victim = random.choice( list( verify_dir.visit(lambda f: stat.S_ISREG(f.lstat().mode), lambda f: not f.fnmatch(".wal-e")))) print "Appending to victim file {0}".format(unicode(victim)) with open(unicode(victim), 'ab') as fileobj: fileobj.write('xyzzy') config.main('backup-verify', unicode(verify_dir)) # By default checksums aren't being checked verify_dir = tmpdir.join('checksum-mismatch') fetch_dir.copy(verify_dir, True) victim = random.choice( list( verify_dir.visit( lambda f: (stat.S_ISREG(f.lstat().mode) and f.size() > len('xyzzy')), lambda f: not f.fnmatch(".wal-e")))) print "Overwriting victim file {0} (size {1})".format( unicode(victim), victim.size()) with open(unicode(victim), 'r+b') as fileobj: # hopefully this string is not in our existing files fileobj.seek(0, os.SEEK_SET) fileobj.write('xyzzy') config.main('backup-verify', unicode(verify_dir)) # But with --verify-checksums they are with pytest.raises(SystemExit): config.main('backup-verify', '--verify-checksums', unicode(verify_dir))