def test_wal_push_fetch(pg_xlog, tmpdir, main):
contents = 'abcdefghijlmnopqrstuvwxyz\n' * 10000
seg_name = '00000001' * 3
pg_xlog.touch(seg_name, '.ready')
pg_xlog.seg(seg_name).write(contents)
main('wal-push', 'pg_xlog/' + seg_name)
# Recall file and check for equality.
download_file = tmpdir.join('TEST-DOWNLOADED')
main('wal-fetch', seg_name, unicode(download_file))
assert download_file.read() == contents
def test_wal_push_fetch(pg_xlog, tmpdir, config):
contents = 'abcdefghijlmnopqrstuvwxyz\n' * 10000
seg_name = '00000001' * 3
pg_xlog.touch(seg_name, '.ready')
pg_xlog.seg(seg_name).write(contents)
config.main('wal-push', 'pg_xlog/' + seg_name)
# Recall file and check for equality.
download_file = tmpdir.join('TEST-DOWNLOADED')
config.main('wal-fetch', seg_name, unicode(download_file))
assert download_file.read() == contents
def test_wal_push_parallel(pg_xlog, config, monkeypatch):
from wal_e.worker import upload
old_info = upload.logger.info
class GatherActions(object):
def __init__(self):
self.actions = set()
def __call__(self, *args, **kwargs):
s = kwargs['structured']
self.actions.add((s['action'], s['state']))
return old_info(*args, **kwargs)
ga = GatherActions()
monkeypatch.setattr(upload.logger, 'info', ga)
def seg_name(*parts):
return ''.join(str(p).zfill(8) for p in parts)
segments = [seg_name(1, 1, x) for x in range(1, 4)]
for s in segments:
pg_xlog.touch(s, '.ready')
# Prepare the second segment with *only* a ready file, to make
# sure parallel-push doesn't crash when pg_xlog's file is missing.
pg_xlog.seg(segments[1]).remove()
# This push has enough parallelism that it should attempt all the
# wal segments staged.
config.main('wal-push', '-p8', 'pg_xlog/' + segments[0])
# Ensure all three action types, particularly the "skip" state,
# are encountered.
assert ga.actions == set([('push-wal', 'begin'),
('push-wal', 'skip'),
('push-wal', 'complete')])
# An explicit request to upload a segment that doesn't exist must
# yield a failure.
#
# NB: Normally one would use pytest.raises, but in this case,
# e.value was *sometimes* giving an integer value, and sometimes
# the SystemExit value, whereas the builtin try/except constructs
# appear reliable by comparison.
try:
config.main('wal-push', '-p8', 'pg_xlog/' + segments[1])
except SystemExit as e:
assert e.code == 1
else:
assert False
def test_wal_push_parallel(pg_xlog, config, monkeypatch):
from wal_e.worker import upload
old_info = upload.logger.info
class GatherActions(object):
def __init__(self):
self.actions = set()
def __call__(self, *args, **kwargs):
s = kwargs['structured']
self.actions.add((s['action'], s['state']))
return old_info(*args, **kwargs)
ga = GatherActions()
monkeypatch.setattr(upload.logger, 'info', ga)
def seg_name(*parts):
return ''.join(str(p).zfill(8) for p in parts)
segments = [seg_name(1, 1, x) for x in range(1, 4)]
for s in segments:
pg_xlog.touch(s, '.ready')
# Prepare the second segment with *only* a ready file, to make
# sure parallel-push doesn't crash when pg_xlog's file is missing.
pg_xlog.seg(segments[1]).remove()
# This push has enough parallelism that it should attempt all the
# wal segments staged.
config.main('wal-push', '-p8', 'pg_xlog/' + segments[0])
# Ensure all three action types, particularly the "skip" state,
# are encountered.
assert ga.actions == set([('push-wal', 'begin'), ('push-wal', 'skip'),
('push-wal', 'complete')])
# An explicit request to upload a segment that doesn't exist must
# yield a failure.
#
# NB: Normally one would use pytest.raises, but in this case,
# e.value was *sometimes* giving an integer value, and sometimes
# the SystemExit value, whereas the builtin try/except constructs
# appear reliable by comparison.
try:
config.main('wal-push', '-p8', 'pg_xlog/' + segments[1])
except SystemExit as e:
assert e.code == 1
else:
assert False
def test_wal_push_fetch(pg_xlog, tmpdir, config):
contents = 'abcdefghijlmnopqrstuvwxyz\n' * 10000
seg_name = '00000001' * 3
pg_xlog.touch(seg_name, '.ready')
pg_xlog.seg(seg_name).write(contents)
config.main('wal-push', 'pg_xlog/' + seg_name)
# Recall file and check for equality.
download_file = tmpdir.join('TEST-DOWNLOADED')
config.main('wal-fetch', '-p0', seg_name, str(download_file))
assert download_file.read() == contents
config.main('wal-prefetch', path.dirname(str(download_file)), seg_name)
assert tmpdir.join('.wal-e', 'prefetch', seg_name).check(file=1)