def test_simple_search(pg_xlog):
"""Must find a .ready file"""
name = '1' * 8 * 3
pg_xlog.touch(name, '.ready')
segs = worker.WalSegment.from_ready_archive_status('pg_xlog')
assert next(segs).path == 'pg_xlog/' + name
with pytest.raises(StopIteration):
next(segs)
def test_simple_search(pg_xlog):
"""Must find a .ready file"""
name = "1" * 8 * 3
pg_xlog.touch(name, ".ready")
segs = worker.WalSegment.from_ready_archive_status("pg_xlog")
assert segs.next().path == "pg_xlog/" + name
with pytest.raises(StopIteration):
segs.next()
def test_wal_push_fetch(pg_xlog, tmpdir, main):
contents = 'abcdefghijlmnopqrstuvwxyz\n' * 10000
seg_name = '00000001' * 3
pg_xlog.touch(seg_name, '.ready')
pg_xlog.seg(seg_name).write(contents)
main('wal-push', 'pg_xlog/' + seg_name)
# Recall file and check for equality.
download_file = tmpdir.join('TEST-DOWNLOADED')
main('wal-fetch', seg_name, unicode(download_file))
assert download_file.read() == contents
def test_wal_push_fetch(pg_xlog, tmpdir, config):
contents = 'abcdefghijlmnopqrstuvwxyz\n' * 10000
seg_name = '00000001' * 3
pg_xlog.touch(seg_name, '.ready')
pg_xlog.seg(seg_name).write(contents)
config.main('wal-push', 'pg_xlog/' + seg_name)
# Recall file and check for equality.
download_file = tmpdir.join('TEST-DOWNLOADED')
config.main('wal-fetch', seg_name, unicode(download_file))
assert download_file.read() == contents
def test_wal_push_parallel(pg_xlog, config, monkeypatch):
from wal_e.worker import upload
old_info = upload.logger.info
class GatherActions(object):
def __init__(self):
self.actions = set()
def __call__(self, *args, **kwargs):
s = kwargs['structured']
self.actions.add((s['action'], s['state']))
return old_info(*args, **kwargs)
ga = GatherActions()
monkeypatch.setattr(upload.logger, 'info', ga)
def seg_name(*parts):
return ''.join(str(p).zfill(8) for p in parts)
segments = [seg_name(1, 1, x) for x in range(1, 4)]
for s in segments:
pg_xlog.touch(s, '.ready')
# Prepare the second segment with *only* a ready file, to make
# sure parallel-push doesn't crash when pg_xlog's file is missing.
pg_xlog.seg(segments[1]).remove()
# This push has enough parallelism that it should attempt all the
# wal segments staged.
config.main('wal-push', '-p8', 'pg_xlog/' + segments[0])
# Ensure all three action types, particularly the "skip" state,
# are encountered.
assert ga.actions == set([('push-wal', 'begin'),
('push-wal', 'skip'),
('push-wal', 'complete')])
# An explicit request to upload a segment that doesn't exist must
# yield a failure.
#
# NB: Normally one would use pytest.raises, but in this case,
# e.value was *sometimes* giving an integer value, and sometimes
# the SystemExit value, whereas the builtin try/except constructs
# appear reliable by comparison.
try:
config.main('wal-push', '-p8', 'pg_xlog/' + segments[1])
except SystemExit as e:
assert e.code == 1
else:
assert False
def test_wal_push_parallel(pg_xlog, config, monkeypatch):
from wal_e.worker import upload
old_info = upload.logger.info
class GatherActions(object):
def __init__(self):
self.actions = set()
def __call__(self, *args, **kwargs):
s = kwargs['structured']
self.actions.add((s['action'], s['state']))
return old_info(*args, **kwargs)
ga = GatherActions()
monkeypatch.setattr(upload.logger, 'info', ga)
def seg_name(*parts):
return ''.join(str(p).zfill(8) for p in parts)
segments = [seg_name(1, 1, x) for x in range(1, 4)]
for s in segments:
pg_xlog.touch(s, '.ready')
# Prepare the second segment with *only* a ready file, to make
# sure parallel-push doesn't crash when pg_xlog's file is missing.
pg_xlog.seg(segments[1]).remove()
# This push has enough parallelism that it should attempt all the
# wal segments staged.
config.main('wal-push', '-p8', 'pg_xlog/' + segments[0])
# Ensure all three action types, particularly the "skip" state,
# are encountered.
assert ga.actions == set([('push-wal', 'begin'), ('push-wal', 'skip'),
('push-wal', 'complete')])
# An explicit request to upload a segment that doesn't exist must
# yield a failure.
#
# NB: Normally one would use pytest.raises, but in this case,
# e.value was *sometimes* giving an integer value, and sometimes
# the SystemExit value, whereas the builtin try/except constructs
# appear reliable by comparison.
try:
config.main('wal-push', '-p8', 'pg_xlog/' + segments[1])
except SystemExit as e:
assert e.code == 1
else:
assert False
def test_wal_push_fetch(pg_xlog, tmpdir, config):
contents = 'abcdefghijlmnopqrstuvwxyz\n' * 10000
seg_name = '00000001' * 3
pg_xlog.touch(seg_name, '.ready')
pg_xlog.seg(seg_name).write(contents)
config.main('wal-push', 'pg_xlog/' + seg_name)
# Recall file and check for equality.
download_file = tmpdir.join('TEST-DOWNLOADED')
config.main('wal-fetch', '-p0', seg_name, str(download_file))
assert download_file.read() == contents
config.main('wal-prefetch', path.dirname(str(download_file)), seg_name)
assert tmpdir.join('.wal-e', 'prefetch', seg_name).check(file=1)
def test_multi_search(pg_xlog):
"""Test finding a few ready files.
Also throw in some random junk to make sure they are filtered out
from processing correctly.
"""
for i in range(3):
ready = str(i) * 8 * 3
pg_xlog.touch(ready, '.ready')
# Throw in a complete segment that should be ignored.
complete_segment_name = 'F' * 8 * 3
pg_xlog.touch(complete_segment_name, '.done')
# Throw in a history-file-alike that also should not be found,
# even if it's ready.
ready_history_file_name = ('F' * 8) + '.history'
pg_xlog.touch(ready_history_file_name, '.ready')
segs = worker.WalSegment.from_ready_archive_status(str(pg_xlog.pg_xlog))
for i, seg in enumerate(segs):
assert seg.name == str(i) * 8 * 3
assert i == 2
# Make sure nothing interesting happened to ignored files.
pg_xlog.assert_exists(complete_segment_name, '.done')
pg_xlog.assert_exists(ready_history_file_name, '.ready')
def test_mark_done(pg_xlog):
"""Check non-explicit segments can be .mark_done'd."""
seg = make_segment(1, explicit=False)
pg_xlog.touch(seg.name, '.ready')
seg.mark_done()