def test_file_list(tmpdir):
f = FileSystemRepository(str(tmpdir))
d = tmpdir.mkdir('dir')
for i in range(100):
d.mkdir('d{0}'.format(i))
assert (frozenset(f.list(['dir'])) ==
frozenset('d{0}'.format(i) for i in range(100)))
with raises(RepositoryKeyError):
f.list(['not-exist'])
def test_file_list(tmpdir):
f = FileSystemRepository(str(tmpdir))
d = tmpdir.mkdir('dir')
for i in range(100):
d.mkdir('d{0}'.format(i))
assert (frozenset(f.list(['dir'])) == frozenset('d{0}'.format(i)
for i in range(100)))
with raises(RepositoryKeyError):
f.list(['not-exist'])
def test_read_write_same_file(tmpdir):
repo = FileSystemRepository(str(tmpdir))
repo.write(['key'], itertools.repeat(b'first revision\n', 1024))
first_iterator = iter(repo.read(['key']))
second_iterator = iter(repo.read(['key']))
assert first_iterator is not second_iterator
first_stop = False
second_stop = False
first_list = []
second_list = []
while not (first_stop or second_stop):
if not first_stop:
try:
first_chunk = next(first_iterator)
except StopIteration:
first_stop = True
else:
if not first_list:
repo.write(['key'], [b'second ', b'revision'])
third_iterator = repo.read(['key'])
first_list.append(first_chunk)
if not second_stop:
try:
second_chunk = next(second_iterator)
except StopIteration:
second_stop = True
else:
second_list.append(second_chunk)
assert (b''.join(first_list) == b''.join(second_list) == b''.join(
itertools.repeat(b'first revision\n', 1024)))
assert b''.join(third_iterator) == b'second revision'
def test_read_write_same_file(tmpdir):
repo = FileSystemRepository(str(tmpdir))
repo.write(['key'], itertools.repeat(b'first revision\n', 1024))
first_iterator = iter(repo.read(['key']))
second_iterator = iter(repo.read(['key']))
assert first_iterator is not second_iterator
first_stop = False
second_stop = False
first_list = []
second_list = []
while not (first_stop or second_stop):
if not first_stop:
try:
first_chunk = next(first_iterator)
except StopIteration:
first_stop = True
else:
if not first_list:
repo.write(['key'], [b'second ', b'revision'])
third_iterator = repo.read(['key'])
first_list.append(first_chunk)
if not second_stop:
try:
second_chunk = next(second_iterator)
except StopIteration:
second_stop = True
else:
second_list.append(second_chunk)
assert (b''.join(first_list) ==
b''.join(second_list) ==
b''.join(itertools.repeat(b'first revision\n', 1024)))
assert b''.join(third_iterator) == b'second revision'
def test_file_write(tmpdir):
f = FileSystemRepository(str(tmpdir))
f.write(['key'], [b'file ', b'content'])
assert tmpdir.join('key').read() == 'file content'
f.write(['dir', 'dir2', 'key'], [b'deep ', b'file ', b'content'])
assert tmpdir.join('dir', 'dir2', 'key').read() == 'deep file content'
with raises(RepositoryKeyError):
f.write([], [b'file ', b'content'])
def test_dirty_buffer(tmpdir):
if IRON_PYTHON:
repo = MemoryRepository()
else:
repo = FileSystemRepository(str(tmpdir))
dirty = DirtyBuffer(repo, threading.RLock())
key = ['key']
dir_key = []
with raises(RepositoryKeyError):
dirty.read(key)
assert not dirty.exists(key)
assert frozenset(dirty.list(dir_key)) == frozenset([])
with raises(RepositoryKeyError):
repo.read(key)
assert not repo.exists(key)
assert frozenset(repo.list(dir_key)) == frozenset([])
dirty.write(key, [b'dirty ', b'value'])
assert b''.join(dirty.read(key)) == b'dirty value'
assert dirty.exists(key)
assert frozenset(dirty.list(dir_key)) == frozenset(key)
with raises(RepositoryKeyError):
repo.read(key)
assert not repo.exists(key)
assert frozenset(repo.list(dir_key)) == frozenset([])
assert dirty.dictionary
dirty.flush()
assert not dirty.dictionary
assert b''.join(repo.read(key)) == b'dirty value'
assert repo.exists(key)
assert frozenset(repo.list(dir_key)) == frozenset(key)
def get_stage():
try:
return app.config['STAGE']
except KeyError:
session_id = app.config['SESSION_ID']
if request.environ['wsgi.multiprocess']:
# Stage doesn't offer safe synchronization between multiprocess.
# Unique session identifiers are actually needed to distinguish
# different "installations" which technically means "processes,"
# hence we append pid to the session identifier configured by
# the user to make them unique.
# Note that it probably causes N times more disk usage
# where N = the number of processes. So we should discourage
# using web servers of prefork/worker model in the docs.
session_id = '{0}.{1}'.format(session_id, os.getpid())
session = Session(session_id)
url = urlparse.urlparse(app.config['REPOSITORY'])
if url.scheme == 'file':
repository = FileSystemRepository(
url.path, atomic=request.environ['wsgi.multithread'])
else:
repository = from_url(app.config['REPOSITORY'])
stage = Stage(session, repository)
app.config['STAGE'] = stage
return stage
def test_file_from_to_url(tmpdir):
url = 'file://' + str(tmpdir)
parsed = urlparse.urlparse(url)
fs = FileSystemRepository.from_url(parsed)
assert isinstance(fs, FileSystemRepository)
assert fs.path == str(tmpdir)
assert fs.to_url('file') == url
assert fs.to_url('fs') == 'fs://' + str(tmpdir)
def test_file_from_to_url__posix(tmpdir):
url = 'file://' + str(tmpdir)
parsed = urlparse.urlparse(url)
fs = FileSystemRepository.from_url(parsed)
assert isinstance(fs, FileSystemRepository)
assert fs.path == str(tmpdir)
assert fs.to_url('file') == url
assert fs.to_url('fs') == 'fs://' + str(tmpdir)
def test_file_read(tmpdir):
f = FileSystemRepository(str(tmpdir))
with raises(RepositoryKeyError):
f.read(['key'])
tmpdir.join('key').write('file content')
assert b''.join(f.read(['key'])) == b'file content'
with raises(RepositoryKeyError):
f.read(['dir', 'dir2', 'key'])
tmpdir.mkdir('dir').mkdir('dir2').join('key').write('file content')
assert b''.join(f.read(['dir', 'dir2', 'key'])) == b'file content'
def test_file_from_to_url__windows(tmpdir):
url_tail = '/'.join(str(tmpdir).split(tmpdir.sep))
url = 'file:///' + url_tail
parsed = urlparse.urlparse(url)
fs = FileSystemRepository.from_url(parsed)
assert isinstance(fs, FileSystemRepository)
assert fs.path == str(tmpdir)
assert fs.to_url('file') == url
assert fs.to_url('fs') == 'fs:///' + url_tail
def test_file_from_to_url__windows(tmpdir):
url_tail = '/'.join(str(tmpdir).split(tmpdir.sep))
url = 'file:///' + url_tail
parsed = urlparse.urlparse(url)
fs = FileSystemRepository.from_url(parsed)
assert isinstance(fs, FileSystemRepository)
assert fs.path == str(tmpdir)
assert fs.to_url('file') == url
assert fs.to_url('fs') == 'fs:///' + url_tail
def test_file_write(tmpdir):
f = FileSystemRepository(str(tmpdir))
f.write(['key'], [b'file ', b'content'])
assert tmpdir.join('key').read() == 'file content'
f.write(['dir', 'dir2', 'key'], [b'deep ', b'file ', b'content'])
assert tmpdir.join('dir', 'dir2', 'key').read() == 'deep file content'
with raises(RepositoryKeyError):
f.write([], [b'file ', b'content'])
def fx_stages(tmpdir):
if IRON_PYTHON:
repo = MemoryRepository()
else:
repo = FileSystemRepository(str(tmpdir))
session_a = Session(identifier='a')
session_b = Session(identifier='b')
stage_a = Stage(session_a, repo)
stage_b = Stage(session_b, repo)
return stage_a, stage_b
def test_file_exists(tmpdir):
f = FileSystemRepository(str(tmpdir))
tmpdir.mkdir('dir').join('file').write('content')
tmpdir.join('file').write('content')
assert f.exists(['dir'])
assert f.exists(['dir', 'file'])
assert f.exists(['file'])
assert not f.exists(['dir', 'file-not-exist'])
assert not f.exists(['dir-not-exist'])
def test_file_read(tmpdir):
f = FileSystemRepository(str(tmpdir))
with raises(RepositoryKeyError):
f.read(['key'])
tmpdir.join('key').write('file content')
assert b''.join(f.read(['key'])) == b'file content'
with raises(RepositoryKeyError):
f.read(['dir', 'dir2', 'key'])
tmpdir.mkdir('dir').mkdir('dir2').join('key').write('file content')
assert b''.join(f.read(['dir', 'dir2', 'key'])) == b'file content'
def test_atomicity(tmpdir):
repo = FileSystemRepository(str(tmpdir), atomic=True)
repo.write(['key'], [b'first ', b'revision'])
def gen():
assert b''.join(repo.read(['key'])) == b'first revision'
yield b'second '
assert b''.join(repo.read(['key'])) == b'first revision'
yield b'revision'
assert b''.join(repo.read(['key'])) == b'first revision'
repo.write(['key'], gen())
assert b''.join(repo.read(['key'])) == b'second revision'
def test_file_exists(tmpdir):
f = FileSystemRepository(str(tmpdir))
tmpdir.mkdir('dir').join('file').write('content')
tmpdir.join('file').write('content')
assert f.exists(['dir'])
assert f.exists(['dir', 'file'])
assert f.exists(['file'])
assert not f.exists(['dir', 'file-not-exist'])
assert not f.exists(['dir-not-exist'])
def test_atomicity(tmpdir):
repo = FileSystemRepository(str(tmpdir), atomic=True)
repo.write(['key'], [b'first ', b'revision'])
def gen():
assert b''.join(repo.read(['key'])) == b'first revision'
yield b'second '
assert b''.join(repo.read(['key'])) == b'first revision'
yield b'revision'
assert b''.join(repo.read(['key'])) == b'first revision'
repo.write(['key'], gen())
assert b''.join(repo.read(['key'])) == b'second revision'
def fx_test_stage(tmpdir):
session = Session()
repo = FileSystemRepository(str(tmpdir))
stage = Stage(session, repo)
app.config['STAGE'] = stage
return stage
def repositories():
yield FileSystemRepository(tempfile.mkdtemp())
yield DirtyBuffer(FileSystemRepository(tempfile.mkdtemp()),
threading.RLock())
def test_not_dir(tmpdir):
path = tmpdir.join('not-dir.txt')
path.write('')
with raises(NotADirectoryError):
FileSystemRepository(str(path))
def test_file_not_found(tmpdir):
path = tmpdir.join('not-exist')
with raises(FileNotFoundError):
FileSystemRepository(str(path), mkdir=False)
FileSystemRepository(str(path))
assert os.path.isdir(str(path))
