def test_dirty(nbd_server):
nbd_server.start()
with nbd.open(nbd_server.url, "r+") as b:
# backend created clean
assert not b.dirty
# write and zero dirty the backend
b.write(b"01234")
assert b.dirty
b.flush()
assert not b.dirty
b.zero(5)
assert b.dirty
b.flush()
assert not b.dirty
# readinto, seek do not affect dirty.
b.seek(0)
assert not b.dirty
with closing(util.aligned_buffer(10)) as buf:
b.readinto(buf)
assert not b.dirty
def test_dirty(nbd_server):
nbd_server.start()
with nbd.open(nbd_server.url, "r+") as b:
# backend created clean
assert not b.dirty
# write and zero dirty the backend
b.write(b"01234")
assert b.dirty
b.flush()
assert not b.dirty
b.zero(5)
assert b.dirty
b.flush()
assert not b.dirty
# readinto, seek do not affect dirty.
b.seek(0)
assert not b.dirty
with closing(util.aligned_buffer(10)) as buf:
b.readinto(buf)
assert not b.dirty
def test_open_write_only(tmpurl):
with file.open(tmpurl, "w") as f, \
closing(util.aligned_buffer(512)) as buf:
assert not f.readable()
assert f.writable()
buf.write(b"x" * 512)
f.write(buf)
with io.open(tmpurl.path, "rb") as f:
assert f.read() == b"x" * 512
def test_open_read_only(tmpurl):
with io.open(tmpurl.path, "wb") as f:
f.write(b"x" * 512)
with file.open(tmpurl, "r") as f, \
closing(util.aligned_buffer(512)) as buf:
assert f.readable()
assert not f.writable()
f.readinto(buf)
assert buf[:] == b"x" * 512
def test_readinto(tmpurl):
with io.open(tmpurl.path, "wb") as f:
f.write(b"a" * 4096)
with file.open(tmpurl, "r") as f, \
closing(util.aligned_buffer(4096)) as buf:
n = f.readinto(buf)
assert n == len(buf)
assert f.tell() == len(buf)
assert buf[:] == b"a" * 4096
def test_readinto(user_file):
with io.open(user_file.path, "wb") as f:
f.write(b"a" * 4096)
with file.open(user_file.url, "r") as f, \
closing(util.aligned_buffer(4096)) as buf:
n = f.readinto(buf)
assert n == len(buf)
assert f.tell() == len(buf)
assert buf[:] == b"a" * 4096
def test_open_write_only(user_file):
with file.open(user_file.url, "w") as f, \
closing(util.aligned_buffer(user_file.sector_size)) as buf:
assert not f.readable()
assert f.writable()
buf.write(b"x" * user_file.sector_size)
f.write(buf)
with io.open(user_file.path, "rb") as f:
assert f.read() == b"x" * user_file.sector_size
def test_open_read_only(user_file):
with io.open(user_file.path, "wb") as f:
f.write(b"x" * user_file.sector_size)
with file.open(user_file.url, "r") as f, \
closing(util.aligned_buffer(user_file.sector_size)) as buf:
assert f.readable()
assert not f.writable()
f.readinto(buf)
assert buf[:] == b"x" * user_file.sector_size
def test_open_read_larger_buffer(nbd_server):
nbd_server.start()
with nbd.open(nbd_server.url, "r") as b:
# buffer larger than remaining data
buffer_length = 10
data_length = 5
with closing(util.aligned_buffer(buffer_length)) as buf:
b.seek(b._client.export_size - data_length)
assert b.readinto(buf) == data_length
b.flush()
def test_zeroout_end(loop_device):
with util.open(loop_device, "r+") as f:
ioutil.blkzeroout(f.fileno(), BLOCKSIZE * 2, BLOCKSIZE)
with util.open(loop_device, "r") as f:
buf = util.aligned_buffer(BLOCKSIZE * 3)
with closing(buf):
f.readinto(buf)
assert buf[:-BLOCKSIZE] == b"x" * BLOCKSIZE * 2
assert buf[-BLOCKSIZE:] == b"\0" * BLOCKSIZE
def test_readinto_short_ulinged(tmpurl):
with io.open(tmpurl.path, "wb") as f:
f.write(b"a" * 4096)
with file.open(tmpurl, "r") as f, \
closing(util.aligned_buffer(8192)) as buf:
n = f.readinto(buf)
assert n == 4096
assert f.tell() == 4096
assert buf[:4096] == b"a" * 4096
assert buf[4096:] == b"\0" * 4096
def test_readinto_short_unaligned(tmpurl):
with io.open(tmpurl.path, "wb") as f:
f.write(b"a" * 42)
with file.open(tmpurl, "r") as f, \
closing(util.aligned_buffer(4096)) as buf:
n = f.readinto(buf)
assert n == 42
assert f.tell() == 42
assert buf[:42] == b"a" * 42
assert buf[42:] == b"\0" * (4096 - 42)
def test_open_read_larger_buffer(nbd_server):
nbd_server.start()
with nbd.open(nbd_server.url, "r") as b:
# buffer larger than remaining data
buffer_length = 10
data_length = 5
with closing(util.aligned_buffer(buffer_length)) as buf:
b.seek(b._client.export_size - data_length)
assert b.readinto(buf) == data_length
b.flush()
def test_zeroout_end(loop_device):
with util.open(loop_device, "r+") as f:
ioutil.blkzeroout(f.fileno(), BLOCKSIZE * 2, BLOCKSIZE)
with util.open(loop_device, "r") as f:
buf = util.aligned_buffer(BLOCKSIZE * 3)
with closing(buf):
f.readinto(buf)
assert buf[:-BLOCKSIZE] == b"x" * BLOCKSIZE * 2
assert buf[-BLOCKSIZE:] == b"\0" * BLOCKSIZE
def test_open_read_write(user_file):
with io.open(user_file.path, "wb") as f:
f.write(b"a" * user_file.sector_size)
with file.open(user_file.url, "r+") as f, \
closing(util.aligned_buffer(user_file.sector_size)) as buf:
assert f.readable()
assert f.writable()
f.readinto(buf)
buf[:] = b"b" * user_file.sector_size
f.seek(0)
f.write(buf)
with io.open(user_file.path, "rb") as f:
assert f.read() == b"b" * user_file.sector_size
def test_open_read_write(tmpurl):
with io.open(tmpurl.path, "wb") as f:
f.write(b"a" * 512)
with file.open(tmpurl, "r+") as f, \
closing(util.aligned_buffer(512)) as buf:
assert f.readable()
assert f.writable()
f.readinto(buf)
buf[:] = b"b" * 512
f.seek(0)
f.write(buf)
with io.open(tmpurl.path, "rb") as f:
assert f.read() == b"b" * 512
def test_write_aligned_at_end(user_file):
with io.open(user_file.path, "wb") as f:
f.write(b"a" * 8192)
with file.open(user_file.url, "r+") as f, \
closing(util.aligned_buffer(8192)) as buf:
buf.write(b"b" * 8192)
f.seek(4096)
n = f.write(buf)
assert n == len(buf)
assert f.tell() == 4096 + len(buf)
with io.open(user_file.path, "rb") as f:
assert f.read(4096) == b"a" * 4096
assert f.read() == b"b" * 8192
def test_write_aligned_at_end(tmpurl):
with io.open(tmpurl.path, "wb") as f:
f.write(b"a" * 8192)
with file.open(tmpurl, "r+") as f, \
closing(util.aligned_buffer(8192)) as buf:
buf.write(b"b" * 8192)
f.seek(4096)
n = f.write(buf)
assert n == len(buf)
assert f.tell() == 4096 + len(buf)
with io.open(tmpurl.path, "rb") as f:
assert f.read(4096) == b"a" * 4096
assert f.read() == b"b" * 8192
def test_zero_middle(nbd_server, sparse):
nbd_server.start()
with nbd.open(nbd_server.url, "r+", sparse=sparse) as b:
# nbd backend is always sparse.
assert b.sparse
b.write(b"xxxxxxxxxxxx")
b.seek(4)
assert b.zero(4) == 4
with closing(util.aligned_buffer(12)) as buf:
b.seek(0)
assert b.readinto(buf) == 12
assert buf[:] == b"xxxx\x00\x00\x00\x00xxxx"
def test_fallocate_zero_end(tmpdir, mode):
path = str(tmpdir.join("file"))
with open(path, "wb") as f:
f.write(b"x" * BLOCKSIZE * 3)
buf = util.aligned_buffer(BLOCKSIZE * 3)
with closing(buf), util.open(path, "r+") as f:
try_fallocate(f.fileno(), mode, BLOCKSIZE * 2, BLOCKSIZE)
n = f.readinto(buf)
assert n == BLOCKSIZE * 3
assert buf[:-BLOCKSIZE] == b"x" * BLOCKSIZE * 2
assert buf[-BLOCKSIZE:] == b"\0" * BLOCKSIZE
assert f.readinto(buf) == 0
def test_zero_middle(nbd_server, sparse):
nbd_server.start()
with nbd.open(nbd_server.url, "r+", sparse=sparse) as b:
# nbd backend is always sparse.
assert b.sparse
b.write(b"xxxxxxxxxxxx")
b.seek(4)
assert b.zero(4) == 4
with closing(util.aligned_buffer(12)) as buf:
b.seek(0)
assert b.readinto(buf) == 12
assert buf[:] == b"xxxx\x00\x00\x00\x00xxxx"
def loop_device(tmpdir):
backing_file = str(tmpdir.join("backing_file"))
with util.open(backing_file, "w") as f:
buf = util.aligned_buffer(BLOCKSIZE * 3)
with closing(buf):
buf[:] = b"x" * BLOCKSIZE * 3
f.write(buf)
out = subprocess.check_output(
["losetup", "--find", backing_file, "--show"])
try:
loop = out.strip().decode("ascii")
yield loop
finally:
subprocess.check_call(["losetup", "--detach", loop])
def test_readinto_short_unaligned(user_file):
size = 42
buf_size = user_file.sector_size
with io.open(user_file.path, "wb") as f:
f.write(b"a" * size)
with file.open(user_file.url, "r") as f, \
closing(util.aligned_buffer(buf_size)) as buf:
n = f.readinto(buf)
assert n == size
assert f.tell() == size
assert buf[:size] == b"a" * size
assert buf[size:] == b"\0" * (buf_size - size)
def test_fallocate_zero_end(tmpdir, mode):
path = str(tmpdir.join("file"))
with open(path, "wb") as f:
f.write(b"x" * BLOCKSIZE * 3)
buf = util.aligned_buffer(BLOCKSIZE * 3)
with closing(buf), util.open(path, "r+") as f:
try_fallocate(f.fileno(), mode, BLOCKSIZE * 2, BLOCKSIZE)
n = f.readinto(buf)
assert n == BLOCKSIZE * 3
assert buf[:-BLOCKSIZE] == b"x" * BLOCKSIZE * 2
assert buf[-BLOCKSIZE:] == b"\0" * BLOCKSIZE
assert f.readinto(buf) == 0
def loop_device(tmpdir):
backing_file = str(tmpdir.join("backing_file"))
with util.open(backing_file, "w") as f:
buf = util.aligned_buffer(BLOCKSIZE * 3)
with closing(buf):
buf[:] = b"x" * BLOCKSIZE * 3
f.write(buf)
out = subprocess.check_output(
["losetup", "--find", backing_file, "--show"])
try:
loop = out.strip().decode("ascii")
yield loop
finally:
subprocess.check_call(["losetup", "--detach", loop])
def test_close(nbd_server):
nbd_server.start()
with nbd.open(nbd_server.url, "r+") as b:
pass
# Closing twice does not do anything.
b.close()
# But other operations should fail now with:
# socket.error: Bad file descriptor
with pytest.raises(IOError):
b.write("more")
with pytest.raises(IOError):
with closing(util.aligned_buffer(100)) as buf:
b.readinto(buf)
def test_fallocate_punch_hole_after_end(tmpdir):
path = str(tmpdir.join("file"))
with open(path, "wb") as f:
f.write(b"x" * BLOCKSIZE * 3)
buf = util.aligned_buffer(BLOCKSIZE * 3)
with closing(buf), util.open(path, "r+") as f:
# This does not change file contents or size.
mode = ioutil.FALLOC_FL_PUNCH_HOLE | ioutil.FALLOC_FL_KEEP_SIZE
try_fallocate(f.fileno(), mode, BLOCKSIZE * 3, BLOCKSIZE)
n = f.readinto(buf)
assert n == BLOCKSIZE * 3
assert buf[:] == b"x" * BLOCKSIZE * 3
assert f.readinto(buf) == 0
def test_fallocate_punch_hole_after_end(tmpdir):
path = str(tmpdir.join("file"))
with open(path, "wb") as f:
f.write(b"x" * BLOCKSIZE * 3)
buf = util.aligned_buffer(BLOCKSIZE * 3)
with closing(buf), util.open(path, "r+") as f:
# This does not change file contents or size.
mode = ioutil.FALLOC_FL_PUNCH_HOLE | ioutil.FALLOC_FL_KEEP_SIZE
try_fallocate(f.fileno(), mode, BLOCKSIZE * 3, BLOCKSIZE)
n = f.readinto(buf)
assert n == BLOCKSIZE * 3
assert buf[:] == b"x" * BLOCKSIZE * 3
assert f.readinto(buf) == 0
def test_close(nbd_server):
nbd_server.start()
with nbd.open(nbd_server.url, "r+") as b:
pass
# Closing twice does not do anything.
b.close()
# But other operations should fail now with:
# socket.error: Bad file descriptor
with pytest.raises(IOError):
b.write("more")
with pytest.raises(IOError):
with closing(util.aligned_buffer(100)) as buf:
b.readinto(buf)
def test_fallocate_zero_after_end(tmpdir):
path = str(tmpdir.join("file"))
with open(path, "wb") as f:
f.write(b"x" * BLOCKSIZE * 3)
buf = util.aligned_buffer(BLOCKSIZE * 4)
with closing(buf), util.open(path, "r+") as f:
# Will allocate more space that will return zeros when read.
mode = ioutil.FALLOC_FL_ZERO_RANGE
try_fallocate(f.fileno(), mode, BLOCKSIZE * 3, BLOCKSIZE)
n = f.readinto(buf)
assert n == BLOCKSIZE * 4
assert buf[:-BLOCKSIZE] == b"x" * BLOCKSIZE * 3
assert buf[-BLOCKSIZE:] == b"\0" * BLOCKSIZE
assert f.readinto(buf) == 0
def test_fallocate_zero_after_end(tmpdir):
path = str(tmpdir.join("file"))
with open(path, "wb") as f:
f.write(b"x" * BLOCKSIZE * 3)
buf = util.aligned_buffer(BLOCKSIZE * 4)
with closing(buf), util.open(path, "r+") as f:
# Will allocate more space that will return zeros when read.
mode = ioutil.FALLOC_FL_ZERO_RANGE
try_fallocate(f.fileno(), mode, BLOCKSIZE * 3, BLOCKSIZE)
n = f.readinto(buf)
assert n == BLOCKSIZE * 4
assert buf[:-BLOCKSIZE] == b"x" * BLOCKSIZE * 3
assert buf[-BLOCKSIZE:] == b"\0" * BLOCKSIZE
assert f.readinto(buf) == 0
def test_write_unaligned_buffer_fast_path(tmpurl):
with io.open(tmpurl.path, "wb") as f:
f.write(b"x" * 4096)
# Perform short fast write of 6 blocks.
buf = util.aligned_buffer(3073)
buf.write(b"y" * 3073)
with closing(buf):
with file.open(tmpurl, "r+") as f:
f.seek(512)
n = f.write(buf)
assert n == 3072
assert f.tell() == 3584
with io.open(tmpurl.path, "rb") as f:
assert f.read(512) == b"x" * 512
assert f.read(3072) == b"y" * 3072
assert f.read() == b"x" * 512
def test_open_read_write(nbd_server):
nbd_server.start()
with nbd.open(nbd_server.url, "r+") as b:
assert b.readable()
assert b.writable()
data = b"data"
b.write(data)
assert b.tell() == len(data)
b.zero(4)
size = len(data) + 4
assert b.tell() == size
with closing(util.aligned_buffer(size)) as buf:
b.seek(0)
assert b.readinto(buf) == size
assert buf[:] == data + b"\0" * 4
b.flush()
def test_open_read_write(nbd_server):
nbd_server.start()
with nbd.open(nbd_server.url, "r+") as b:
assert b.readable()
assert b.writable()
data = b"data"
b.write(data)
assert b.tell() == len(data)
b.zero(4)
size = len(data) + 4
assert b.tell() == size
with closing(util.aligned_buffer(size)) as buf:
b.seek(0)
assert b.readinto(buf) == size
assert buf[:] == data + b"\0" * 4
b.flush()
def test_open_writeonly(nbd_server):
nbd_server.start()
with nbd.open(nbd_server.url, "w") as b:
assert not b.readable()
assert b.writable()
data = b"data"
b.write(data)
assert b.tell() == len(data)
with pytest.raises(IOError):
with closing(util.aligned_buffer(100)) as buf:
b.readinto(buf)
dst = io.BytesIO()
with pytest.raises(IOError):
b.write_to(dst, 100)
b.flush()
def test_open_writeonly(nbd_server):
nbd_server.start()
with nbd.open(nbd_server.url, "w") as b:
assert not b.readable()
assert b.writable()
data = b"data"
b.write(data)
assert b.tell() == len(data)
with pytest.raises(IOError):
with closing(util.aligned_buffer(100)) as buf:
b.readinto(buf)
dst = io.BytesIO()
with pytest.raises(IOError):
b.write_to(dst, 100)
b.flush()
def test_open_readonly(nbd_server):
nbd_server.read_only = True
nbd_server.start()
with nbd.open(nbd_server.url, "r") as b:
assert b.readable()
assert not b.writable()
with pytest.raises(IOError):
b.write(b"data")
assert b.tell() == 0
with pytest.raises(IOError):
b.zero(4)
assert b.tell() == 0
with closing(util.aligned_buffer(100)) as buf:
buf.write(b"x" * 100)
assert b.readinto(buf) == len(buf)
assert buf[:] == b"\0" * len(buf)
b.flush()
def test_open_readonly(nbd_server):
nbd_server.read_only = True
nbd_server.start()
with nbd.open(nbd_server.url, "r") as b:
assert b.readable()
assert not b.writable()
with pytest.raises(IOError):
b.write(b"data")
assert b.tell() == 0
with pytest.raises(IOError):
b.zero(4)
assert b.tell() == 0
with closing(util.aligned_buffer(100)) as buf:
buf.write(b"x" * 100)
assert b.readinto(buf) == len(buf)
assert buf[:] == b"\0" * len(buf)
b.flush()
def test_dirty(user_file):
# backend created clean
with file.open(user_file.url, "r+", sparse=True) as f:
assert not f.dirty
buf = util.aligned_buffer(4096)
with closing(buf):
# write ans zero dirty the backend
buf.write(b"x" * 4096)
f.write(buf)
assert f.dirty
f.flush()
assert not f.dirty
f.zero(4096)
assert f.dirty
f.flush()
assert not f.dirty
# readinto, seek do not affect dirty.
f.seek(0)
assert not f.dirty
f.readinto(buf)
assert not f.dirty
def test_dirty(tmpurl):
# backend created clean
m = file.open(tmpurl, "r+")
assert not m.dirty
buf = util.aligned_buffer(4096)
with closing(buf):
# write ans zero dirty the backend
buf.write(b"x" * 4096)
m.write(buf)
assert m.dirty
m.flush()
assert not m.dirty
m.zero(4096)
assert m.dirty
m.flush()
assert not m.dirty
# readinto, seek do not affect dirty.
m.seek(0)
assert not m.dirty
m.readinto(buf)
assert not m.dirty
def test_write_unaligned_buffer_fast_path(user_file):
size = user_file.sector_size * 4
start = user_file.sector_size
length = user_file.sector_size * 2
with io.open(user_file.path, "wb") as f:
f.write(b"x" * size)
# Create buffer of 2 blcoks + 1 byte.
buf = util.aligned_buffer(length + 1)
buf.write(b"y" * len(buf))
# Perform short fast write of 2 blocks.
with closing(buf):
with file.open(user_file.url, "r+") as f:
f.seek(start)
n = f.write(buf)
assert n == length
assert f.tell() == start + length
with io.open(user_file.path, "rb") as f:
assert f.read(start) == b"x" * start
assert f.read(length) == b"y" * length
assert f.read() == b"x" * user_file.sector_size
def aligned_buffer():
buf = util.aligned_buffer(BLOCKSIZE)
with closing(buf):
yield buf
def aligned_buffer():
buf = util.aligned_buffer(BLOCKSIZE)
with closing(buf):
yield buf
