def test_open_write_only_truncate(tmpurl): with io.open(tmpurl.path, "wb") as f: f.write(b"x" * 512) with file.open(tmpurl, "w") as f: pass with io.open(tmpurl.path, "rb") as f: assert f.read() == b""
def test_open_write_only_truncate(user_file): with io.open(user_file.path, "wb") as f: f.write(b"x" * user_file.sector_size) with file.open(user_file.url, "w") as f: pass with io.open(user_file.path, "rb") as f: assert f.read() == b""
def receive_unbuffered(tmpurl, chunks, size, bufsize): src = testutil.UnbufferedStream(chunks) with file.open(tmpurl, "r+") as dst: op = ops.Receive(dst, src, size, buffersize=bufsize) op.run() with io.open(tmpurl.path, "rb") as f: return f.read()
def test_open_write_only(user_file): with file.open(user_file.url, "w") as f, \ closing(util.aligned_buffer(user_file.sector_size)) as buf: assert not f.readable() assert f.writable() buf.write(b"x" * user_file.sector_size) f.write(buf) with io.open(user_file.path, "rb") as f: assert f.read() == b"x" * user_file.sector_size
def test_readinto(tmpurl): with io.open(tmpurl.path, "wb") as f: f.write(b"a" * 4096) with file.open(tmpurl, "r") as f, \ closing(util.aligned_buffer(4096)) as buf: n = f.readinto(buf) assert n == len(buf) assert f.tell() == len(buf) assert buf[:] == b"a" * 4096
def test_readinto(user_file): with io.open(user_file.path, "wb") as f: f.write(b"a" * 4096) with file.open(user_file.url, "r") as f, \ closing(util.aligned_buffer(4096)) as buf: n = f.readinto(buf) assert n == len(buf) assert f.tell() == len(buf) assert buf[:] == b"a" * 4096
def test_open_write_only(tmpurl): with file.open(tmpurl, "w") as f, \ closing(util.aligned_buffer(512)) as buf: assert not f.readable() assert f.writable() buf.write(b"x" * 512) f.write(buf) with io.open(tmpurl.path, "rb") as f: assert f.read() == b"x" * 512
def test_block_size_allocated(user_file, size): with io.open(user_file.path, "wb") as f: f.write(b"x" * size) with file.open(user_file.url, "r") as f: assert f.block_size == user_file.sector_size with io.open(user_file.path, "rb") as f: assert f.read() == b"x" * size
def test_open_read_only(user_file): with io.open(user_file.path, "wb") as f: f.write(b"x" * user_file.sector_size) with file.open(user_file.url, "r") as f, \ closing(util.aligned_buffer(user_file.sector_size)) as buf: assert f.readable() assert not f.writable() f.readinto(buf) assert buf[:] == b"x" * user_file.sector_size
def test_receive_unbuffered_stream_partial_content(user_file): chunks = [b"a" * 8192, b"b" * 42, b"c" * (8192 - 42)] src = util.UnbufferedStream(chunks) size = sum(len(c) for c in chunks) with file.open(user_file.url, "r+") as dst: op = ops.Receive(dst, src, size + 1) with pytest.raises(errors.PartialContent): op.run()
def test_open_read_only(tmpurl): with io.open(tmpurl.path, "wb") as f: f.write(b"x" * 512) with file.open(tmpurl, "r") as f, \ closing(util.aligned_buffer(512)) as buf: assert f.readable() assert not f.writable() f.readinto(buf) assert buf[:] == b"x" * 512
def test_block_size_sparse(user_file, size): with io.open(user_file.path, "wb") as f: f.truncate(size) with file.open(user_file.url, "r") as f: assert f.block_size == user_file.sector_size with io.open(user_file.path, "rb") as f: assert f.read(size) == b"\0" * size
def test_readinto_short_unaligned(tmpurl): with io.open(tmpurl.path, "wb") as f: f.write(b"a" * 42) with file.open(tmpurl, "r") as f, \ closing(util.aligned_buffer(4096)) as buf: n = f.readinto(buf) assert n == 42 assert f.tell() == 42 assert buf[:42] == b"a" * 42 assert buf[42:] == b"\0" * (4096 - 42)
def test_readinto_short_ulinged(tmpurl): with io.open(tmpurl.path, "wb") as f: f.write(b"a" * 4096) with file.open(tmpurl, "r") as f, \ closing(util.aligned_buffer(8192)) as buf: n = f.readinto(buf) assert n == 4096 assert f.tell() == 4096 assert buf[:4096] == b"a" * 4096 assert buf[4096:] == b"\0" * 4096
def test_receive_no_size(tmpurl, data, offset): with io.open(tmpurl.path, "wb") as f: f.write("x" * offset) src = io.BytesIO(data) with file.open(tmpurl, "r+") as dst: op = ops.Receive(dst, src, offset=offset) op.run() with io.open(tmpurl.path, "rb") as f: f.seek(offset) assert f.read(len(data)) == data
def receive(tmpurl, data, size, offset=0): with io.open(tmpurl.path, "wb") as f: f.write("x" * offset) with file.open(tmpurl, "r+") as dst: src = io.BytesIO(data) op = ops.Receive(dst, src, size, offset=offset) op.run() with io.open(tmpurl.path, "rb") as f: f.seek(offset) return f.read(size)
def test_zero_aligned_after_end(tmpurl, sparse): with io.open(tmpurl.path, "wb") as f: f.write(b"x" * 4096) with file.open(tmpurl, "r+", sparse=sparse) as f: f.seek(8192) n = f.zero(4096) assert n == 4096 assert f.tell() == 12288 with io.open(tmpurl.path, "rb") as f: assert f.read(4096) == b"x" * 4096 assert f.read() == b"\0" * 8192
def test_zero(tmpurl, offset, size, sparse): data = "x" * ops.BUFFERSIZE * 2 with io.open(tmpurl.path, "wb") as f: f.write(data) with file.open(tmpurl, "r+", sparse=sparse) as dst: op = ops.Zero(dst, size, offset=offset) op.run() with io.open(tmpurl.path, "rb") as f: assert f.read(offset) == data[:offset] assert f.read(size) == b"\0" * size assert f.read() == data[offset + size:]
def test_zero_aligned_after_end(user_file, sparse): with io.open(user_file.path, "wb") as f: f.write(b"x" * 4096) with file.open(user_file.url, "r+", sparse=sparse) as f: f.seek(8192) n = f.zero(4096) assert n == 4096 assert f.tell() == 12288 with io.open(user_file.path, "rb") as f: assert f.read(4096) == b"x" * 4096 assert f.read() == b"\0" * 8192
def test_flush(user_file, monkeypatch): count = [0] def fsync(fd): count[0] += 1 # This is ugly but probably the only way to test that we call fsync. monkeypatch.setattr(os, "fsync", fsync) with file.open(user_file.url, "r+") as f: f.write(b"x") f.flush() assert count[0] == 1
def test_zero_aligned_middle(user_file, sparse): with io.open(user_file.path, "wb") as f: f.write(b"x" * 3 * 4096) with file.open(user_file.url, "r+", sparse=sparse) as f: f.seek(4096) n = f.zero(4096) assert n == 4096 assert f.tell() == 8192 with io.open(user_file.path, "rb") as f: assert f.read(4096) == b"x" * 4096 assert f.read(4096) == b"\0" * 4096 assert f.read() == b"x" * 4096
def test_send_partial_content(user_file, offset, size): with io.open(user_file.path, "wb") as f: f.truncate(offset + size - 1) dst = io.BytesIO() with file.open(user_file.url, "r") as src: op = ops.Send(src, dst, size, offset=offset) with pytest.raises(errors.PartialContent) as e: op.run() assert e.value.requested == size assert e.value.available == size - 1
def test_zero_aligned_middle(tmpurl, sparse): with io.open(tmpurl.path, "wb") as f: f.write(b"x" * 3 * 4096) with file.open(tmpurl, "r+", sparse=sparse) as f: f.seek(4096) n = f.zero(4096) assert n == 4096 assert f.tell() == 8192 with io.open(tmpurl.path, "rb") as f: assert f.read(4096) == b"x" * 4096 assert f.read(4096) == b"\0" * 4096 assert f.read() == b"x" * 4096
def test_flush(tmpurl, monkeypatch): count = [0] def fsync(fd): count[0] += 1 # This is ugly but probably the only way to test that we call fsync. monkeypatch.setattr(os, "fsync", fsync) with file.open(tmpurl, "r+") as f: f.write(b"x") f.flush() assert count[0] == 1
def test_receive_partial_content(user_file, offset, size): with io.open(user_file.path, "wb") as f: f.truncate(size + offset) src = io.BytesIO(b"x" * (size - 1)) with file.open(user_file.url, "r+") as dst: op = ops.Receive(dst, src, size, offset=offset) with pytest.raises(errors.PartialContent) as e: op.run() assert e.value.requested == size assert e.value.available == size - 1
def test_block_size_preallocated(user_file, size): # This is how vdsm preallocates volumes. This uses fallocate() or fallback # to writing one byte per block. subprocess.check_output( ["fallocate", "--posix", "--length", str(size), user_file.path]) with file.open(user_file.url, "r") as f: assert f.block_size == user_file.sector_size with io.open(user_file.path, "rb") as f: assert f.read(size) == b"\0" * size
def test_send_no_size(user_file, offset, size): data = b"b" * size with io.open(user_file.path, "wb") as f: f.write(b"a" * offset) f.write(data) dst = io.BytesIO() with file.open(user_file.url, "r") as src: op = ops.Send(src, dst, offset=offset) op.run() assert dst.getvalue() == data
def test_write_aligned_at_end(tmpurl): with io.open(tmpurl.path, "wb") as f: f.write(b"a" * 8192) with file.open(tmpurl, "r+") as f, \ closing(util.aligned_buffer(8192)) as buf: buf.write(b"b" * 8192) f.seek(4096) n = f.write(buf) assert n == len(buf) assert f.tell() == 4096 + len(buf) with io.open(tmpurl.path, "rb") as f: assert f.read(4096) == b"a" * 4096 assert f.read() == b"b" * 8192
def test_zero_unaligned_offset_after_end(tmpurl): with io.open(tmpurl.path, "wb") as f: f.write(b"x" * 512) with file.open(tmpurl, "r+") as f: f.seek(600) n = f.zero(10) assert n == 10 assert f.tell() == 610 with io.open(tmpurl.path, "rb") as f: assert f.read(512) == b"x" * 512 assert f.read() == b"\0" * 512
def test_receive_unbuffered_stream(user_file): chunks = [b"a" * 8192, b"b" * 42, b"c" * (8192 - 42)] src = util.UnbufferedStream(chunks) size = sum(len(c) for c in chunks) with file.open(user_file.url, "r+") as dst: op = ops.Receive(dst, src, size) op.run() with io.open(user_file.path, "rb") as f: for c in chunks: assert f.read(len(c)) == c assert f.read() == b""
def test_open_read_write(user_file): with io.open(user_file.path, "wb") as f: f.write(b"a" * user_file.sector_size) with file.open(user_file.url, "r+") as f, \ closing(util.aligned_buffer(user_file.sector_size)) as buf: assert f.readable() assert f.writable() f.readinto(buf) buf[:] = b"b" * user_file.sector_size f.seek(0) f.write(buf) with io.open(user_file.path, "rb") as f: assert f.read() == b"b" * user_file.sector_size
def test_write_aligned_at_end(user_file): with io.open(user_file.path, "wb") as f: f.write(b"a" * 8192) with file.open(user_file.url, "r+") as f, \ closing(util.aligned_buffer(8192)) as buf: buf.write(b"b" * 8192) f.seek(4096) n = f.write(buf) assert n == len(buf) assert f.tell() == 4096 + len(buf) with io.open(user_file.path, "rb") as f: assert f.read(4096) == b"a" * 4096 assert f.read() == b"b" * 8192
def test_open_read_write(tmpurl): with io.open(tmpurl.path, "wb") as f: f.write(b"a" * 512) with file.open(tmpurl, "r+") as f, \ closing(util.aligned_buffer(512)) as buf: assert f.readable() assert f.writable() f.readinto(buf) buf[:] = b"b" * 512 f.seek(0) f.write(buf) with io.open(tmpurl.path, "rb") as f: assert f.read() == b"b" * 512
def test_block_size_hole(user_file, hole_size): data_size = 8192 - hole_size with io.open(user_file.path, "wb") as f: f.seek(hole_size) f.write(b"x" * data_size) with file.open(user_file.url, "r") as f: assert f.block_size == user_file.sector_size with io.open(user_file.path, "rb") as f: assert f.read(hole_size) == b"\0" * hole_size assert f.read(data_size) == b"x" * data_size
def test_readinto_short_unaligned(user_file): size = 42 buf_size = user_file.sector_size with io.open(user_file.path, "wb") as f: f.write(b"a" * size) with file.open(user_file.url, "r") as f, \ closing(util.aligned_buffer(buf_size)) as buf: n = f.readinto(buf) assert n == size assert f.tell() == size assert buf[:size] == b"a" * size assert buf[size:] == b"\0" * (buf_size - size)
def test_zero_unaligned_offset_at_end(tmpurl): with io.open(tmpurl.path, "wb") as f: f.write(b"x" * 1024) # Zero 24 bytes into the last block. with file.open(tmpurl, "r+") as f: f.seek(1000) n = f.zero(100) assert n == 24 assert f.tell() == 1024 with io.open(tmpurl.path, "rb") as f: assert f.read(1000) == b"x" * 1000 assert f.read() == b"\0" * 24
def test_zero_unaligned_offset_complete(tmpurl): with io.open(tmpurl.path, "wb") as f: f.write(b"x" * 1024) # Zero 10 bytes into the second block. with file.open(tmpurl, "r+") as f: f.seek(600) n = f.zero(10) assert n == 10 assert f.tell() == 610 with io.open(tmpurl.path, "rb") as f: assert f.read(600) == b"x" * 600 assert f.read(10) == b"\0" * 10 assert f.read() == b"x" * (1024 - 610)
def test_zero_unaligned_offset_inside(tmpurl): with io.open(tmpurl.path, "wb") as f: f.write(b"x" * 1024) # Zero 12 bytes into the first block. with file.open(tmpurl, "r+") as f: f.seek(500) n = f.zero(100) assert n == 12 assert f.tell() == 512 with io.open(tmpurl.path, "rb") as f: assert f.read(500) == b"x" * 500 assert f.read(12) == b"\0" * 12 assert f.read() == b"x" * 512
def test_zero_unaligned_buffer_slow_path(tmpurl): with io.open(tmpurl.path, "wb") as f: f.write(b"x" * 1024) # Perform slow read-modify-write in the second block. with file.open(tmpurl, "r+") as f: f.seek(512) n = f.zero(10) assert n == 10 assert f.tell() == 522 with io.open(tmpurl.path, "rb") as f: assert f.read(512) == b"x" * 512 assert f.read(10) == b"\0" * 10 assert f.read() == b"x" * 502
def test_zero_unaligned_buffer_fast_path(tmpurl): with io.open(tmpurl.path, "wb") as f: f.write(b"x" * 4096) # Perform fast short zero of 6 blocks. with file.open(tmpurl, "r+") as f: f.seek(512) n = f.zero(3073) assert n == 3072 assert f.tell() == 3584 with io.open(tmpurl.path, "rb") as f: assert f.read(512) == b"x" * 512 assert f.read(3072) == b"\0" * 3072 assert f.read() == b"x" * 512
def test_zero_unaligned_offset_after_end(user_file): size = user_file.sector_size start = size + 10 with io.open(user_file.path, "wb") as f: f.write(b"x" * size) with file.open(user_file.url, "r+") as f: f.seek(start) n = f.zero(10) assert n == 10 assert f.tell() == start + 10 with io.open(user_file.path, "rb") as f: assert f.read(size) == b"x" * size assert f.read() == b"\0" * user_file.sector_size
def test_size(user_file): size = user_file.sector_size * 2 with io.open(user_file.path, "wb") as f: f.truncate(size) with file.open(user_file.url, "r+", sparse=True) as f: # Check initial size. f.seek(100) assert f.size() == size assert f.tell() == 100 # Check that size() updates when file size is modified. f.seek(size) f.zero(user_file.sector_size) assert f.size() == size + user_file.sector_size
def test_write_unaligned_offset_at_end(user_file): size = user_file.sector_size * 2 start = size - 24 with io.open(user_file.path, "wb") as f: f.write(b"x" * size) # Write 24 bytes into the last block. with file.open(user_file.url, "r+") as f: f.seek(size - 24) n = f.write(b"y" * 100) assert n == 24 assert f.tell() == size with io.open(user_file.path, "rb") as f: assert f.read(start) == b"x" * start assert f.read() == b"y" * 24
def test_write_unaligned_offset_after_end(user_file): size = user_file.sector_size with io.open(user_file.path, "wb") as f: f.write(b"x" * size) with file.open(user_file.url, "r+") as f: f.seek(size + 10) n = f.write(b"y" * 10) assert n == 10 assert f.tell() == size + 20 with io.open(user_file.path, "rb") as f: assert f.read(size) == b"x" * size assert f.read(10) == b"\0" * 10 assert f.read(10) == b"y" * 10 assert f.read() == b"\0" * (user_file.sector_size - 20)
def test_receive_padd_to_block_size(tmpurl): with io.open(tmpurl.path, "wb") as f: f.write("x" * 400) size = 200 offset = 300 padding = BLOCKSIZE - size - offset src = io.BytesIO(b"y" * size) with file.open(tmpurl, "r+") as dst: op = ops.Receive(dst, src, size, offset=offset) op.run() with io.open(tmpurl.path, "rb") as f: # Data before offset is not modified. assert f.read(300) == b"x" * offset # Data after offset is modifed, flie extended. assert f.read(200) == b"y" * size # File padded to block size with zeroes. assert f.read() == b"\0" * padding
def test_receive_no_size(user_file, offset, size): with io.open(user_file.path, "wb") as f: f.truncate(offset + size) src = io.BytesIO(b"x" * size) with file.open(user_file.url, "r+") as dst: op = ops.Receive(dst, src, offset=offset) op.run() with io.open(user_file.path, "rb") as f: assert f.read(offset) == b"\0" * offset assert f.read(size) == src.getvalue() file_size = os.path.getsize(user_file.path) trailer = file_size - offset - size assert file_size % user_file.sector_size == 0 assert f.read() == b"\0" * trailer
def test_write_unaligned_buffer_slow_path(user_file): size = user_file.sector_size * 2 with io.open(user_file.path, "wb") as f: f.write(b"x" * size) # Perform slow read-modify-write in the second block. with file.open(user_file.url, "r+") as f: f.seek(user_file.sector_size) n = f.write(b"y" * 10) assert n == 10 assert f.tell() == user_file.sector_size + 10 with io.open(user_file.path, "rb") as f: assert f.read(user_file.sector_size) == b"x" * user_file.sector_size assert f.read(10) == b"y" * 10 assert f.read() == b"x" * (user_file.sector_size - 10)
def test_zero_unaligned_offset_at_end(user_file): size = user_file.sector_size * 2 start = size - 10 with io.open(user_file.path, "wb") as f: f.write(b"x" * size) # Zero 10 bytes into the last block. with file.open(user_file.url, "r+") as f: f.seek(start) n = f.zero(100) assert n == 10 assert f.tell() == size with io.open(user_file.path, "rb") as f: assert f.read(start) == b"x" * start assert f.read() == b"\0" * 10
def test_zero_unaligned_buffer_slow_path(user_file): size = user_file.sector_size * 2 start = user_file.sector_size with io.open(user_file.path, "wb") as f: f.write(b"x" * size) # Perform slow read-modify-write in the second block. with file.open(user_file.url, "r+") as f: f.seek(start) n = f.zero(10) assert n == 10 assert f.tell() == start + 10 with io.open(user_file.path, "rb") as f: assert f.read(start) == b"x" * start assert f.read(10) == b"\0" * 10 assert f.read() == b"x" * (size - start - 10)
def test_zero_unaligned_offset_complete(user_file): size = user_file.sector_size * 2 start = user_file.sector_size + 10 with io.open(user_file.path, "wb") as f: f.write(b"x" * size) # Zero 10 bytes into the second block. with file.open(user_file.url, "r+") as f: f.seek(start) n = f.zero(10) assert n == 10 assert f.tell() == start + 10 with io.open(user_file.path, "rb") as f: assert f.read(start) == b"x" * start assert f.read(10) == b"\0" * 10 assert f.read() == b"x" * (size - start - 10)
def test_write_unaligned_buffer_fast_path(tmpurl): with io.open(tmpurl.path, "wb") as f: f.write(b"x" * 4096) # Perform short fast write of 6 blocks. buf = util.aligned_buffer(3073) buf.write(b"y" * 3073) with closing(buf): with file.open(tmpurl, "r+") as f: f.seek(512) n = f.write(buf) assert n == 3072 assert f.tell() == 3584 with io.open(tmpurl.path, "rb") as f: assert f.read(512) == b"x" * 512 assert f.read(3072) == b"y" * 3072 assert f.read() == b"x" * 512
def test_write_unaligned_offset_inside(user_file): size = user_file.sector_size * 2 start = user_file.sector_size - 12 with io.open(user_file.path, "wb") as f: f.write(b"x" * size) # Write 12 bytes into the first block. with file.open(user_file.url, "r+") as f: f.seek(start) n = f.write(b"y" * 100) assert n == 12 assert f.tell() == user_file.sector_size with io.open(user_file.path, "rb") as f: assert f.read(start) == b"x" * start assert f.read(12) == b"y" * 12 assert f.read() == b"x" * user_file.sector_size
def test_zero_unaligned_buffer_fast_path(user_file): size = user_file.sector_size * 4 start = user_file.sector_size length = user_file.sector_size * 2 with io.open(user_file.path, "wb") as f: f.write(b"x" * size) # Perform fast short zero of 2 blocks. with file.open(user_file.url, "r+") as f: f.seek(start) n = f.zero(length + 1) assert n == length assert f.tell() == start + length with io.open(user_file.path, "rb") as f: assert f.read(start) == b"x" * start assert f.read(length) == b"\0" * length assert f.read() == b"x" * (size - start - length)
def test_dirty(tmpurl): # backend created clean m = file.open(tmpurl, "r+") assert not m.dirty buf = util.aligned_buffer(4096) with closing(buf): # write ans zero dirty the backend buf.write(b"x" * 4096) m.write(buf) assert m.dirty m.flush() assert not m.dirty m.zero(4096) assert m.dirty m.flush() assert not m.dirty # readinto, seek do not affect dirty. m.seek(0) assert not m.dirty m.readinto(buf) assert not m.dirty
parser.add_argument( "-s", "--sparse", dest="sparse", action="store_true", help="Deallocate zeroed space (punch holes)") parser.add_argument( "filename", help="file or block device to fill with zeros") args = parser.parse_args() start_time = time.time() with file.open(args.filename, "r+") as f: if args.length is None: st = os.stat(args.filename) if stat.S_ISBLK(st.st_mode): f.seek(0, os.SEEK_END) size = f.tell() else: size = st.st_size args.length = size - args.offset if args.step is None: args.step = args.length f.seek(args.offset) count = args.length
def test_open_no_create(mode): with pytest.raises(OSError) as e: missing = urllib_parse.urlparse("file:/no/such/path") with file.open(missing, mode): pass assert e.value.errno == errno.ENOENT
def test_debugging_interface(tmpurl): with file.open(tmpurl, "r+") as f: assert f.readable() assert f.writable() assert not f.sparse assert f.name == "file"
def test_block_size(tmpurl): with file.open(tmpurl, "r") as f: # We don't support yet 4k drives. assert f.block_size == 512
def test_zero_allocate_space(tmpurl): with file.open(tmpurl, "r+", sparse=False) as f: f.zero(8192) # File system may report more than file size. assert os.stat(tmpurl.path).st_blocks * 512 >= 8192
def test_zero_sparse_deallocate_space(tmpurl): with io.open(tmpurl.path, "wb") as f: f.write(b"x" * 8192) with file.open(tmpurl, "r+", sparse=True) as f: f.zero(8192) assert os.stat(tmpurl.path).st_blocks * 512 < 8192