def test_detect_zeroes_disabled(nbd_env, format): vol = create_volume(nbd_env, format, "sparse") config = { "sd_id": vol.sdUUID, "img_id": vol.imgUUID, "vol_id": vol.volUUID, } with nbd_server(config) as nbd_url: # Fill image with zeroes. with nbd_client.open(urlparse(nbd_url)) as c: c.write(0 * MiB, b"\0" * nbd_env.virtual_size) c.flush() extents = c.extents(0, nbd_env.virtual_size) image_info = qemuimg.info(vol.volumePath) if format == "raw": assert image_info["actual-size"] == nbd_env.virtual_size else: assert image_info["actual-size"] >= nbd_env.virtual_size log.debug("image extents: %s", extents) # Entire image should be data extent. base_alloc = [(e.length, e.zero) for e in extents["base:allocation"]] assert base_alloc == [(nbd_env.virtual_size, False)] if format != "raw": # Entire image should be allocated. alloc_depth = [(e.length, e.hole) for e in extents["qemu:allocation-depth"]] assert alloc_depth == [(nbd_env.virtual_size, False)]
def test_base_allocation_some_data(nbd_server, user_file, fmt, zero_flags): size = 1024**3 create_image(user_file.path, fmt, size) nbd_server.image = user_file.path nbd_server.fmt = fmt nbd_server.start() # Use qcow2 cluster size to avoid inconsistent results on CentOS and # Fedora. data_length = 64 * 1024 zero_length = size // 2 - data_length with nbd.open(nbd_server.url) as c: # Create 4 extents: data, zero, data, zero. c.write(0, b"x" * data_length) c.write(size // 2, b"x" * data_length) extents = list(nbdutil.extents(c)) assert extents == [ nbd.Extent(length=data_length, flags=0), nbd.Extent(length=zero_length, flags=zero_flags), nbd.Extent(length=data_length, flags=0), nbd.Extent(length=zero_length, flags=zero_flags), ]
def test_base_allocation_many_extents(nbd_server, user_file): # Tested only with raw since qcow2 minimal extent is cluster size (64K), # and writing 1000 extents (62.5 MiB) will be too slow in the CI. # Extents must be multiple of file system block size. extent_length = os.statvfs(user_file.path).f_bsize # Use number which is not a multiple of our buffer capacity (1024 extents) # to ensure we read partial buffers correctly. extents_count = 2000 size = extents_count * extent_length create_image(user_file.path, "raw", size) nbd_server.image = user_file.path nbd_server.fmt = "raw" nbd_server.start() with nbd.open(nbd_server.url) as c: # Write data to all even extents. data = b"x" * extent_length for i in range(0, size, extent_length * 2): c.write(i, data) extents = list(nbdutil.extents(c)) assert len(extents) == extents_count for i, ext in enumerate(extents): assert ext.length == extent_length assert ext.zero == bool(i % 2)
def copy_disk(nbd_url, backup_disk): log.info("Backing up data extents from %s to %s", nbd_url, backup_disk) backup_url = urlparse(nbd_url) with nbd.open(backup_url) as src_client, \ qemu_nbd.open(backup_disk, "qcow2") as dst_client: nbdutil.copy(src_client, dst_client)
def test_open_tcp(tmpdir, url_template, host, export): image = str(tmpdir.join("image")) with open(image, "wb") as f: f.truncate(1024**3) sock = nbd.TCPAddress(host, testutil.random_tcp_port()) url = url_template.format(port=sock.port) log.debug("Trying url=%r export=%r", url, export) with qemu_nbd.run(image, "raw", sock, export_name=export): with nbd.open(urlparse(url)) as c: assert c.export_size == 1024**3
def test_open_unix(tmpdir, url, export): image = str(tmpdir.join("image")) with open(image, "wb") as f: f.truncate(1024**3) sock = nbd.UnixAddress(tmpdir.join("sock")) url = url.replace("/path", sock) log.debug("Trying url=%r export=%r", url, export) with qemu_nbd.run(image, "raw", sock, export_name=export): with nbd.open(urlparse(url)) as c: assert c.export_size == 1024**3
def test_raw_readinto(nbd_server): offset = 1024**2 data = b"can read from raw" with io.open(nbd_server.image, "wb") as f: f.truncate(2 * 1024**2) f.seek(1024**2) f.write(data) nbd_server.start() with nbd.open(nbd_server.url) as c: buf = bytearray(len(data)) c.readinto(offset, buf) assert buf == data
def test_allocation_depth(nbd_env, format, backing_chain): # Check that qemu-nbd exposes the "qemu:allocation-depth" meta # context for qcow2 format. vol = create_volume(nbd_env, format, "sparse") config = { "sd_id": vol.sdUUID, "img_id": vol.imgUUID, "vol_id": vol.volUUID, "readonly": True, "backing_chain": backing_chain, } with nbd_server(config) as nbd_url: with nbd_client.open(urlparse(nbd_url)) as c: extents = c.extents(0, nbd_env.virtual_size) log.debug("extents: %s", extents) assert ("qemu:allocation-depth" in extents) == (format != "raw")
def test_extents_reply_error(nbd_server, user_file): """ The server SHOULD return NBD_EINVAL if it receives a NBD_CMD_BLOCK_STATUS request including one or more sectors beyond the size of the device. """ size = 1024**2 create_image(user_file.path, "raw", size) nbd_server.image = user_file.path nbd_server.fmt = "raw" nbd_server.start() with nbd.open(nbd_server.url) as c: with pytest.raises(nbd.ReplyError) as e: c.extents(0, size + 1) # Server should return this, qemu does. assert e.value.code == errno.EINVAL # The next request should succeed. assert c.read(4096, 1) == b"\0"
def test_detect_zeroes_discard(nbd_env, format): vol = create_volume(nbd_env, format, "sparse") config = { "sd_id": vol.sdUUID, "img_id": vol.imgUUID, "vol_id": vol.volUUID, "discard": True, "detect_zeroes": True, } with nbd_server(config) as nbd_url: # Fill image with zeroes. The writes should be converted to write # zeroes command. with nbd_client.open(urlparse(nbd_url)) as c: c.write(0 * MiB, b"\0" * nbd_env.virtual_size) c.flush() extents = c.extents(0, nbd_env.virtual_size) image_info = qemuimg.info(vol.volumePath) if format == "raw": assert image_info["actual-size"] == 0 else: # Image contains only metadata. assert image_info["actual-size"] <= MiB log.debug("image extents: %s", extents) # Entire image should be zero extent. For qcow2 image, qemu-nbd writes zero # clusters. For raw image, qemu-nbd deallocates the entire image. base_alloc = [(e.length, e.zero) for e in extents["base:allocation"]] assert base_alloc == [(nbd_env.virtual_size, True)] if format != "raw": # Entire image should be allocated. For qcow2 image, zero # clusters are always allocated. alloc_depth = [(e.length, e.hole) for e in extents["qemu:allocation-depth"]] assert alloc_depth == [(nbd_env.virtual_size, False)]
def copy_dirty(nbd_url, backup_disk): log.info("Backing up dirty extents from %s to %s", nbd_url, backup_disk) backup_url = urlparse(nbd_url) with nbd.open(backup_url, dirty=True) as src_client, \ qemu_nbd.open(backup_disk, "qcow2") as dst_client: buf = bytearray(4 * 1024**2) offset = 0 for ext in nbdutil.extents(src_client, dirty=True): if ext.dirty: todo = ext.length while todo: step = min(todo, len(buf)) view = memoryview(buf)[:step] src_client.readinto(offset, view) dst_client.write(offset, view) offset += step todo -= step else: offset += ext.length
def test_base_allocation_full(nbd_server, user_file, fmt): size = 1024**2 create_image(user_file.path, fmt, size) nbd_server.image = user_file.path nbd_server.fmt = fmt nbd_server.start() with nbd.open(nbd_server.url) as c: c.write(0, b"x" * size) # Entire image. extents = c.extents(0, size)["base:allocation"] assert extents == [nbd.Extent(length=size, flags=0)] # First block. extents = c.extents(0, 4096)["base:allocation"] assert extents == [nbd.Extent(length=4096, flags=0)] # Last block. extents = c.extents(size - 4096, 4096)["base:allocation"] assert extents == [nbd.Extent(length=4096, flags=0)] # Some block. extents = c.extents(4096, 4096)["base:allocation"] assert extents == [nbd.Extent(length=4096, flags=0)] # Unaligned start. extents = c.extents(4096 - 1, 4096 + 1)["base:allocation"] assert extents == [nbd.Extent(length=4096 + 1, flags=0)] # Unaligned end. extents = c.extents(4096, 4096 + 1)["base:allocation"] assert extents == [nbd.Extent(length=4096 + 1, flags=0)] # Unaligned start and end. extents = c.extents(4096 - 1, 4096 + 2)["base:allocation"] assert extents == [nbd.Extent(length=4096 + 2, flags=0)]
def test_detect_zeroes_no_discard(nbd_env, format): vol = create_volume(nbd_env, format, "sparse") config = { "sd_id": vol.sdUUID, "img_id": vol.imgUUID, "vol_id": vol.volUUID, "discard": False, "detect_zeroes": True, } with nbd_server(config) as nbd_url: # Fill image with zeroes. The writes should be converted to write # zeroes command. with nbd_client.open(urlparse(nbd_url)) as c: c.write(0 * MiB, b"\0" * nbd_env.virtual_size) c.flush() extents = c.extents(0, nbd_env.virtual_size) # For qcow2 format, in qemu-nd < 6.2 actual zeroes written to image. # In 6.2 zeroes seem to be converted to zero clusters. Since the # behavior is not consistent we don't have easy way to verify. if format == "raw": image_info = qemuimg.info(vol.volumePath) assert image_info["actual-size"] == nbd_env.virtual_size log.debug("image extents: %s", extents) # Entire image should be zero extent. base_alloc = [(e.length, e.zero) for e in extents["base:allocation"]] assert base_alloc == [(nbd_env.virtual_size, True)] if format != "raw": # qemu-nbd allocated entire image in both cases. alloc_depth = [(e.length, e.hole) for e in extents["qemu:allocation-depth"]] assert alloc_depth == [(nbd_env.virtual_size, False)]
def test_base_allocation_empty(nbd_server, user_file, fmt, hole_flags): size = 1024**3 create_image(user_file.path, fmt, size) nbd_server.image = user_file.path nbd_server.fmt = fmt nbd_server.start() with nbd.open(nbd_server.url) as c: # Entire image. extents = list(nbdutil.extents(c)) assert extents == [nbd.Extent(length=size, flags=hole_flags)] # First block. extents = list(nbdutil.extents(c, length=4096)) assert extents == [nbd.Extent(length=4096, flags=hole_flags)] # Last block. extents = list(nbdutil.extents(c, offset=size - 4096, length=4096)) assert extents == [nbd.Extent(length=4096, flags=hole_flags)] # Some block. extents = list(nbdutil.extents(c, offset=4096, length=4096)) assert extents == [nbd.Extent(length=4096, flags=hole_flags)] # Unaligned start. extents = list(nbdutil.extents(c, offset=4096 - 1, length=4096 + 1)) assert extents == [nbd.Extent(length=4096 + 1, flags=hole_flags)] # Unaligned end. extents = list(nbdutil.extents(c, offset=4096, length=4096 + 1)) assert extents == [nbd.Extent(length=4096 + 1, flags=hole_flags)] # Unaligned start and end. extents = list(nbdutil.extents(c, offset=4096 - 1, length=4096 + 2)) assert extents == [nbd.Extent(length=4096 + 2, flags=hole_flags)]
def test_base_allocation_some_data_unaligned(nbd_server, user_file, fmt, zero_flags): size = 1024**2 create_image(user_file.path, fmt, size) nbd_server.image = user_file.path nbd_server.fmt = fmt nbd_server.start() data_length = 64 * 1024 data_offset = 2 * data_length with nbd.open(nbd_server.url) as c: # Create 3 extents: zero, data, zero. c.write(data_offset, b"x" * data_length) # Unaligned part from first extent and last extent. extents = list(nbdutil.extents(c, data_offset - 1, data_length + 2)) assert extents == [ nbd.Extent(length=1, flags=zero_flags), nbd.Extent(length=data_length, flags=0), nbd.Extent(length=1, flags=zero_flags), ] # Unaligned part from second extent. extents = list(nbdutil.extents(c, data_offset + 1, data_length - 2)) assert extents == [ nbd.Extent(length=data_length - 2, flags=0), ] # Unaligned part from second and last extents. extents = list(nbdutil.extents(c, data_offset + 1, data_length)) assert extents == [ nbd.Extent(length=data_length - 1, flags=0), nbd.Extent(length=1, flags=zero_flags), ]
def test_base_allocation_empty(nbd_server, user_file, fmt, zero_flags): size = nbd.MAX_LENGTH create_image(user_file.path, fmt, size) nbd_server.image = user_file.path nbd_server.fmt = fmt nbd_server.start() with nbd.open(nbd_server.url) as c: # Entire image. extents = c.extents(0, size)["base:allocation"] assert extents == [nbd.Extent(length=size, flags=zero_flags)] # First block. extents = c.extents(0, 4096)["base:allocation"] assert extents == [nbd.Extent(length=4096, flags=zero_flags)] # Last block. extents = c.extents(size - 4096, 4096)["base:allocation"] assert extents == [nbd.Extent(length=4096, flags=zero_flags)] # Some block. extents = c.extents(4096, 4096)["base:allocation"] assert extents == [nbd.Extent(length=4096, flags=zero_flags)] # Unaligned start. extents = c.extents(4096 - 1, 4096 + 1)["base:allocation"] assert extents == [nbd.Extent(length=4096 + 1, flags=zero_flags)] # Unaligned end. extents = c.extents(4096, 4096 + 1)["base:allocation"] assert extents == [nbd.Extent(length=4096 + 1, flags=zero_flags)] # Unaligned start and end. extents = c.extents(4096 - 1, 4096 + 2)["base:allocation"] assert extents == [nbd.Extent(length=4096 + 2, flags=zero_flags)]
def test_bitmap_backing_chain(nbd_env): vol1 = create_volume(nbd_env, "raw", "sparse") # Write first cluster to vol1 - this cluster is not recorded in any bitmap. qemuio.write_pattern(vol1.volumePath, "raw", offset=1 * MiB, len=64 * KiB, pattern=0xf1) # Simulate a snapshot - bitmap1 is created empty in vol2. vol2 = create_volume(nbd_env, "qcow2", "sparse", parent=vol1) bitmap1 = str(uuid.uuid4()) qemuimg.bitmap_add(vol2.volumePath, bitmap1).run() # Write second cluster in vol2 - this cluster is recorded only in bitmap 1. qemuio.write_pattern(vol2.volumePath, "qcow2", offset=2 * MiB, len=64 * KiB, pattern=0xf2) # Simulate another snapshot - bitmap1 is created empty in vol3. vol3 = create_volume(nbd_env, "qcow2", "sparse", parent=vol2) qemuimg.bitmap_add(vol3.volumePath, bitmap1).run() # Simulate backup - bitmap 2 is created in vol3. bitmap2 = str(uuid.uuid4()) qemuimg.bitmap_add(vol3.volumePath, bitmap2).run() # Write third cluster in vol3. This cluster is recorded in both bitmaps. qemuio.write_pattern(vol3.volumePath, "qcow2", offset=3 * MiB, len=64 * KiB, pattern=0xf3) # Test bitmap 1 - recording changes since bitmap 1 was added. config = { "sd_id": vol3.sdUUID, "img_id": vol3.imgUUID, "vol_id": vol3.volUUID, "readonly": True, "bitmap": bitmap1, } with nbd_server(config) as nbd_url: with nbd_client.open(urlparse(nbd_url), dirty=True) as c: extents = c.extents(0, nbd_env.virtual_size) dirty_extents = [(e.length, e.dirty) for e in extents[c.dirty_bitmap]] assert dirty_extents == [ (2 * MiB, False), (64 * KiB, True), (1 * MiB - 64 * KiB, False), (64 * KiB, True), (nbd_env.virtual_size - 3 * MiB - 64 * KiB, False), ] assert c.read(1 * MiB, 64 * KiB) == b"\xf1" * 64 * KiB assert c.read(2 * MiB, 64 * KiB) == b"\xf2" * 64 * KiB # Test bitmap 2 - recording changes since bitmap 2 was added. config = { "sd_id": vol3.sdUUID, "img_id": vol3.imgUUID, "vol_id": vol3.volUUID, "readonly": True, "bitmap": bitmap2, } with nbd_server(config) as nbd_url: with nbd_client.open(urlparse(nbd_url), dirty=True) as c: extents = c.extents(0, nbd_env.virtual_size) dirty_extents = [(e.length, e.dirty) for e in extents[c.dirty_bitmap]] assert dirty_extents == [ (3 * MiB, False), (64 * KiB, True), (nbd_env.virtual_size - 3 * MiB - 64 * KiB, False), ] assert c.read(2 * MiB, 64 * KiB) == b"\xf2" * 64 * KiB
def test_bitmap_single_volume(nbd_env): vol = create_volume(nbd_env, "qcow2", "sparse") # Write first cluster - this cluster is not recorded in any bitmap. qemuio.write_pattern(vol.volumePath, "qcow2", offset=1 * MiB, len=64 * KiB, pattern=0xf1) # Add bitmap 1 and write second cluster. bitmap1 = str(uuid.uuid4()) qemuimg.bitmap_add(vol.volumePath, bitmap1).run() qemuio.write_pattern(vol.volumePath, "qcow2", offset=2 * MiB, len=64 * KiB, pattern=0xf2) # Add bitmap 2 and write third cluster. bitmap2 = str(uuid.uuid4()) qemuimg.bitmap_add(vol.volumePath, bitmap2).run() qemuio.write_pattern(vol.volumePath, "qcow2", offset=3 * MiB, len=64 * KiB, pattern=0xf3) # Test bitmap 1 - recording changes since bitmap 1 was added. config = { "sd_id": vol.sdUUID, "img_id": vol.imgUUID, "vol_id": vol.volUUID, "readonly": True, "bitmap": bitmap1, } with nbd_server(config) as nbd_url: with nbd_client.open(urlparse(nbd_url), dirty=True) as c: extents = c.extents(0, nbd_env.virtual_size) dirty_extents = [(e.length, e.dirty) for e in extents[c.dirty_bitmap]] assert dirty_extents == [ (2 * MiB, False), (64 * KiB, True), (1 * MiB - 64 * KiB, False), (64 * KiB, True), (nbd_env.virtual_size - 3 * MiB - 64 * KiB, False), ] assert c.read(1 * MiB, 64 * KiB) == b"\xf1" * 64 * KiB assert c.read(2 * MiB, 64 * KiB) == b"\xf2" * 64 * KiB # Test bitmap 2 - recording changes since bitmap 2 was added. config = { "sd_id": vol.sdUUID, "img_id": vol.imgUUID, "vol_id": vol.volUUID, "readonly": True, "bitmap": bitmap2, } with nbd_server(config) as nbd_url: with nbd_client.open(urlparse(nbd_url), dirty=True) as c: extents = c.extents(0, nbd_env.virtual_size) dirty_extents = [(e.length, e.dirty) for e in extents[c.dirty_bitmap]] assert dirty_extents == [ (3 * MiB, False), (64 * KiB, True), (nbd_env.virtual_size - 3 * MiB - 64 * KiB, False), ] assert c.read(2 * MiB, 64 * KiB) == b"\xf2" * 64 * KiB