def test_merge_simple(): n = GiB a = [nbd.Extent(n, 0)] b = [nbd.Extent(n, 0)] merged = list(nbdutil.merged(a, b)) assert merged == a
def test_base_allocation_some_data(nbd_server, user_file, fmt, zero_flags): size = 1024**3 create_image(user_file.path, fmt, size) nbd_server.image = user_file.path nbd_server.fmt = fmt nbd_server.start() # Use qcow2 cluster size to avoid inconsistent results on CentOS and # Fedora. data_length = 64 * 1024 zero_length = size // 2 - data_length with nbd.open(nbd_server.url) as c: # Create 4 extents: data, zero, data, zero. c.write(0, b"x" * data_length) c.write(size // 2, b"x" * data_length) extents = list(nbdutil.extents(c)) assert extents == [ nbd.Extent(length=data_length, flags=0), nbd.Extent(length=zero_length, flags=zero_flags), nbd.Extent(length=data_length, flags=0), nbd.Extent(length=zero_length, flags=zero_flags), ]
def test_dirty_bitmap(tmpdir): size = 1024**2 # Create image with empty bitmap. img = str(tmpdir.join("img.qcow2")) qemu_img.create(img, "qcow2", size=size) qemu_img.bitmap_add(img, "b0") # Write data to image, modifying the bitmap. with qemu_nbd.open(img, "qcow2") as c: # This will allocate one cluster. By default bitmap granularity is also # one cluster, so this will make the first extent dirty. c.write(0, b"a") c.flush() # Read dirty extents. with qemu_nbd.open(img, "qcow2", read_only=True, bitmap="b0") as c: extents = c.extents(0, size)[nbd.QEMU_DIRTY_BITMAP + "b0"] bitmap = qemu_img.info(img)["format-specific"]["data"]["bitmaps"][0] assert extents == [ nbd.Extent(length=bitmap["granularity"], flags=nbd.EXTENT_DIRTY), nbd.Extent(length=size - bitmap["granularity"], flags=0), ]
def test_extents_offset(max_extents): n = GiB c = fake_client(n, max_extents=max_extents) extents = list(nbdutil.extents(c, offset=3 * n)) assert extents == [ nbd.Extent(1 * n, STATE_ZERO | STATE_HOLE), nbd.Extent(2 * n, STATE_ZERO | STATE_HOLE | EXTENT_BACKING), ]
def test_extents_offset_dirty(max_extents): n = GiB c = fake_client(n, max_extents=max_extents) extents = list(nbdutil.extents(c, offset=3 * n, dirty=True)) assert extents == [ nbd.Extent(1 * n, STATE_ZERO | STATE_HOLE | EXTENT_DIRTY), nbd.Extent(2 * n, STATE_ZERO | STATE_HOLE), ]
def test_extents_length(max_extents): n = GiB c = fake_client(n, max_extents=max_extents) extents = list(nbdutil.extents(c, length=3 * n)) assert extents == [ nbd.Extent(2 * n, 0), nbd.Extent(1 * n, STATE_ZERO | STATE_HOLE), ]
def test_extent_dirty_bitmap(): # Clean area. ext = nbd.Extent(4096, 0) assert not ext.dirty assert ext.flags == 0 # Dirty area. ext = nbd.Extent(4096, nbd.STATE_DIRTY) assert ext.dirty assert ext.flags == nbd.STATE_DIRTY
def test_fake_client_max_extents(): n = MiB c = fake_client(n, max_extents=1) res = c.extents(0, 6 * n) assert res == { nbd.BASE_ALLOCATION: [ nbd.Extent(2 * n, 0), ], nbd.QEMU_ALLOCATION_DEPTH: [ nbd.Extent(3 * n, 0), # depth=1 ], c.dirty_bitmap: [ nbd.Extent(1 * n, 0), ], } c = fake_client(n, max_extents=2) res = c.extents(n, 4 * n) assert res == { nbd.BASE_ALLOCATION: [ nbd.Extent(1 * n, 0), nbd.Extent(2 * n, STATE_ZERO | STATE_HOLE), ], nbd.QEMU_ALLOCATION_DEPTH: [ nbd.Extent(2 * n, 0), # depth=1 nbd.Extent(1 * n, 0), # depth=2 ], c.dirty_bitmap: [ nbd.Extent(3 * n, EXTENT_DIRTY), nbd.Extent(1 * n, 0) ], }
def fake_client(n, max_extents=0): """ A client simulating few interesting cases: - 3 alloction types: data, zero cluster, and unallocated extent. - dirty extents convering both data and zero cluster. - extents of different meta context of different length. - server returning short reply. """ return FakeClient( alloc=[ nbd.Extent(2 * n, 0), nbd.Extent(2 * n, STATE_ZERO | STATE_HOLE), nbd.Extent(2 * n, STATE_ZERO | STATE_HOLE), ], depth=[ nbd.Extent(3 * n, 0), # depth=1 nbd.Extent(1 * n, 0), # depth=2 nbd.Extent(2 * n, EXTENT_BACKING), # depth=0 ], dirty=[ nbd.Extent(1 * n, 0), nbd.Extent(3 * n, EXTENT_DIRTY), nbd.Extent(2 * n, 0) ], max_extents=max_extents, )
def test_extents_all_no_depth(max_extents): n = GiB c = fake_client(n, max_extents=max_extents) # Simulate the case when server does not report allocation depth. c.depth = None extents = list(nbdutil.extents(c)) assert extents == [ nbd.Extent(2 * n, 0), nbd.Extent(4 * n, STATE_ZERO | STATE_HOLE), ]
def test_extents_last_extent_excceeds_export_size(): n = GiB c = fake_client(n) # Clip export size so we get extra extent info exceeding the request # length. c.export_size -= GiB # Merge base:allocation and qemu:allocation-depth. extents = list(nbdutil.extents(c)) assert extents == [ nbd.Extent(2 * n, 0), nbd.Extent(2 * n, STATE_ZERO | STATE_HOLE), nbd.Extent(1 * n, STATE_ZERO | STATE_HOLE | EXTENT_BACKING), ]
def test_extents_last_extent_excceeds_export_size_dirty(): n = GiB c = fake_client(n) # Clip export size so we get extra extent info exceeding the request # length. c.export_size -= GiB extents = list(nbdutil.extents(c, dirty=True)) assert extents == [ nbd.Extent(1 * n, 0), nbd.Extent(1 * n, EXTENT_DIRTY), nbd.Extent(2 * n, STATE_ZERO | STATE_HOLE | EXTENT_DIRTY), nbd.Extent(1 * n, STATE_ZERO | STATE_HOLE), ]
def test_fake_client_clip_both(): n = MiB c = fake_client(n) res = c.extents(2 * n, 2 * n) assert res == { nbd.BASE_ALLOCATION: [ nbd.Extent(2 * n, STATE_ZERO | STATE_HOLE), ], nbd.QEMU_ALLOCATION_DEPTH: [ nbd.Extent(1 * n, 0), # depth=1 nbd.Extent(1 * n, 0), # depth=2 ], c.dirty_bitmap: [ nbd.Extent(2 * n, EXTENT_DIRTY), ], }
def test_extent_base_allocation(): # Allocated aread with data. ext = nbd.Extent(4096, 0) assert not ext.zero assert not ext.hole assert ext.flags == 0 # Allocated aread that reads as zero. ext = nbd.Extent(4096, nbd.STATE_ZERO) assert ext.zero assert not ext.hole assert ext.flags == nbd.STATE_ZERO # Unallocated aread that reads as zero. ext = nbd.Extent(4096, nbd.STATE_ZERO | nbd.STATE_HOLE) assert ext.zero assert ext.hole assert ext.flags == nbd.STATE_ZERO | nbd.STATE_HOLE
def test_detect_zeroes_disabled(tmpdir, fmt, detect_zeroes): size = 1024**2 disk = str(tmpdir.join("disk." + fmt)) qemu_img.create(disk, fmt, size=size) with qemu_nbd.open(disk, fmt, detect_zeroes=detect_zeroes) as c: # These zeroes should not be detected. c.write(0, b"\0" * size) c.flush() extents = c.extents(0, size) assert extents["base:allocation"] == [ nbd.Extent(length=1048576, flags=0), ] if fmt != "raw": assert extents["qemu:allocation-depth"] == [ nbd.Extent(length=1048576, flags=0), ]
def test_extent_merge_qcow2_unallocated_cluster(): alloc_data = nbd.Extent.pack(65536, nbd.STATE_ZERO | nbd.STATE_HOLE) alloc = nbd.Extent.unpack(alloc_data) depth_data = nbd.Extent.pack(65536, 0) depth = nbd.Extent.unpack(depth_data, nbd.Extent.DEPTH) merged = nbd.Extent(65536, alloc.flags | depth.flags) assert merged.length == 65536 assert merged.zero is True assert merged.hole is True
def test_extent_merge_qcow2_data_cluster(): alloc_data = nbd.Extent.pack(65536, 0) alloc = nbd.Extent.unpack(alloc_data) depth_data = nbd.Extent.pack(65536, 1) depth = nbd.Extent.unpack(depth_data, nbd.Extent.DEPTH) merged = nbd.Extent(65536, alloc.flags | depth.flags) assert merged.length == 65536 assert merged.zero is False assert merged.hole is False
def test_extent_merge_dirty_data(): alloc_data = nbd.Extent.pack(4096, 0) alloc = nbd.Extent.unpack(alloc_data, nbd.Extent.DIRTY) dirty_data = nbd.Extent.pack(4096, nbd.STATE_DIRTY) dirty = nbd.Extent.unpack(dirty_data, nbd.Extent.DIRTY) merged = nbd.Extent(4096, alloc.flags | dirty.flags) assert merged.length == 4096 assert not merged.zero assert not merged.hole assert merged.dirty
def test_extent_merge_clean_hole(): alloc_data = nbd.Extent.pack(4096, nbd.STATE_ZERO | nbd.STATE_HOLE) alloc = nbd.Extent.unpack(alloc_data) dirty_data = nbd.Extent.pack(4096, 0) dirty = nbd.Extent.unpack(dirty_data, nbd.Extent.DIRTY) merged = nbd.Extent(4096, alloc.flags | dirty.flags) assert merged.length == 4096 assert merged.zero is True # Hole can be detected only when using depth data with qcow2 image. assert merged.hole is False assert not merged.dirty
def lookup(self, offset, length, extents): end = offset + length start = 0 count = 0 for e in extents: # Skip before the requested range: # request: [ ] # extent: |----| if start + e.length <= offset: start += e.length continue length = e.length # Clip extent before offset: # request: [ ] # extent: |-------| # result: |===| if start < offset: clip = offset - start length -= clip start += clip # Clip extent after end: # request: [ ] # extent: |-------| # result: |====| if start + length > end: clip = start + length - end length -= clip yield nbd.Extent(length, e.flags) # NBD server is allowed to return short reply with one or more # extents. count += 1 if self.max_extents and count == self.max_extents: break start += length # Stop lookup after the requested range: # request: [ ] # extent: |----| if start >= end: break
def test_merge_split_both(): n = GiB a = [ nbd.Extent(n * 1, 1), nbd.Extent(n * 2, 2), ] b = [ nbd.Extent(n * 2, 4), nbd.Extent(n * 1, 8), ] merged1 = list(nbdutil.merged(a, b)) assert merged1 == [ nbd.Extent(n, 1 | 4), nbd.Extent(n, 2 | 4), nbd.Extent(n, 2 | 8), ] merged2 = list(nbdutil.merged(b, a)) assert merged2 == merged1
def test_merge_split_one(): n = GiB a = [ nbd.Extent(n, 1), nbd.Extent(n, 2), nbd.Extent(n, 4), ] b = [ nbd.Extent(n * 3, 8) ] merged1 = list(nbdutil.merged(a, b)) assert merged1 == [ nbd.Extent(n, 1 | 8), nbd.Extent(n, 2 | 8), nbd.Extent(n, 4 | 8), ] merged2 = list(nbdutil.merged(b, a)) assert merged2 == merged1
def test_merge_clip(): n = GiB a = [ nbd.Extent(n * 1, 1), nbd.Extent(n * 1, 2), ] b = [ nbd.Extent(n * 1, 4), nbd.Extent(n * 2, 8), ] merged1 = list(nbdutil.merged(a, b)) assert merged1 == [ nbd.Extent(n * 1, 1 | 4), nbd.Extent(n * 1, 2 | 8), ] merged2 = list(nbdutil.merged(b, a)) assert merged2 == merged1
def test_base_allocation_full(nbd_server, user_file, fmt): size = 1024**2 create_image(user_file.path, fmt, size) nbd_server.image = user_file.path nbd_server.fmt = fmt nbd_server.start() with nbd.open(nbd_server.url) as c: c.write(0, b"x" * size) # Entire image. extents = c.extents(0, size)["base:allocation"] assert extents == [nbd.Extent(length=size, flags=0)] # First block. extents = c.extents(0, 4096)["base:allocation"] assert extents == [nbd.Extent(length=4096, flags=0)] # Last block. extents = c.extents(size - 4096, 4096)["base:allocation"] assert extents == [nbd.Extent(length=4096, flags=0)] # Some block. extents = c.extents(4096, 4096)["base:allocation"] assert extents == [nbd.Extent(length=4096, flags=0)] # Unaligned start. extents = c.extents(4096 - 1, 4096 + 1)["base:allocation"] assert extents == [nbd.Extent(length=4096 + 1, flags=0)] # Unaligned end. extents = c.extents(4096, 4096 + 1)["base:allocation"] assert extents == [nbd.Extent(length=4096 + 1, flags=0)] # Unaligned start and end. extents = c.extents(4096 - 1, 4096 + 2)["base:allocation"] assert extents == [nbd.Extent(length=4096 + 2, flags=0)]
def test_base_allocation_empty(nbd_server, user_file, fmt, hole_flags): size = 1024**3 create_image(user_file.path, fmt, size) nbd_server.image = user_file.path nbd_server.fmt = fmt nbd_server.start() with nbd.open(nbd_server.url) as c: # Entire image. extents = list(nbdutil.extents(c)) assert extents == [nbd.Extent(length=size, flags=hole_flags)] # First block. extents = list(nbdutil.extents(c, length=4096)) assert extents == [nbd.Extent(length=4096, flags=hole_flags)] # Last block. extents = list(nbdutil.extents(c, offset=size - 4096, length=4096)) assert extents == [nbd.Extent(length=4096, flags=hole_flags)] # Some block. extents = list(nbdutil.extents(c, offset=4096, length=4096)) assert extents == [nbd.Extent(length=4096, flags=hole_flags)] # Unaligned start. extents = list(nbdutil.extents(c, offset=4096 - 1, length=4096 + 1)) assert extents == [nbd.Extent(length=4096 + 1, flags=hole_flags)] # Unaligned end. extents = list(nbdutil.extents(c, offset=4096, length=4096 + 1)) assert extents == [nbd.Extent(length=4096 + 1, flags=hole_flags)] # Unaligned start and end. extents = list(nbdutil.extents(c, offset=4096 - 1, length=4096 + 2)) assert extents == [nbd.Extent(length=4096 + 2, flags=hole_flags)]
def test_base_allocation_empty(nbd_server, user_file, fmt, zero_flags): size = nbd.MAX_LENGTH create_image(user_file.path, fmt, size) nbd_server.image = user_file.path nbd_server.fmt = fmt nbd_server.start() with nbd.open(nbd_server.url) as c: # Entire image. extents = c.extents(0, size)["base:allocation"] assert extents == [nbd.Extent(length=size, flags=zero_flags)] # First block. extents = c.extents(0, 4096)["base:allocation"] assert extents == [nbd.Extent(length=4096, flags=zero_flags)] # Last block. extents = c.extents(size - 4096, 4096)["base:allocation"] assert extents == [nbd.Extent(length=4096, flags=zero_flags)] # Some block. extents = c.extents(4096, 4096)["base:allocation"] assert extents == [nbd.Extent(length=4096, flags=zero_flags)] # Unaligned start. extents = c.extents(4096 - 1, 4096 + 1)["base:allocation"] assert extents == [nbd.Extent(length=4096 + 1, flags=zero_flags)] # Unaligned end. extents = c.extents(4096, 4096 + 1)["base:allocation"] assert extents == [nbd.Extent(length=4096 + 1, flags=zero_flags)] # Unaligned start and end. extents = c.extents(4096 - 1, 4096 + 2)["base:allocation"] assert extents == [nbd.Extent(length=4096 + 2, flags=zero_flags)]
def test_base_allocation_some_data_unaligned(nbd_server, user_file, fmt, zero_flags): size = 1024**2 create_image(user_file.path, fmt, size) nbd_server.image = user_file.path nbd_server.fmt = fmt nbd_server.start() data_length = 64 * 1024 data_offset = 2 * data_length with nbd.open(nbd_server.url) as c: # Create 3 extents: zero, data, zero. c.write(data_offset, b"x" * data_length) # Unaligned part from first extent and last extent. extents = list(nbdutil.extents(c, data_offset - 1, data_length + 2)) assert extents == [ nbd.Extent(length=1, flags=zero_flags), nbd.Extent(length=data_length, flags=0), nbd.Extent(length=1, flags=zero_flags), ] # Unaligned part from second extent. extents = list(nbdutil.extents(c, data_offset + 1, data_length - 2)) assert extents == [ nbd.Extent(length=data_length - 2, flags=0), ] # Unaligned part from second and last extents. extents = list(nbdutil.extents(c, data_offset + 1, data_length)) assert extents == [ nbd.Extent(length=data_length - 1, flags=0), nbd.Extent(length=1, flags=zero_flags), ]
def test_bitmap_single_volume(nbd_env): vol = create_volume(nbd_env, "qcow2", "sparse") # Write first cluster - this cluster is not recorded in any bitmap. qemuio.write_pattern(vol.volumePath, "qcow2", offset=1 * MiB, len=64 * KiB, pattern=0xf1) # Add bitmap 1 and write second cluster. bitmap1 = str(uuid.uuid4()) qemuimg.bitmap_add(vol.volumePath, bitmap1).run() qemuio.write_pattern(vol.volumePath, "qcow2", offset=2 * MiB, len=64 * KiB, pattern=0xf2) # Add bitmap 2 and write third cluster. bitmap2 = str(uuid.uuid4()) qemuimg.bitmap_add(vol.volumePath, bitmap2).run() qemuio.write_pattern(vol.volumePath, "qcow2", offset=3 * MiB, len=64 * KiB, pattern=0xf3) # Test bitmap 1 - recording changes since bitmap 1 was added. config = { "sd_id": vol.sdUUID, "img_id": vol.imgUUID, "vol_id": vol.volUUID, "readonly": True, "bitmap": bitmap1, } with nbd_server(config) as nbd_url: with nbd_client.open(urlparse(nbd_url), dirty=True) as c: extents = c.extents(0, nbd_env.virtual_size) assert extents[c.dirty_bitmap] == [ nbd_client.Extent(2 * MiB, 0), nbd_client.Extent(64 * KiB, 1), nbd_client.Extent(1 * MiB - 64 * KiB, 0), nbd_client.Extent(64 * KiB, 1), nbd_client.Extent(nbd_env.virtual_size - 3 * MiB - 64 * KiB, 0), ] assert c.read(1 * MiB, 64 * KiB) == b"\xf1" * 64 * KiB assert c.read(2 * MiB, 64 * KiB) == b"\xf2" * 64 * KiB # Test bitmap 2 - recording changes since bitmap 2 was added. config = { "sd_id": vol.sdUUID, "img_id": vol.imgUUID, "vol_id": vol.volUUID, "readonly": True, "bitmap": bitmap2, } with nbd_server(config) as nbd_url: with nbd_client.open(urlparse(nbd_url), dirty=True) as c: extents = c.extents(0, nbd_env.virtual_size) assert extents[c.dirty_bitmap] == [ nbd_client.Extent(3 * MiB, 0), nbd_client.Extent(64 * KiB, 1), nbd_client.Extent(nbd_env.virtual_size - 3 * MiB - 64 * KiB, 0), ] assert c.read(2 * MiB, 64 * KiB) == b"\xf2" * 64 * KiB
def test_extent_compare(): assert nbd.Extent(4096, 0) == nbd.Extent(4096, 0) assert nbd.Extent(4096, 0) != nbd.Extent(4096, nbd.STATE_ZERO)
def test_bitmap_backing_chain(nbd_env): vol1 = create_volume(nbd_env, "raw", "sparse") # Write first cluster to vol1 - this cluster is not recorded in any bitmap. qemuio.write_pattern(vol1.volumePath, "raw", offset=1 * MiB, len=64 * KiB, pattern=0xf1) # Simulate a snapshot - bitmap1 is created empty in vol2. vol2 = create_volume(nbd_env, "qcow2", "sparse", parent=vol1) bitmap1 = str(uuid.uuid4()) qemuimg.bitmap_add(vol2.volumePath, bitmap1).run() # Write second cluster in vol2 - this cluster is recorded only in bitmap 1. qemuio.write_pattern(vol2.volumePath, "qcow2", offset=2 * MiB, len=64 * KiB, pattern=0xf2) # Simulate another snapshot - bitmap1 is created empty in vol3. vol3 = create_volume(nbd_env, "qcow2", "sparse", parent=vol2) qemuimg.bitmap_add(vol3.volumePath, bitmap1).run() # Simulate backup - bitmap 2 is created in vol3. bitmap2 = str(uuid.uuid4()) qemuimg.bitmap_add(vol3.volumePath, bitmap2).run() # Write third cluster in vol3. This cluster is recorded in both bitmaps. qemuio.write_pattern(vol3.volumePath, "qcow2", offset=3 * MiB, len=64 * KiB, pattern=0xf3) # Test bitmap 1 - recording changes since bitmap 1 was added. config = { "sd_id": vol3.sdUUID, "img_id": vol3.imgUUID, "vol_id": vol3.volUUID, "readonly": True, "bitmap": bitmap1, } with nbd_server(config) as nbd_url: with nbd_client.open(urlparse(nbd_url), dirty=True) as c: extents = c.extents(0, nbd_env.virtual_size) assert extents[c.dirty_bitmap] == [ nbd_client.Extent(2 * MiB, 0), nbd_client.Extent(64 * KiB, 1), nbd_client.Extent(1 * MiB - 64 * KiB, 0), nbd_client.Extent(64 * KiB, 1), nbd_client.Extent(nbd_env.virtual_size - 3 * MiB - 64 * KiB, 0), ] assert c.read(1 * MiB, 64 * KiB) == b"\xf1" * 64 * KiB assert c.read(2 * MiB, 64 * KiB) == b"\xf2" * 64 * KiB # Test bitmap 2 - recording changes since bitmap 2 was added. config = { "sd_id": vol3.sdUUID, "img_id": vol3.imgUUID, "vol_id": vol3.volUUID, "readonly": True, "bitmap": bitmap2, } with nbd_server(config) as nbd_url: with nbd_client.open(urlparse(nbd_url), dirty=True) as c: extents = c.extents(0, nbd_env.virtual_size) assert extents[c.dirty_bitmap] == [ nbd_client.Extent(3 * MiB, 0), nbd_client.Extent(64 * KiB, 1), nbd_client.Extent(nbd_env.virtual_size - 3 * MiB - 64 * KiB, 0), ] assert c.read(2 * MiB, 64 * KiB) == b"\xf2" * 64 * KiB