def test_merge_simple():
n = GiB
a = [nbd.Extent(n, 0)]
b = [nbd.Extent(n, 0)]
merged = list(nbdutil.merged(a, b))
assert merged == a
def test_base_allocation_some_data(nbd_server, user_file, fmt, zero_flags):
size = 1024**3
create_image(user_file.path, fmt, size)
nbd_server.image = user_file.path
nbd_server.fmt = fmt
nbd_server.start()
# Use qcow2 cluster size to avoid inconsistent results on CentOS and
# Fedora.
data_length = 64 * 1024
zero_length = size // 2 - data_length
with nbd.open(nbd_server.url) as c:
# Create 4 extents: data, zero, data, zero.
c.write(0, b"x" * data_length)
c.write(size // 2, b"x" * data_length)
extents = list(nbdutil.extents(c))
assert extents == [
nbd.Extent(length=data_length, flags=0),
nbd.Extent(length=zero_length, flags=zero_flags),
nbd.Extent(length=data_length, flags=0),
nbd.Extent(length=zero_length, flags=zero_flags),
]
def test_dirty_bitmap(tmpdir):
size = 1024**2
# Create image with empty bitmap.
img = str(tmpdir.join("img.qcow2"))
qemu_img.create(img, "qcow2", size=size)
qemu_img.bitmap_add(img, "b0")
# Write data to image, modifying the bitmap.
with qemu_nbd.open(img, "qcow2") as c:
# This will allocate one cluster. By default bitmap granularity is also
# one cluster, so this will make the first extent dirty.
c.write(0, b"a")
c.flush()
# Read dirty extents.
with qemu_nbd.open(img, "qcow2", read_only=True, bitmap="b0") as c:
extents = c.extents(0, size)[nbd.QEMU_DIRTY_BITMAP + "b0"]
bitmap = qemu_img.info(img)["format-specific"]["data"]["bitmaps"][0]
assert extents == [
nbd.Extent(length=bitmap["granularity"], flags=nbd.EXTENT_DIRTY),
nbd.Extent(length=size - bitmap["granularity"], flags=0),
]
def test_extents_offset(max_extents):
n = GiB
c = fake_client(n, max_extents=max_extents)
extents = list(nbdutil.extents(c, offset=3 * n))
assert extents == [
nbd.Extent(1 * n, STATE_ZERO | STATE_HOLE),
nbd.Extent(2 * n, STATE_ZERO | STATE_HOLE | EXTENT_BACKING),
]
def test_extents_offset_dirty(max_extents):
n = GiB
c = fake_client(n, max_extents=max_extents)
extents = list(nbdutil.extents(c, offset=3 * n, dirty=True))
assert extents == [
nbd.Extent(1 * n, STATE_ZERO | STATE_HOLE | EXTENT_DIRTY),
nbd.Extent(2 * n, STATE_ZERO | STATE_HOLE),
]
def test_extents_length(max_extents):
n = GiB
c = fake_client(n, max_extents=max_extents)
extents = list(nbdutil.extents(c, length=3 * n))
assert extents == [
nbd.Extent(2 * n, 0),
nbd.Extent(1 * n, STATE_ZERO | STATE_HOLE),
]
def test_extent_dirty_bitmap():
# Clean area.
ext = nbd.Extent(4096, 0)
assert not ext.dirty
assert ext.flags == 0
# Dirty area.
ext = nbd.Extent(4096, nbd.STATE_DIRTY)
assert ext.dirty
assert ext.flags == nbd.STATE_DIRTY
def test_fake_client_max_extents():
n = MiB
c = fake_client(n, max_extents=1)
res = c.extents(0, 6 * n)
assert res == {
nbd.BASE_ALLOCATION: [
nbd.Extent(2 * n, 0),
],
nbd.QEMU_ALLOCATION_DEPTH: [
nbd.Extent(3 * n, 0), # depth=1
],
c.dirty_bitmap: [
nbd.Extent(1 * n, 0),
],
}
c = fake_client(n, max_extents=2)
res = c.extents(n, 4 * n)
assert res == {
nbd.BASE_ALLOCATION: [
nbd.Extent(1 * n, 0),
nbd.Extent(2 * n, STATE_ZERO | STATE_HOLE),
],
nbd.QEMU_ALLOCATION_DEPTH: [
nbd.Extent(2 * n, 0), # depth=1
nbd.Extent(1 * n, 0), # depth=2
],
c.dirty_bitmap: [
nbd.Extent(3 * n, EXTENT_DIRTY),
nbd.Extent(1 * n, 0)
],
}
def fake_client(n, max_extents=0):
"""
A client simulating few interesting cases:
- 3 alloction types: data, zero cluster, and unallocated extent.
- dirty extents convering both data and zero cluster.
- extents of different meta context of different length.
- server returning short reply.
"""
return FakeClient(
alloc=[
nbd.Extent(2 * n, 0),
nbd.Extent(2 * n, STATE_ZERO | STATE_HOLE),
nbd.Extent(2 * n, STATE_ZERO | STATE_HOLE),
],
depth=[
nbd.Extent(3 * n, 0), # depth=1
nbd.Extent(1 * n, 0), # depth=2
nbd.Extent(2 * n, EXTENT_BACKING), # depth=0
],
dirty=[
nbd.Extent(1 * n, 0),
nbd.Extent(3 * n, EXTENT_DIRTY),
nbd.Extent(2 * n, 0)
],
max_extents=max_extents,
)
def test_extents_all_no_depth(max_extents):
n = GiB
c = fake_client(n, max_extents=max_extents)
# Simulate the case when server does not report allocation depth.
c.depth = None
extents = list(nbdutil.extents(c))
assert extents == [
nbd.Extent(2 * n, 0),
nbd.Extent(4 * n, STATE_ZERO | STATE_HOLE),
]
def test_extents_last_extent_excceeds_export_size():
n = GiB
c = fake_client(n)
# Clip export size so we get extra extent info exceeding the request
# length.
c.export_size -= GiB
# Merge base:allocation and qemu:allocation-depth.
extents = list(nbdutil.extents(c))
assert extents == [
nbd.Extent(2 * n, 0),
nbd.Extent(2 * n, STATE_ZERO | STATE_HOLE),
nbd.Extent(1 * n, STATE_ZERO | STATE_HOLE | EXTENT_BACKING),
]
def test_extents_last_extent_excceeds_export_size_dirty():
n = GiB
c = fake_client(n)
# Clip export size so we get extra extent info exceeding the request
# length.
c.export_size -= GiB
extents = list(nbdutil.extents(c, dirty=True))
assert extents == [
nbd.Extent(1 * n, 0),
nbd.Extent(1 * n, EXTENT_DIRTY),
nbd.Extent(2 * n, STATE_ZERO | STATE_HOLE | EXTENT_DIRTY),
nbd.Extent(1 * n, STATE_ZERO | STATE_HOLE),
]
def test_fake_client_clip_both():
n = MiB
c = fake_client(n)
res = c.extents(2 * n, 2 * n)
assert res == {
nbd.BASE_ALLOCATION: [
nbd.Extent(2 * n, STATE_ZERO | STATE_HOLE),
],
nbd.QEMU_ALLOCATION_DEPTH: [
nbd.Extent(1 * n, 0), # depth=1
nbd.Extent(1 * n, 0), # depth=2
],
c.dirty_bitmap: [
nbd.Extent(2 * n, EXTENT_DIRTY),
],
}
def test_extent_base_allocation():
# Allocated aread with data.
ext = nbd.Extent(4096, 0)
assert not ext.zero
assert not ext.hole
assert ext.flags == 0
# Allocated aread that reads as zero.
ext = nbd.Extent(4096, nbd.STATE_ZERO)
assert ext.zero
assert not ext.hole
assert ext.flags == nbd.STATE_ZERO
# Unallocated aread that reads as zero.
ext = nbd.Extent(4096, nbd.STATE_ZERO | nbd.STATE_HOLE)
assert ext.zero
assert ext.hole
assert ext.flags == nbd.STATE_ZERO | nbd.STATE_HOLE
def test_detect_zeroes_disabled(tmpdir, fmt, detect_zeroes):
size = 1024**2
disk = str(tmpdir.join("disk." + fmt))
qemu_img.create(disk, fmt, size=size)
with qemu_nbd.open(disk, fmt, detect_zeroes=detect_zeroes) as c:
# These zeroes should not be detected.
c.write(0, b"\0" * size)
c.flush()
extents = c.extents(0, size)
assert extents["base:allocation"] == [
nbd.Extent(length=1048576, flags=0),
]
if fmt != "raw":
assert extents["qemu:allocation-depth"] == [
nbd.Extent(length=1048576, flags=0),
]
def test_extent_merge_qcow2_unallocated_cluster():
alloc_data = nbd.Extent.pack(65536, nbd.STATE_ZERO | nbd.STATE_HOLE)
alloc = nbd.Extent.unpack(alloc_data)
depth_data = nbd.Extent.pack(65536, 0)
depth = nbd.Extent.unpack(depth_data, nbd.Extent.DEPTH)
merged = nbd.Extent(65536, alloc.flags | depth.flags)
assert merged.length == 65536
assert merged.zero is True
assert merged.hole is True
def test_extent_merge_qcow2_data_cluster():
alloc_data = nbd.Extent.pack(65536, 0)
alloc = nbd.Extent.unpack(alloc_data)
depth_data = nbd.Extent.pack(65536, 1)
depth = nbd.Extent.unpack(depth_data, nbd.Extent.DEPTH)
merged = nbd.Extent(65536, alloc.flags | depth.flags)
assert merged.length == 65536
assert merged.zero is False
assert merged.hole is False
def test_extent_merge_dirty_data():
alloc_data = nbd.Extent.pack(4096, 0)
alloc = nbd.Extent.unpack(alloc_data, nbd.Extent.DIRTY)
dirty_data = nbd.Extent.pack(4096, nbd.STATE_DIRTY)
dirty = nbd.Extent.unpack(dirty_data, nbd.Extent.DIRTY)
merged = nbd.Extent(4096, alloc.flags | dirty.flags)
assert merged.length == 4096
assert not merged.zero
assert not merged.hole
assert merged.dirty
def test_extent_merge_clean_hole():
alloc_data = nbd.Extent.pack(4096, nbd.STATE_ZERO | nbd.STATE_HOLE)
alloc = nbd.Extent.unpack(alloc_data)
dirty_data = nbd.Extent.pack(4096, 0)
dirty = nbd.Extent.unpack(dirty_data, nbd.Extent.DIRTY)
merged = nbd.Extent(4096, alloc.flags | dirty.flags)
assert merged.length == 4096
assert merged.zero is True
# Hole can be detected only when using depth data with qcow2 image.
assert merged.hole is False
assert not merged.dirty
def lookup(self, offset, length, extents):
end = offset + length
start = 0
count = 0
for e in extents:
# Skip before the requested range:
# request: [ ]
# extent: |----|
if start + e.length <= offset:
start += e.length
continue
length = e.length
# Clip extent before offset:
# request: [ ]
# extent: |-------|
# result: |===|
if start < offset:
clip = offset - start
length -= clip
start += clip
# Clip extent after end:
# request: [ ]
# extent: |-------|
# result: |====|
if start + length > end:
clip = start + length - end
length -= clip
yield nbd.Extent(length, e.flags)
# NBD server is allowed to return short reply with one or more
# extents.
count += 1
if self.max_extents and count == self.max_extents:
break
start += length
# Stop lookup after the requested range:
# request: [ ]
# extent: |----|
if start >= end:
break
def test_merge_split_both():
n = GiB
a = [
nbd.Extent(n * 1, 1),
nbd.Extent(n * 2, 2),
]
b = [
nbd.Extent(n * 2, 4),
nbd.Extent(n * 1, 8),
]
merged1 = list(nbdutil.merged(a, b))
assert merged1 == [
nbd.Extent(n, 1 | 4),
nbd.Extent(n, 2 | 4),
nbd.Extent(n, 2 | 8),
]
merged2 = list(nbdutil.merged(b, a))
assert merged2 == merged1
def test_merge_split_one():
n = GiB
a = [
nbd.Extent(n, 1),
nbd.Extent(n, 2),
nbd.Extent(n, 4),
]
b = [
nbd.Extent(n * 3, 8)
]
merged1 = list(nbdutil.merged(a, b))
assert merged1 == [
nbd.Extent(n, 1 | 8),
nbd.Extent(n, 2 | 8),
nbd.Extent(n, 4 | 8),
]
merged2 = list(nbdutil.merged(b, a))
assert merged2 == merged1
def test_merge_clip():
n = GiB
a = [
nbd.Extent(n * 1, 1),
nbd.Extent(n * 1, 2),
]
b = [
nbd.Extent(n * 1, 4),
nbd.Extent(n * 2, 8),
]
merged1 = list(nbdutil.merged(a, b))
assert merged1 == [
nbd.Extent(n * 1, 1 | 4),
nbd.Extent(n * 1, 2 | 8),
]
merged2 = list(nbdutil.merged(b, a))
assert merged2 == merged1
def test_base_allocation_full(nbd_server, user_file, fmt):
size = 1024**2
create_image(user_file.path, fmt, size)
nbd_server.image = user_file.path
nbd_server.fmt = fmt
nbd_server.start()
with nbd.open(nbd_server.url) as c:
c.write(0, b"x" * size)
# Entire image.
extents = c.extents(0, size)["base:allocation"]
assert extents == [nbd.Extent(length=size, flags=0)]
# First block.
extents = c.extents(0, 4096)["base:allocation"]
assert extents == [nbd.Extent(length=4096, flags=0)]
# Last block.
extents = c.extents(size - 4096, 4096)["base:allocation"]
assert extents == [nbd.Extent(length=4096, flags=0)]
# Some block.
extents = c.extents(4096, 4096)["base:allocation"]
assert extents == [nbd.Extent(length=4096, flags=0)]
# Unaligned start.
extents = c.extents(4096 - 1, 4096 + 1)["base:allocation"]
assert extents == [nbd.Extent(length=4096 + 1, flags=0)]
# Unaligned end.
extents = c.extents(4096, 4096 + 1)["base:allocation"]
assert extents == [nbd.Extent(length=4096 + 1, flags=0)]
# Unaligned start and end.
extents = c.extents(4096 - 1, 4096 + 2)["base:allocation"]
assert extents == [nbd.Extent(length=4096 + 2, flags=0)]
def test_base_allocation_empty(nbd_server, user_file, fmt, hole_flags):
size = 1024**3
create_image(user_file.path, fmt, size)
nbd_server.image = user_file.path
nbd_server.fmt = fmt
nbd_server.start()
with nbd.open(nbd_server.url) as c:
# Entire image.
extents = list(nbdutil.extents(c))
assert extents == [nbd.Extent(length=size, flags=hole_flags)]
# First block.
extents = list(nbdutil.extents(c, length=4096))
assert extents == [nbd.Extent(length=4096, flags=hole_flags)]
# Last block.
extents = list(nbdutil.extents(c, offset=size - 4096, length=4096))
assert extents == [nbd.Extent(length=4096, flags=hole_flags)]
# Some block.
extents = list(nbdutil.extents(c, offset=4096, length=4096))
assert extents == [nbd.Extent(length=4096, flags=hole_flags)]
# Unaligned start.
extents = list(nbdutil.extents(c, offset=4096 - 1, length=4096 + 1))
assert extents == [nbd.Extent(length=4096 + 1, flags=hole_flags)]
# Unaligned end.
extents = list(nbdutil.extents(c, offset=4096, length=4096 + 1))
assert extents == [nbd.Extent(length=4096 + 1, flags=hole_flags)]
# Unaligned start and end.
extents = list(nbdutil.extents(c, offset=4096 - 1, length=4096 + 2))
assert extents == [nbd.Extent(length=4096 + 2, flags=hole_flags)]
def test_base_allocation_empty(nbd_server, user_file, fmt, zero_flags):
size = nbd.MAX_LENGTH
create_image(user_file.path, fmt, size)
nbd_server.image = user_file.path
nbd_server.fmt = fmt
nbd_server.start()
with nbd.open(nbd_server.url) as c:
# Entire image.
extents = c.extents(0, size)["base:allocation"]
assert extents == [nbd.Extent(length=size, flags=zero_flags)]
# First block.
extents = c.extents(0, 4096)["base:allocation"]
assert extents == [nbd.Extent(length=4096, flags=zero_flags)]
# Last block.
extents = c.extents(size - 4096, 4096)["base:allocation"]
assert extents == [nbd.Extent(length=4096, flags=zero_flags)]
# Some block.
extents = c.extents(4096, 4096)["base:allocation"]
assert extents == [nbd.Extent(length=4096, flags=zero_flags)]
# Unaligned start.
extents = c.extents(4096 - 1, 4096 + 1)["base:allocation"]
assert extents == [nbd.Extent(length=4096 + 1, flags=zero_flags)]
# Unaligned end.
extents = c.extents(4096, 4096 + 1)["base:allocation"]
assert extents == [nbd.Extent(length=4096 + 1, flags=zero_flags)]
# Unaligned start and end.
extents = c.extents(4096 - 1, 4096 + 2)["base:allocation"]
assert extents == [nbd.Extent(length=4096 + 2, flags=zero_flags)]
def test_base_allocation_some_data_unaligned(nbd_server, user_file, fmt,
zero_flags):
size = 1024**2
create_image(user_file.path, fmt, size)
nbd_server.image = user_file.path
nbd_server.fmt = fmt
nbd_server.start()
data_length = 64 * 1024
data_offset = 2 * data_length
with nbd.open(nbd_server.url) as c:
# Create 3 extents: zero, data, zero.
c.write(data_offset, b"x" * data_length)
# Unaligned part from first extent and last extent.
extents = list(nbdutil.extents(c, data_offset - 1, data_length + 2))
assert extents == [
nbd.Extent(length=1, flags=zero_flags),
nbd.Extent(length=data_length, flags=0),
nbd.Extent(length=1, flags=zero_flags),
]
# Unaligned part from second extent.
extents = list(nbdutil.extents(c, data_offset + 1, data_length - 2))
assert extents == [
nbd.Extent(length=data_length - 2, flags=0),
]
# Unaligned part from second and last extents.
extents = list(nbdutil.extents(c, data_offset + 1, data_length))
assert extents == [
nbd.Extent(length=data_length - 1, flags=0),
nbd.Extent(length=1, flags=zero_flags),
]
def test_bitmap_single_volume(nbd_env):
vol = create_volume(nbd_env, "qcow2", "sparse")
# Write first cluster - this cluster is not recorded in any bitmap.
qemuio.write_pattern(vol.volumePath,
"qcow2",
offset=1 * MiB,
len=64 * KiB,
pattern=0xf1)
# Add bitmap 1 and write second cluster.
bitmap1 = str(uuid.uuid4())
qemuimg.bitmap_add(vol.volumePath, bitmap1).run()
qemuio.write_pattern(vol.volumePath,
"qcow2",
offset=2 * MiB,
len=64 * KiB,
pattern=0xf2)
# Add bitmap 2 and write third cluster.
bitmap2 = str(uuid.uuid4())
qemuimg.bitmap_add(vol.volumePath, bitmap2).run()
qemuio.write_pattern(vol.volumePath,
"qcow2",
offset=3 * MiB,
len=64 * KiB,
pattern=0xf3)
# Test bitmap 1 - recording changes since bitmap 1 was added.
config = {
"sd_id": vol.sdUUID,
"img_id": vol.imgUUID,
"vol_id": vol.volUUID,
"readonly": True,
"bitmap": bitmap1,
}
with nbd_server(config) as nbd_url:
with nbd_client.open(urlparse(nbd_url), dirty=True) as c:
extents = c.extents(0, nbd_env.virtual_size)
assert extents[c.dirty_bitmap] == [
nbd_client.Extent(2 * MiB, 0),
nbd_client.Extent(64 * KiB, 1),
nbd_client.Extent(1 * MiB - 64 * KiB, 0),
nbd_client.Extent(64 * KiB, 1),
nbd_client.Extent(nbd_env.virtual_size - 3 * MiB - 64 * KiB,
0),
]
assert c.read(1 * MiB, 64 * KiB) == b"\xf1" * 64 * KiB
assert c.read(2 * MiB, 64 * KiB) == b"\xf2" * 64 * KiB
# Test bitmap 2 - recording changes since bitmap 2 was added.
config = {
"sd_id": vol.sdUUID,
"img_id": vol.imgUUID,
"vol_id": vol.volUUID,
"readonly": True,
"bitmap": bitmap2,
}
with nbd_server(config) as nbd_url:
with nbd_client.open(urlparse(nbd_url), dirty=True) as c:
extents = c.extents(0, nbd_env.virtual_size)
assert extents[c.dirty_bitmap] == [
nbd_client.Extent(3 * MiB, 0),
nbd_client.Extent(64 * KiB, 1),
nbd_client.Extent(nbd_env.virtual_size - 3 * MiB - 64 * KiB,
0),
]
assert c.read(2 * MiB, 64 * KiB) == b"\xf2" * 64 * KiB
def test_extent_compare():
assert nbd.Extent(4096, 0) == nbd.Extent(4096, 0)
assert nbd.Extent(4096, 0) != nbd.Extent(4096, nbd.STATE_ZERO)
def test_bitmap_backing_chain(nbd_env):
vol1 = create_volume(nbd_env, "raw", "sparse")
# Write first cluster to vol1 - this cluster is not recorded in any bitmap.
qemuio.write_pattern(vol1.volumePath,
"raw",
offset=1 * MiB,
len=64 * KiB,
pattern=0xf1)
# Simulate a snapshot - bitmap1 is created empty in vol2.
vol2 = create_volume(nbd_env, "qcow2", "sparse", parent=vol1)
bitmap1 = str(uuid.uuid4())
qemuimg.bitmap_add(vol2.volumePath, bitmap1).run()
# Write second cluster in vol2 - this cluster is recorded only in bitmap 1.
qemuio.write_pattern(vol2.volumePath,
"qcow2",
offset=2 * MiB,
len=64 * KiB,
pattern=0xf2)
# Simulate another snapshot - bitmap1 is created empty in vol3.
vol3 = create_volume(nbd_env, "qcow2", "sparse", parent=vol2)
qemuimg.bitmap_add(vol3.volumePath, bitmap1).run()
# Simulate backup - bitmap 2 is created in vol3.
bitmap2 = str(uuid.uuid4())
qemuimg.bitmap_add(vol3.volumePath, bitmap2).run()
# Write third cluster in vol3. This cluster is recorded in both bitmaps.
qemuio.write_pattern(vol3.volumePath,
"qcow2",
offset=3 * MiB,
len=64 * KiB,
pattern=0xf3)
# Test bitmap 1 - recording changes since bitmap 1 was added.
config = {
"sd_id": vol3.sdUUID,
"img_id": vol3.imgUUID,
"vol_id": vol3.volUUID,
"readonly": True,
"bitmap": bitmap1,
}
with nbd_server(config) as nbd_url:
with nbd_client.open(urlparse(nbd_url), dirty=True) as c:
extents = c.extents(0, nbd_env.virtual_size)
assert extents[c.dirty_bitmap] == [
nbd_client.Extent(2 * MiB, 0),
nbd_client.Extent(64 * KiB, 1),
nbd_client.Extent(1 * MiB - 64 * KiB, 0),
nbd_client.Extent(64 * KiB, 1),
nbd_client.Extent(nbd_env.virtual_size - 3 * MiB - 64 * KiB,
0),
]
assert c.read(1 * MiB, 64 * KiB) == b"\xf1" * 64 * KiB
assert c.read(2 * MiB, 64 * KiB) == b"\xf2" * 64 * KiB
# Test bitmap 2 - recording changes since bitmap 2 was added.
config = {
"sd_id": vol3.sdUUID,
"img_id": vol3.imgUUID,
"vol_id": vol3.volUUID,
"readonly": True,
"bitmap": bitmap2,
}
with nbd_server(config) as nbd_url:
with nbd_client.open(urlparse(nbd_url), dirty=True) as c:
extents = c.extents(0, nbd_env.virtual_size)
assert extents[c.dirty_bitmap] == [
nbd_client.Extent(3 * MiB, 0),
nbd_client.Extent(64 * KiB, 1),
nbd_client.Extent(nbd_env.virtual_size - 3 * MiB - 64 * KiB,
0),
]
assert c.read(2 * MiB, 64 * KiB) == b"\xf2" * 64 * KiB
