def test_detect_zeroes_disabled(nbd_env, format):
vol = create_volume(nbd_env, format, "sparse")
config = {
"sd_id": vol.sdUUID,
"img_id": vol.imgUUID,
"vol_id": vol.volUUID,
}
with nbd_server(config) as nbd_url:
# Fill image with zeroes.
with nbd_client.open(urlparse(nbd_url)) as c:
c.write(0 * MiB, b"\0" * nbd_env.virtual_size)
c.flush()
extents = c.extents(0, nbd_env.virtual_size)
image_info = qemuimg.info(vol.volumePath)
if format == "raw":
assert image_info["actual-size"] == nbd_env.virtual_size
else:
assert image_info["actual-size"] >= nbd_env.virtual_size
log.debug("image extents: %s", extents)
# Entire image should be data extent.
base_alloc = [(e.length, e.zero) for e in extents["base:allocation"]]
assert base_alloc == [(nbd_env.virtual_size, False)]
if format != "raw":
# Entire image should be allocated.
alloc_depth = [(e.length, e.hole)
for e in extents["qemu:allocation-depth"]]
assert alloc_depth == [(nbd_env.virtual_size, False)]
def test_base_allocation_some_data(nbd_server, user_file, fmt, zero_flags):
size = 1024**3
create_image(user_file.path, fmt, size)
nbd_server.image = user_file.path
nbd_server.fmt = fmt
nbd_server.start()
# Use qcow2 cluster size to avoid inconsistent results on CentOS and
# Fedora.
data_length = 64 * 1024
zero_length = size // 2 - data_length
with nbd.open(nbd_server.url) as c:
# Create 4 extents: data, zero, data, zero.
c.write(0, b"x" * data_length)
c.write(size // 2, b"x" * data_length)
extents = list(nbdutil.extents(c))
assert extents == [
nbd.Extent(length=data_length, flags=0),
nbd.Extent(length=zero_length, flags=zero_flags),
nbd.Extent(length=data_length, flags=0),
nbd.Extent(length=zero_length, flags=zero_flags),
]
def test_base_allocation_many_extents(nbd_server, user_file):
# Tested only with raw since qcow2 minimal extent is cluster size (64K),
# and writing 1000 extents (62.5 MiB) will be too slow in the CI.
# Extents must be multiple of file system block size.
extent_length = os.statvfs(user_file.path).f_bsize
# Use number which is not a multiple of our buffer capacity (1024 extents)
# to ensure we read partial buffers correctly.
extents_count = 2000
size = extents_count * extent_length
create_image(user_file.path, "raw", size)
nbd_server.image = user_file.path
nbd_server.fmt = "raw"
nbd_server.start()
with nbd.open(nbd_server.url) as c:
# Write data to all even extents.
data = b"x" * extent_length
for i in range(0, size, extent_length * 2):
c.write(i, data)
extents = list(nbdutil.extents(c))
assert len(extents) == extents_count
for i, ext in enumerate(extents):
assert ext.length == extent_length
assert ext.zero == bool(i % 2)
def copy_disk(nbd_url, backup_disk):
log.info("Backing up data extents from %s to %s", nbd_url, backup_disk)
backup_url = urlparse(nbd_url)
with nbd.open(backup_url) as src_client, \
qemu_nbd.open(backup_disk, "qcow2") as dst_client:
nbdutil.copy(src_client, dst_client)
def test_open_tcp(tmpdir, url_template, host, export):
image = str(tmpdir.join("image"))
with open(image, "wb") as f:
f.truncate(1024**3)
sock = nbd.TCPAddress(host, testutil.random_tcp_port())
url = url_template.format(port=sock.port)
log.debug("Trying url=%r export=%r", url, export)
with qemu_nbd.run(image, "raw", sock, export_name=export):
with nbd.open(urlparse(url)) as c:
assert c.export_size == 1024**3
def test_open_unix(tmpdir, url, export):
image = str(tmpdir.join("image"))
with open(image, "wb") as f:
f.truncate(1024**3)
sock = nbd.UnixAddress(tmpdir.join("sock"))
url = url.replace("/path", sock)
log.debug("Trying url=%r export=%r", url, export)
with qemu_nbd.run(image, "raw", sock, export_name=export):
with nbd.open(urlparse(url)) as c:
assert c.export_size == 1024**3
def test_raw_readinto(nbd_server):
offset = 1024**2
data = b"can read from raw"
with io.open(nbd_server.image, "wb") as f:
f.truncate(2 * 1024**2)
f.seek(1024**2)
f.write(data)
nbd_server.start()
with nbd.open(nbd_server.url) as c:
buf = bytearray(len(data))
c.readinto(offset, buf)
assert buf == data
def test_allocation_depth(nbd_env, format, backing_chain):
# Check that qemu-nbd exposes the "qemu:allocation-depth" meta
# context for qcow2 format.
vol = create_volume(nbd_env, format, "sparse")
config = {
"sd_id": vol.sdUUID,
"img_id": vol.imgUUID,
"vol_id": vol.volUUID,
"readonly": True,
"backing_chain": backing_chain,
}
with nbd_server(config) as nbd_url:
with nbd_client.open(urlparse(nbd_url)) as c:
extents = c.extents(0, nbd_env.virtual_size)
log.debug("extents: %s", extents)
assert ("qemu:allocation-depth" in extents) == (format != "raw")
def test_extents_reply_error(nbd_server, user_file):
"""
The server SHOULD return NBD_EINVAL if it receives a NBD_CMD_BLOCK_STATUS
request including one or more sectors beyond the size of the device.
"""
size = 1024**2
create_image(user_file.path, "raw", size)
nbd_server.image = user_file.path
nbd_server.fmt = "raw"
nbd_server.start()
with nbd.open(nbd_server.url) as c:
with pytest.raises(nbd.ReplyError) as e:
c.extents(0, size + 1)
# Server should return this, qemu does.
assert e.value.code == errno.EINVAL
# The next request should succeed.
assert c.read(4096, 1) == b"\0"
def test_detect_zeroes_discard(nbd_env, format):
vol = create_volume(nbd_env, format, "sparse")
config = {
"sd_id": vol.sdUUID,
"img_id": vol.imgUUID,
"vol_id": vol.volUUID,
"discard": True,
"detect_zeroes": True,
}
with nbd_server(config) as nbd_url:
# Fill image with zeroes. The writes should be converted to write
# zeroes command.
with nbd_client.open(urlparse(nbd_url)) as c:
c.write(0 * MiB, b"\0" * nbd_env.virtual_size)
c.flush()
extents = c.extents(0, nbd_env.virtual_size)
image_info = qemuimg.info(vol.volumePath)
if format == "raw":
assert image_info["actual-size"] == 0
else:
# Image contains only metadata.
assert image_info["actual-size"] <= MiB
log.debug("image extents: %s", extents)
# Entire image should be zero extent. For qcow2 image, qemu-nbd writes zero
# clusters. For raw image, qemu-nbd deallocates the entire image.
base_alloc = [(e.length, e.zero) for e in extents["base:allocation"]]
assert base_alloc == [(nbd_env.virtual_size, True)]
if format != "raw":
# Entire image should be allocated. For qcow2 image, zero
# clusters are always allocated.
alloc_depth = [(e.length, e.hole)
for e in extents["qemu:allocation-depth"]]
assert alloc_depth == [(nbd_env.virtual_size, False)]
def copy_dirty(nbd_url, backup_disk):
log.info("Backing up dirty extents from %s to %s", nbd_url, backup_disk)
backup_url = urlparse(nbd_url)
with nbd.open(backup_url, dirty=True) as src_client, \
qemu_nbd.open(backup_disk, "qcow2") as dst_client:
buf = bytearray(4 * 1024**2)
offset = 0
for ext in nbdutil.extents(src_client, dirty=True):
if ext.dirty:
todo = ext.length
while todo:
step = min(todo, len(buf))
view = memoryview(buf)[:step]
src_client.readinto(offset, view)
dst_client.write(offset, view)
offset += step
todo -= step
else:
offset += ext.length
def test_base_allocation_full(nbd_server, user_file, fmt):
size = 1024**2
create_image(user_file.path, fmt, size)
nbd_server.image = user_file.path
nbd_server.fmt = fmt
nbd_server.start()
with nbd.open(nbd_server.url) as c:
c.write(0, b"x" * size)
# Entire image.
extents = c.extents(0, size)["base:allocation"]
assert extents == [nbd.Extent(length=size, flags=0)]
# First block.
extents = c.extents(0, 4096)["base:allocation"]
assert extents == [nbd.Extent(length=4096, flags=0)]
# Last block.
extents = c.extents(size - 4096, 4096)["base:allocation"]
assert extents == [nbd.Extent(length=4096, flags=0)]
# Some block.
extents = c.extents(4096, 4096)["base:allocation"]
assert extents == [nbd.Extent(length=4096, flags=0)]
# Unaligned start.
extents = c.extents(4096 - 1, 4096 + 1)["base:allocation"]
assert extents == [nbd.Extent(length=4096 + 1, flags=0)]
# Unaligned end.
extents = c.extents(4096, 4096 + 1)["base:allocation"]
assert extents == [nbd.Extent(length=4096 + 1, flags=0)]
# Unaligned start and end.
extents = c.extents(4096 - 1, 4096 + 2)["base:allocation"]
assert extents == [nbd.Extent(length=4096 + 2, flags=0)]
def test_detect_zeroes_no_discard(nbd_env, format):
vol = create_volume(nbd_env, format, "sparse")
config = {
"sd_id": vol.sdUUID,
"img_id": vol.imgUUID,
"vol_id": vol.volUUID,
"discard": False,
"detect_zeroes": True,
}
with nbd_server(config) as nbd_url:
# Fill image with zeroes. The writes should be converted to write
# zeroes command.
with nbd_client.open(urlparse(nbd_url)) as c:
c.write(0 * MiB, b"\0" * nbd_env.virtual_size)
c.flush()
extents = c.extents(0, nbd_env.virtual_size)
# For qcow2 format, in qemu-nd < 6.2 actual zeroes written to image.
# In 6.2 zeroes seem to be converted to zero clusters. Since the
# behavior is not consistent we don't have easy way to verify.
if format == "raw":
image_info = qemuimg.info(vol.volumePath)
assert image_info["actual-size"] == nbd_env.virtual_size
log.debug("image extents: %s", extents)
# Entire image should be zero extent.
base_alloc = [(e.length, e.zero) for e in extents["base:allocation"]]
assert base_alloc == [(nbd_env.virtual_size, True)]
if format != "raw":
# qemu-nbd allocated entire image in both cases.
alloc_depth = [(e.length, e.hole)
for e in extents["qemu:allocation-depth"]]
assert alloc_depth == [(nbd_env.virtual_size, False)]
def test_base_allocation_empty(nbd_server, user_file, fmt, hole_flags):
size = 1024**3
create_image(user_file.path, fmt, size)
nbd_server.image = user_file.path
nbd_server.fmt = fmt
nbd_server.start()
with nbd.open(nbd_server.url) as c:
# Entire image.
extents = list(nbdutil.extents(c))
assert extents == [nbd.Extent(length=size, flags=hole_flags)]
# First block.
extents = list(nbdutil.extents(c, length=4096))
assert extents == [nbd.Extent(length=4096, flags=hole_flags)]
# Last block.
extents = list(nbdutil.extents(c, offset=size - 4096, length=4096))
assert extents == [nbd.Extent(length=4096, flags=hole_flags)]
# Some block.
extents = list(nbdutil.extents(c, offset=4096, length=4096))
assert extents == [nbd.Extent(length=4096, flags=hole_flags)]
# Unaligned start.
extents = list(nbdutil.extents(c, offset=4096 - 1, length=4096 + 1))
assert extents == [nbd.Extent(length=4096 + 1, flags=hole_flags)]
# Unaligned end.
extents = list(nbdutil.extents(c, offset=4096, length=4096 + 1))
assert extents == [nbd.Extent(length=4096 + 1, flags=hole_flags)]
# Unaligned start and end.
extents = list(nbdutil.extents(c, offset=4096 - 1, length=4096 + 2))
assert extents == [nbd.Extent(length=4096 + 2, flags=hole_flags)]
def test_base_allocation_some_data_unaligned(nbd_server, user_file, fmt,
zero_flags):
size = 1024**2
create_image(user_file.path, fmt, size)
nbd_server.image = user_file.path
nbd_server.fmt = fmt
nbd_server.start()
data_length = 64 * 1024
data_offset = 2 * data_length
with nbd.open(nbd_server.url) as c:
# Create 3 extents: zero, data, zero.
c.write(data_offset, b"x" * data_length)
# Unaligned part from first extent and last extent.
extents = list(nbdutil.extents(c, data_offset - 1, data_length + 2))
assert extents == [
nbd.Extent(length=1, flags=zero_flags),
nbd.Extent(length=data_length, flags=0),
nbd.Extent(length=1, flags=zero_flags),
]
# Unaligned part from second extent.
extents = list(nbdutil.extents(c, data_offset + 1, data_length - 2))
assert extents == [
nbd.Extent(length=data_length - 2, flags=0),
]
# Unaligned part from second and last extents.
extents = list(nbdutil.extents(c, data_offset + 1, data_length))
assert extents == [
nbd.Extent(length=data_length - 1, flags=0),
nbd.Extent(length=1, flags=zero_flags),
]
def test_base_allocation_empty(nbd_server, user_file, fmt, zero_flags):
size = nbd.MAX_LENGTH
create_image(user_file.path, fmt, size)
nbd_server.image = user_file.path
nbd_server.fmt = fmt
nbd_server.start()
with nbd.open(nbd_server.url) as c:
# Entire image.
extents = c.extents(0, size)["base:allocation"]
assert extents == [nbd.Extent(length=size, flags=zero_flags)]
# First block.
extents = c.extents(0, 4096)["base:allocation"]
assert extents == [nbd.Extent(length=4096, flags=zero_flags)]
# Last block.
extents = c.extents(size - 4096, 4096)["base:allocation"]
assert extents == [nbd.Extent(length=4096, flags=zero_flags)]
# Some block.
extents = c.extents(4096, 4096)["base:allocation"]
assert extents == [nbd.Extent(length=4096, flags=zero_flags)]
# Unaligned start.
extents = c.extents(4096 - 1, 4096 + 1)["base:allocation"]
assert extents == [nbd.Extent(length=4096 + 1, flags=zero_flags)]
# Unaligned end.
extents = c.extents(4096, 4096 + 1)["base:allocation"]
assert extents == [nbd.Extent(length=4096 + 1, flags=zero_flags)]
# Unaligned start and end.
extents = c.extents(4096 - 1, 4096 + 2)["base:allocation"]
assert extents == [nbd.Extent(length=4096 + 2, flags=zero_flags)]
def test_bitmap_backing_chain(nbd_env):
vol1 = create_volume(nbd_env, "raw", "sparse")
# Write first cluster to vol1 - this cluster is not recorded in any bitmap.
qemuio.write_pattern(vol1.volumePath,
"raw",
offset=1 * MiB,
len=64 * KiB,
pattern=0xf1)
# Simulate a snapshot - bitmap1 is created empty in vol2.
vol2 = create_volume(nbd_env, "qcow2", "sparse", parent=vol1)
bitmap1 = str(uuid.uuid4())
qemuimg.bitmap_add(vol2.volumePath, bitmap1).run()
# Write second cluster in vol2 - this cluster is recorded only in bitmap 1.
qemuio.write_pattern(vol2.volumePath,
"qcow2",
offset=2 * MiB,
len=64 * KiB,
pattern=0xf2)
# Simulate another snapshot - bitmap1 is created empty in vol3.
vol3 = create_volume(nbd_env, "qcow2", "sparse", parent=vol2)
qemuimg.bitmap_add(vol3.volumePath, bitmap1).run()
# Simulate backup - bitmap 2 is created in vol3.
bitmap2 = str(uuid.uuid4())
qemuimg.bitmap_add(vol3.volumePath, bitmap2).run()
# Write third cluster in vol3. This cluster is recorded in both bitmaps.
qemuio.write_pattern(vol3.volumePath,
"qcow2",
offset=3 * MiB,
len=64 * KiB,
pattern=0xf3)
# Test bitmap 1 - recording changes since bitmap 1 was added.
config = {
"sd_id": vol3.sdUUID,
"img_id": vol3.imgUUID,
"vol_id": vol3.volUUID,
"readonly": True,
"bitmap": bitmap1,
}
with nbd_server(config) as nbd_url:
with nbd_client.open(urlparse(nbd_url), dirty=True) as c:
extents = c.extents(0, nbd_env.virtual_size)
dirty_extents = [(e.length, e.dirty)
for e in extents[c.dirty_bitmap]]
assert dirty_extents == [
(2 * MiB, False),
(64 * KiB, True),
(1 * MiB - 64 * KiB, False),
(64 * KiB, True),
(nbd_env.virtual_size - 3 * MiB - 64 * KiB, False),
]
assert c.read(1 * MiB, 64 * KiB) == b"\xf1" * 64 * KiB
assert c.read(2 * MiB, 64 * KiB) == b"\xf2" * 64 * KiB
# Test bitmap 2 - recording changes since bitmap 2 was added.
config = {
"sd_id": vol3.sdUUID,
"img_id": vol3.imgUUID,
"vol_id": vol3.volUUID,
"readonly": True,
"bitmap": bitmap2,
}
with nbd_server(config) as nbd_url:
with nbd_client.open(urlparse(nbd_url), dirty=True) as c:
extents = c.extents(0, nbd_env.virtual_size)
dirty_extents = [(e.length, e.dirty)
for e in extents[c.dirty_bitmap]]
assert dirty_extents == [
(3 * MiB, False),
(64 * KiB, True),
(nbd_env.virtual_size - 3 * MiB - 64 * KiB, False),
]
assert c.read(2 * MiB, 64 * KiB) == b"\xf2" * 64 * KiB
def test_bitmap_single_volume(nbd_env):
vol = create_volume(nbd_env, "qcow2", "sparse")
# Write first cluster - this cluster is not recorded in any bitmap.
qemuio.write_pattern(vol.volumePath,
"qcow2",
offset=1 * MiB,
len=64 * KiB,
pattern=0xf1)
# Add bitmap 1 and write second cluster.
bitmap1 = str(uuid.uuid4())
qemuimg.bitmap_add(vol.volumePath, bitmap1).run()
qemuio.write_pattern(vol.volumePath,
"qcow2",
offset=2 * MiB,
len=64 * KiB,
pattern=0xf2)
# Add bitmap 2 and write third cluster.
bitmap2 = str(uuid.uuid4())
qemuimg.bitmap_add(vol.volumePath, bitmap2).run()
qemuio.write_pattern(vol.volumePath,
"qcow2",
offset=3 * MiB,
len=64 * KiB,
pattern=0xf3)
# Test bitmap 1 - recording changes since bitmap 1 was added.
config = {
"sd_id": vol.sdUUID,
"img_id": vol.imgUUID,
"vol_id": vol.volUUID,
"readonly": True,
"bitmap": bitmap1,
}
with nbd_server(config) as nbd_url:
with nbd_client.open(urlparse(nbd_url), dirty=True) as c:
extents = c.extents(0, nbd_env.virtual_size)
dirty_extents = [(e.length, e.dirty)
for e in extents[c.dirty_bitmap]]
assert dirty_extents == [
(2 * MiB, False),
(64 * KiB, True),
(1 * MiB - 64 * KiB, False),
(64 * KiB, True),
(nbd_env.virtual_size - 3 * MiB - 64 * KiB, False),
]
assert c.read(1 * MiB, 64 * KiB) == b"\xf1" * 64 * KiB
assert c.read(2 * MiB, 64 * KiB) == b"\xf2" * 64 * KiB
# Test bitmap 2 - recording changes since bitmap 2 was added.
config = {
"sd_id": vol.sdUUID,
"img_id": vol.imgUUID,
"vol_id": vol.volUUID,
"readonly": True,
"bitmap": bitmap2,
}
with nbd_server(config) as nbd_url:
with nbd_client.open(urlparse(nbd_url), dirty=True) as c:
extents = c.extents(0, nbd_env.virtual_size)
dirty_extents = [(e.length, e.dirty)
for e in extents[c.dirty_bitmap]]
assert dirty_extents == [
(3 * MiB, False),
(64 * KiB, True),
(nbd_env.virtual_size - 3 * MiB - 64 * KiB, False),
]
assert c.read(2 * MiB, 64 * KiB) == b"\xf2" * 64 * KiB
