def test_options(tmpdir, fmt, options):
size = 4 * 1024**2
chunk_size = 128 * 1024
src = str(tmpdir.join("src." + fmt))
qemu_img.create(src, fmt, size=size)
with qemu_nbd.open(src, fmt) as c:
for offset in range(0, size, chunk_size):
c.write(offset, struct.pack(">Q", offset))
c.flush()
dst = str(tmpdir.join("dst." + fmt))
qemu_img.create(dst, fmt, size=size)
src_addr = nbd.UnixAddress(str(tmpdir.join("src.sock")))
dst_addr = nbd.UnixAddress(str(tmpdir.join("dst.sock")))
with qemu_nbd.run(
src, fmt, src_addr,
read_only=True,
**options), \
qemu_nbd.run(
dst, fmt, dst_addr,
**options), \
nbd.Client(src_addr) as src_client, \
nbd.Client(dst_addr) as dst_client:
nbdutil.copy(src_client, dst_client)
qemu_img.compare(src, dst)
def test_zero_extents_raw(tmpdir):
size = 10 * 1024**2
# Create image with some data, zero and holes.
image = str(tmpdir.join("image.raw"))
qemu_img.create(image, "raw", size=size)
with qemu_nbd.open(image, "raw") as c:
c.write(0 * CLUSTER_SIZE, b"A" * CLUSTER_SIZE)
c.zero(1 * CLUSTER_SIZE, CLUSTER_SIZE)
c.write(2 * CLUSTER_SIZE, b"B" * CLUSTER_SIZE)
c.flush()
extents = list(client.extents(image))
# Note: raw files report unallocated as zero, not a a hole.
assert extents == [
ZeroExtent(start=0 * CLUSTER_SIZE,
length=CLUSTER_SIZE,
zero=False,
hole=False),
ZeroExtent(start=1 * CLUSTER_SIZE,
length=CLUSTER_SIZE,
zero=True,
hole=False),
ZeroExtent(start=2 * CLUSTER_SIZE,
length=CLUSTER_SIZE,
zero=False,
hole=False),
ZeroExtent(start=3 * CLUSTER_SIZE,
length=size - 3 * CLUSTER_SIZE,
zero=True,
hole=False),
]
def test_ova(tmpdir, fmt):
size = 1024**2
offset = 64 * 1024
disks = []
# Created disks with unique content.
for i in range(2):
disk = str(tmpdir.join("disk{}.{}".format(i, fmt)))
qemu_img.create(disk, fmt, size=size)
with qemu_nbd.open(disk, fmt) as d:
d.write(offset, disk.encode("utf-8"))
d.flush()
disks.append(disk)
ova = str(tmpdir.join("vm.ova"))
# Create a ova file.
with tarfile.open(ova, "w") as tar:
for disk in disks:
tar.add(disk, arcname=os.path.basename(disk))
# Read disks contents from the ova file.
with tarfile.open(ova) as tar:
for disk in disks:
member = tar.getmember(os.path.basename(disk))
with qemu_nbd.open(
ova,
fmt=fmt,
read_only=True,
offset=member.offset_data,
size=member.size) as d:
assert d.export_size == size
data = disk.encode("utf-8")
assert d.read(offset, len(data)) == data
def test_full_backup_complete_chain(tmpdir, nbd_sock, checkpoint):
depth = 3
chunk_size = 1024**2
disk_size = depth * chunk_size
for i in range(depth):
# Create disk based on previous one.
disk = str(tmpdir.join("disk.{}".format(i)))
if i == 0:
qemu_img.create(disk, "qcow2", size=disk_size)
else:
qemu_img.create(disk,
"qcow2",
backing_file="disk.{}".format(i - 1),
backing_format="qcow2")
# This data can be read only from this disk.
with qemu_nbd.open(disk, "qcow2") as d:
offset = i * chunk_size
d.write(offset, b"%d\n" % offset)
d.flush()
# Start full backup and copy the data, veifying what we read.
with backup.full_backup(tmpdir,
disk,
"qcow2",
nbd_sock,
checkpoint=checkpoint):
verify_full_backup(nbd_sock, "sda")
if checkpoint:
bitmaps = list_bitmaps(disk)
assert len(bitmaps) == 1
assert bitmaps[0]["name"] == checkpoint
def test_zero_extents_from_ova(tmpdir):
size = 10 * 1024**2
# Create image with data, zero and hole clusters.
disk = str(tmpdir.join("disk.qcow2"))
qemu_img.create(disk, "qcow2", size=size)
with qemu_nbd.open(disk, "qcow2") as c:
c.write(0 * CLUSTER_SIZE, b"A" * CLUSTER_SIZE)
c.zero(1 * CLUSTER_SIZE, CLUSTER_SIZE)
c.flush()
# Create OVA whith this image.
ova = str(tmpdir.join("vm.ova"))
with tarfile.open(ova, "w") as tar:
tar.add(disk, arcname=os.path.basename(disk))
extents = list(client.extents(ova, member="disk.qcow2"))
assert extents == [
ZeroExtent(start=0 * CLUSTER_SIZE,
length=CLUSTER_SIZE,
zero=False,
hole=False),
ZeroExtent(start=1 * CLUSTER_SIZE,
length=CLUSTER_SIZE,
zero=True,
hole=False),
ZeroExtent(start=2 * CLUSTER_SIZE,
length=size - 2 * CLUSTER_SIZE,
zero=True,
hole=True),
]
def test_download_qcow2_as_raw(tmpdir, srv):
src = str(tmpdir.join("src.qcow2"))
qemu_img.create(src, "qcow2", size=IMAGE_SIZE)
# Allocate one cluster in the middle of the image.
with qemu_nbd.open(src, "qcow2") as c:
c.write(CLUSTER_SIZE, b"a" * CLUSTER_SIZE)
c.flush()
actual_size = os.path.getsize(src)
url = prepare_transfer(srv, "file://" + src, size=actual_size)
dst = str(tmpdir.join("dst.qcow2"))
# When downloading qcow2 image using the nbd backend, we get raw data and
# we can convert it to any format we want. Howver when downloading using
# the file backend, we get qcow2 bytestream and we cannot convert it.
#
# To store the qcow2 bytestream, we must use fmt="raw". This instructs
# qemu-nbd on the client side to treat the data as raw bytes, storing them
# without any change on the local file.
#
# This is baisically like:
#
# qemu-img convert -f raw -O raw src.qcow2 dst.qcow2
#
client.download(url, dst, srv.config.tls.ca_file, fmt="raw")
# The result should be identical qcow2 image content. Allocation may
# differ but for this test we get identical allocation.
qemu_img.compare(src, dst, format1="qcow2", format2="qcow2", strict=True)
def test_dirty_bitmap(tmpdir):
size = 1024**2
# Create image with empty bitmap.
img = str(tmpdir.join("img.qcow2"))
qemu_img.create(img, "qcow2", size=size)
qemu_img.bitmap_add(img, "b0")
# Write data to image, modifying the bitmap.
with qemu_nbd.open(img, "qcow2") as c:
# This will allocate one cluster. By default bitmap granularity is also
# one cluster, so this will make the first extent dirty.
c.write(0, b"a")
c.flush()
# Read dirty extents.
with qemu_nbd.open(img, "qcow2", read_only=True, bitmap="b0") as c:
extents = c.extents(0, size)[nbd.QEMU_DIRTY_BITMAP + "b0"]
bitmap = qemu_img.info(img)["format-specific"]["data"]["bitmaps"][0]
assert extents == [
nbd.Extent(length=bitmap["granularity"], flags=nbd.EXTENT_DIRTY),
nbd.Extent(length=size - bitmap["granularity"], flags=0),
]
def test_backing_chain(tmpdir):
size = 128 * 1024
base = str(tmpdir.join("base.raw"))
top = str(tmpdir.join("top.qcow2"))
base_data = b"data from base".ljust(32, b"\0")
# Add base image with some data.
qemu_img.create(base, "raw", size=size)
with qemu_nbd.open(base, "raw") as c:
c.write(0, base_data)
c.flush()
# Add empty overlay.
qemu_img.create(top, "qcow2", backing_file=base, backing_format="raw")
top_addr = nbd.UnixAddress(str(tmpdir.join("sock")))
# By default, we see data from base.
with qemu_nbd.run(top, "qcow2", top_addr), \
nbd.Client(top_addr) as c:
assert c.read(0, 32) == base_data
# With backing chain disabled, we see data only from top.
with qemu_nbd.run(top, "qcow2", top_addr, backing_chain=False), \
nbd.Client(top_addr) as c:
assert c.read(0, 32) == b"\0" * 32
def test_extents_zero(nbd_server, user_file, fmt):
size = 6 * 1024**3
qemu_img.create(user_file.path, fmt, size=size)
nbd_server.image = user_file.path
nbd_server.fmt = fmt
nbd_server.start()
with nbd.open(nbd_server.url, "r+") as b:
# qcow2 extents resolution is cluster size.
data = b"x" * 64 * 1024
b.write(data)
# The second extent length is bigger than NBD maximum length, testing
# that our extent length is not limited by NBD limits. The backend
# sends multiple block status commands and merge the returned extents.
b.seek(5 * 1024**3)
b.write(data)
# Holes can be reported only for qcow2 images.
hole = fmt == "qcow2"
assert list(b.extents()) == [
extent.ZeroExtent(0, len(data), False, False),
extent.ZeroExtent(len(data), 5 * 1024**3 - len(data), True, hole),
extent.ZeroExtent(5 * 1024**3, len(data), False, False),
extent.ZeroExtent(5 * 1024**3 + len(data), 1024**3 - len(data),
True, hole),
]
def test_size(nbd_server, fmt):
size = 150 * 1024**2
nbd_server.fmt = fmt
qemu_img.create(nbd_server.image, fmt, size=size)
nbd_server.start()
with nbd.open(nbd_server.url) as b:
assert b.size() == size
def test_zero_extents_qcow2(tmpdir):
size = 10 * 1024**2
# Create base image with one data and one zero cluster.
base = str(tmpdir.join("base.qcow2"))
qemu_img.create(base, "qcow2", size=size)
with qemu_nbd.open(base, "qcow2") as c:
c.write(0 * CLUSTER_SIZE, b"A" * CLUSTER_SIZE)
c.zero(1 * CLUSTER_SIZE, CLUSTER_SIZE)
c.flush()
# Create top image with one data and one zero cluster.
top = str(tmpdir.join("top.qcow2"))
qemu_img.create(
top, "qcow2", backing_file=base, backing_format="qcow2")
with qemu_nbd.open(top, "qcow2") as c:
c.write(3 * CLUSTER_SIZE, b"B" * CLUSTER_SIZE)
c.zero(4 * CLUSTER_SIZE, CLUSTER_SIZE)
c.flush()
extents = list(client.extents(top))
assert extents == [
# Extents from base...
ZeroExtent(
start=0 * CLUSTER_SIZE,
length=CLUSTER_SIZE,
zero=False,
hole=False),
ZeroExtent(
start=1 * CLUSTER_SIZE,
length=CLUSTER_SIZE,
zero=True,
hole=False),
ZeroExtent(
start=2 * CLUSTER_SIZE,
length=CLUSTER_SIZE,
zero=True,
hole=True),
# Extents from top...
ZeroExtent(
start=3 * CLUSTER_SIZE,
length=CLUSTER_SIZE,
zero=False,
hole=False),
ZeroExtent(
start=4 * CLUSTER_SIZE,
length=CLUSTER_SIZE,
zero=True,
hole=False),
# Rest of unallocated data...
ZeroExtent(
start=5 * CLUSTER_SIZE,
length=size - 5 * CLUSTER_SIZE,
zero=True,
hole=True),
]
def test_compare_missing_file(tmpdir):
src = str(tmpdir.join("src.raw"))
dst = str(tmpdir.join("dst.raw"))
qemu_img.create(src, "raw", size=1024**2)
with pytest.raises(qemu_img.OpenImageError):
qemu_img.compare(src, dst)
def test_checksum_algorithm(tmpdir, algorithm, digest_size):
img = str(tmpdir.join("img"))
qemu_img.create(img, "raw", size="2m")
expected = blkhash.checksum(
img, block_size=1024**2, algorithm=algorithm, digest_size=digest_size)
actual = client.checksum(img, block_size=1024**2, algorithm=algorithm)
assert actual == expected
def test_extents_dirty_not_availabe(nbd_server, fmt):
qemu_img.create(nbd_server.image, fmt, 65536)
nbd_server.fmt = fmt
nbd_server.start()
with nbd.open(nbd_server.url, "r+", dirty=True) as b:
with pytest.raises(errors.UnsupportedOperation):
list(b.extents(context="dirty"))
def create_image(path, fmt, size):
if fmt == "raw":
# qemu-img allocates the first block on Fedora, but not on CentOS 8.0.
# Allocate manually for consistent results.
# TODO: Use qemu-img when we have CentOS 8.1 AV.
with io.open(path, "wb") as f:
f.truncate(size)
else:
qemu_img.create(path, "qcow2", size=size)
def test_add_bitmap(tmpdir):
size = 10 * 1024**2
img = str(tmpdir.join("img.qcow2"))
qemu_img.create(img, "qcow2", size=size)
qemu_img.bitmap_add(img, "b0")
bitmaps = qemu_img.info(img)["format-specific"]["data"]["bitmaps"]
assert bitmaps == [
{"name": "b0", "flags": ["auto"], "granularity": 65536}
]
def test_create_info(tmpdir, fmt):
size = 1024**2
image = str(tmpdir.join("image." + fmt))
qemu_img.create(image, fmt, size=size)
info = qemu_img.info(image)
assert info["filename"] == image
assert info["virtual-size"] == size
assert info["format"] == fmt
def test_zero_sparse(nbd_server, user_file, sparse):
size = 10 * 1024**2
qemu_img.create(user_file.path, "raw", size=size)
nbd_server.image = user_file.path
nbd_server.start()
with nbd.open(nbd_server.url, "r+", sparse=sparse) as b:
b.zero(b.size())
b.flush()
actual_size = os.stat(user_file.path).st_blocks * 512
assert actual_size == 0 if sparse else b.size()
def test_copy_nbd_to_nbd(tmpdir, src_fmt, dst_fmt, zero, hole):
# Default cluser size with qcow2 format.
cluster_size = 64 * 1024
extents = [
("data", cluster_size),
("zero", cluster_size),
("data", cluster_size),
("hole", cluster_size + io.MAX_ZERO_SIZE),
("data", cluster_size + io.BUFFER_SIZE),
("hole", cluster_size),
("data", cluster_size),
]
size = sum(length for _, length in extents)
src = str(tmpdir.join("src." + src_fmt))
qemu_img.create(src, src_fmt, size=size)
populate_image(src, src_fmt, extents)
src_sock = UnixAddress(tmpdir.join("src.sock"))
src_url = urlparse(src_sock.url())
dst = str(tmpdir.join("dst." + dst_fmt))
qemu_img.create(dst, dst_fmt, size=size)
dst_sock = UnixAddress(tmpdir.join("dst.sock"))
dst_url = urlparse(dst_sock.url())
# Note: We need extra worker for reading extents for source.
max_workers = 2
with qemu_nbd.run(
src, src_fmt, src_sock,
read_only=True,
shared=max_workers + 1), \
qemu_nbd.run(
dst, dst_fmt, dst_sock,
shared=max_workers), \
nbd.open(src_url, "r") as src_backend, \
nbd.open(dst_url, "r+", sparse=True) as dst_backend:
# Because we copy to new image, we can always use zero=False, but we
# test both to verify that the result is the same.
io.copy(src_backend,
dst_backend,
max_workers=max_workers,
zero=zero,
hole=hole)
# Compare image content - must match.
qemu_img.compare(src, dst)
# Allocation can be compared only with qcow2 images when we write zeroes to
# zero extents and skip holes.
if src_fmt == "qcow2" and dst_fmt == "qcow2" and zero and not hole:
qemu_img.compare(src, dst, strict=True)
def test_open(tmpdir, fmt):
disk = str(tmpdir.join("disk." + fmt))
qemu_img.create(disk, fmt, size=1024**2)
offset = 64 * 1024
data = b"it works"
with qemu_nbd.open(disk, fmt) as d:
d.write(offset, data)
d.flush()
with qemu_nbd.open(disk, fmt, read_only=True) as d:
assert d.read(offset, len(data)) == data
def verify_backup(backup_disk, expected_files):
log.info("Verifying backup")
preview_disk = backup_disk + ".preview"
qemu_img.create(preview_disk,
"qcow2",
backing_file=backup_disk,
backing_format="qcow2")
with qemu.run(preview_disk, "qcow2") as guest:
guest.login("root", "")
out = guest.run("ls -1 --color=never")
assert out.splitlines() == expected_files
def test_compare_wrong_format(tmpdir):
size = 1024**2
src = str(tmpdir.join("src.raw"))
dst = str(tmpdir.join("dst.raw"))
qemu_img.create(src, "raw", size=size)
qemu_img.create(dst, "raw", size=size)
with pytest.raises(qemu_img.OpenImageError):
qemu_img.compare(src, dst, format1="qcow2")
with pytest.raises(qemu_img.OpenImageError):
qemu_img.compare(src, dst, format2="qcow2")
def test_compare_identical_content(tmpdir, src_fmt, dst_fmt):
size = 1024**2
src = str(tmpdir.join("src." + src_fmt))
dst = str(tmpdir.join("dst." + dst_fmt))
qemu_img.create(src, src_fmt, size=size)
qemu_img.create(dst, dst_fmt, size=size)
# Destination image has different allocation.
with qemu_nbd.open(dst, dst_fmt) as c:
c.write(size // 2, b"\0")
c.flush()
qemu_img.compare(src, dst, format1=src_fmt, format2=dst_fmt)
def test_dirty_extents(tmpdir):
size = 1024**2
# Create base image with empty dirty bitmap.
base = str(tmpdir.join("base.qcow2"))
qemu_img.create(base, "qcow2", size=size)
qemu_img.bitmap_add(base, "b0")
# Write data, modifying the dirty bitmap.
with qemu_nbd.open(base, "qcow2") as c:
c.write(0 * CLUSTER_SIZE, b"A" * CLUSTER_SIZE)
c.zero(1 * CLUSTER_SIZE, CLUSTER_SIZE)
c.flush()
# Create top image with empty dirty bitmap.
top = str(tmpdir.join("top.qcow2"))
qemu_img.create(top, "qcow2", backing_file=base, backing_format="qcow2")
qemu_img.bitmap_add(top, "b0")
# Write data, modifying the dirty bitmap.
with qemu_nbd.open(top, "qcow2") as c:
c.write(3 * CLUSTER_SIZE, b"B" * CLUSTER_SIZE)
c.zero(4 * CLUSTER_SIZE, CLUSTER_SIZE)
c.flush()
dirty_extents = list(client.extents(base, bitmap="b0"))
assert dirty_extents == [
DirtyExtent(start=0 * CLUSTER_SIZE,
length=2 * CLUSTER_SIZE,
dirty=True),
DirtyExtent(start=2 * CLUSTER_SIZE,
length=size - 2 * CLUSTER_SIZE,
dirty=False),
]
dirty_extents = list(client.extents(top, bitmap="b0"))
# Note: qemu-nbd reports dirty extents only for the top image.
assert dirty_extents == [
DirtyExtent(start=0 * CLUSTER_SIZE,
length=3 * CLUSTER_SIZE,
dirty=False),
DirtyExtent(start=3 * CLUSTER_SIZE,
length=2 * CLUSTER_SIZE,
dirty=True),
DirtyExtent(start=5 * CLUSTER_SIZE,
length=size - 5 * CLUSTER_SIZE,
dirty=False),
]
def test_upload_preallocated(tmpdir, srv, fmt):
src = str(tmpdir.join("src"))
qemu_img.create(src, fmt, size=IMAGE_SIZE)
dst = str(tmpdir.join("dst"))
with open(dst, "wb") as f:
f.write(b"a" * IMAGE_SIZE)
url = prepare_transfer(srv, "file://" + dst, sparse=False)
client.upload(src, url, srv.config.tls.ca_file)
qemu_img.compare(src, dst)
assert os.stat(dst).st_blocks * 512 == IMAGE_SIZE
def test_incremental_backup_guest(tmpdir, base_image):
base = qemu_img.info(base_image)
disk_size = base["virtual-size"]
disk = str(tmpdir.join("disk.qcow2"))
qemu_img.create(disk,
"qcow2",
backing_file=base_image,
backing_format=base["format"])
scratch_disk = str(tmpdir.join("scratch.qcow2"))
qemu_img.create(scratch_disk, "qcow2", size=disk_size)
full_backup_disk = str(tmpdir.join("full-backup.qcow2"))
qemu_img.create(full_backup_disk, "qcow2", size=disk_size)
incr_backup_disk = str(tmpdir.join("incr-backup.qcow2"))
qemu_img.create(incr_backup_disk, "qcow2", size=disk_size)
qmp_sock = nbd.UnixAddress(tmpdir.join("qmp.sock"))
nbd_sock = nbd.UnixAddress(tmpdir.join("nbd.sock"))
with qemu.run(disk, "qcow2", qmp_sock, shutdown_timeout=10) as guest, \
qmp.Client(qmp_sock) as qmp_client:
guest.login("root", "")
with backup.run(qmp_client,
nbd_sock,
scratch_disk,
checkpoint="check1"):
backup.copy_disk(nbd_sock.url("sda"), full_backup_disk)
qemu_img.create(scratch_disk, "qcow2", size=disk_size)
assert guest.run("touch before-backup; sync") == ""
with backup.run(qmp_client,
nbd_sock,
scratch_disk,
checkpoint="check2",
incremental="check1"):
assert guest.run("touch during-backup; sync") == ""
backup.copy_dirty(nbd_sock.url("sda"), incr_backup_disk)
qemu_img.unsafe_rebase(incr_backup_disk, full_backup_disk)
verify_backup(incr_backup_disk, ["before-backup"])
def full_backup(tmpdir, disk, fmt, sock, checkpoint=None):
"""
Start qemu internal nbd server using address sock, exposing disk for
full backup, creating temporary files in tmpdir.
"""
scratch_disk = str(tmpdir.join("scratch.qcow2"))
qmp_sock = nbd.UnixAddress(tmpdir.join("qmp.sock"))
disk_size = qemu_img.info(disk)["virtual-size"]
qemu_img.create(scratch_disk, "qcow2", size=disk_size)
with qemu.run(disk, fmt, qmp_sock, start_cpu=False, shutdown_timeout=10), \
qmp.Client(qmp_sock) as c, \
run(c, sock, scratch_disk, checkpoint=checkpoint):
yield
def test_compare_different_content(tmpdir, src_fmt, dst_fmt):
size = 1024**2
src = str(tmpdir.join("src." + src_fmt))
dst = str(tmpdir.join("dst." + dst_fmt))
qemu_img.create(src, src_fmt, size=size)
qemu_img.create(dst, dst_fmt, size=size)
# Destination image has different content.
with qemu_nbd.open(dst, dst_fmt) as c:
c.write(size // 2, b"x")
c.flush()
with pytest.raises(qemu_img.ContentMismatch):
qemu_img.compare(src, dst, format1=src_fmt, format2=dst_fmt)
def test_compare_different_allocation(tmpdir, src_fmt, dst_fmt):
# Images has same content, but different allocation.
size = 1024**2
src = str(tmpdir.join("src." + src_fmt))
dst = str(tmpdir.join("dst." + dst_fmt))
qemu_img.create(src, src_fmt, size=size)
qemu_img.create(dst, dst_fmt, size=size)
with qemu_nbd.open(dst, dst_fmt) as c:
c.write(size // 2, b"\0")
c.flush()
with pytest.raises(qemu_img.ContentMismatch):
qemu_img.compare(
src, dst, format1=src_fmt, format2=dst_fmt, strict=True)
def test_upload_empty_sparse(tmpdir, srv, fmt):
src = str(tmpdir.join("src"))
qemu_img.create(src, fmt, size=IMAGE_SIZE)
dst = str(tmpdir.join("dst"))
with open(dst, "wb") as f:
f.write(b"a" * IMAGE_SIZE)
url = prepare_transfer(srv, "file://" + dst)
client.upload(src, url, srv.config.tls.ca_file)
# TODO: Check why allocation differ when src is qcow2. Target image
# allocation is 0 bytes as expected, but comparing with strict=True fail at
# offset 0.
qemu_img.compare(src, dst, format1=fmt, format2="raw", strict=fmt == "raw")
