def test_options(tmpdir, fmt, cache, aio, discard):
size = 4 * 1024**2
chunk_size = 128 * 1024
src = str(tmpdir.join("src." + fmt))
qemu_img.create(src, fmt, size=size)
with qemu_nbd.open(src, fmt) as c:
for offset in range(0, size, chunk_size):
c.write(offset, struct.pack(">Q", offset))
c.flush()
dst = str(tmpdir.join("dst." + fmt))
qemu_img.create(dst, fmt, size=size)
src_addr = nbd.UnixAddress(str(tmpdir.join("src.sock")))
dst_addr = nbd.UnixAddress(str(tmpdir.join("dst.sock")))
with qemu_nbd.run(
src, fmt, src_addr,
read_only=True,
cache=cache,
aio=aio,
discard=discard), \
qemu_nbd.run(
dst, fmt, dst_addr,
cache=cache,
aio=aio,
discard=discard), \
nbd.Client(src_addr) as src_client, \
nbd.Client(dst_addr) as dst_client:
nbdutil.copy(src_client, dst_client)
qemu_img.compare(src, dst)
def test_full_backup_complete_chain(tmpdir, nbd_sock, checkpoint):
depth = 3
chunk_size = 1024**2
disk_size = depth * chunk_size
for i in range(depth):
# Create disk based on previous one.
disk = str(tmpdir.join("disk.{}".format(i)))
if i == 0:
qemu_img.create(disk, "qcow2", size=disk_size)
else:
qemu_img.create(disk, "qcow2", backing="disk.{}".format(i - 1))
# This data can be read only from this disk.
with qemu_nbd.open(disk, "qcow2") as d:
offset = i * chunk_size
d.write(offset, b"%d\n" % offset)
d.flush()
# Start full backup and copy the data, veifying what we read.
with backup.full_backup(tmpdir,
disk,
"qcow2",
nbd_sock,
checkpoint=checkpoint):
verify_full_backup(nbd_sock, "sda")
if checkpoint:
bitmaps = list_bitmaps(disk)
assert len(bitmaps) == 1
assert bitmaps[0]["name"] == checkpoint
def test_size(nbd_server, fmt):
size = 150 * 1024**2
nbd_server.fmt = fmt
qemu_img.create(nbd_server.image, fmt, size=size)
nbd_server.start()
with nbd.open(nbd_server.url, "r") as b:
assert b.size() == size
def test_extents_zero(nbd_server, user_file, fmt):
size = 6 * 1024**3
qemu_img.create(user_file.path, fmt, size=size)
nbd_server.image = user_file.path
nbd_server.fmt = fmt
nbd_server.start()
with nbd.open(nbd_server.url, "r+") as b:
# qcow2 extents resolution is cluster size.
data = b"x" * 64 * 1024
b.write(data)
# The second extent length is bigger than NBD maximum length, testing
# that our extent length is not limited by NBD limits. The backend
# sends multiple block status commands and merge the returned extents.
b.seek(5 * 1024**3)
b.write(data)
assert list(b.extents()) == [
image.ZeroExtent(0, len(data), False),
image.ZeroExtent(len(data), 5 * 1024**3 - len(data), True),
image.ZeroExtent(5 * 1024**3, len(data), False),
image.ZeroExtent(5 * 1024**3 + len(data), 1024**3 - len(data),
True),
]
def test_compare_error(tmpdir):
src = str(tmpdir.join("src.raw"))
dst = str(tmpdir.join("dst.raw"))
qemu_img.create(src, "raw", size=1024**2)
with pytest.raises(RuntimeError):
qemu_img.compare(src, dst)
def test_extents_dirty_not_availabe(nbd_server, fmt):
qemu_img.create(nbd_server.image, fmt, 65536)
nbd_server.fmt = fmt
nbd_server.start()
with nbd.open(nbd_server.url, "r+", dirty=True) as b:
with pytest.raises(errors.UnsupportedOperation):
list(b.extents(context="dirty"))
def test_create_info(tmpdir, fmt):
size = 1024**2
image = str(tmpdir.join("image." + fmt))
qemu_img.create(image, fmt, size=size)
info = qemu_img.info(image)
assert info["filename"] == image
assert info["virtual-size"] == size
assert info["format"] == fmt
def test_compare_identical(tmpdir, src_fmt, dst_fmt):
size = 1024**2
src = str(tmpdir.join("src." + src_fmt))
dst = str(tmpdir.join("dst." + dst_fmt))
qemu_img.create(src, src_fmt, size=size)
qemu_img.create(dst, dst_fmt, size=size)
qemu_img.compare(src, dst)
def create_image(path, fmt, size):
if fmt == "raw":
# qemu-img allocates the first block on Fedora, but not on CentOS 8.0.
# Allocate manually for consistent results.
# TODO: Use qemu-img when we have CentOS 8.1 AV.
with io.open(path, "wb") as f:
f.truncate(size)
else:
qemu_img.create(path, "qcow2", size=size)
def verify_backup(backup_disk, expected_files):
log.info("Verifying backup")
preview_disk = backup_disk + ".preview"
qemu_img.create(preview_disk, "qcow2", backing=backup_disk)
with qemu.run(preview_disk, "qcow2") as guest:
guest.login("root", "")
out = guest.run("ls -1 --color=never")
assert out.splitlines() == expected_files
def test_open(tmpdir, fmt):
disk = str(tmpdir.join("disk." + fmt))
qemu_img.create(disk, fmt, size=1024**2)
offset = 64 * 1024
data = b"it works"
with qemu_nbd.open(disk, fmt) as d:
d.write(offset, data)
d.flush()
with qemu_nbd.open(disk, fmt, read_only=True) as d:
assert d.read(offset, len(data)) == data
def test_compare_different(tmpdir, src_fmt, dst_fmt):
size = 1024**2
src = str(tmpdir.join("src." + src_fmt))
dst = str(tmpdir.join("dst." + dst_fmt))
qemu_img.create(src, src_fmt, size=size)
qemu_img.create(dst, dst_fmt, size=size)
with qemu_nbd.open(dst, dst_fmt) as c:
c.write(size // 2, b"x")
c.flush()
with pytest.raises(qemu_img.ContentMismatch):
qemu_img.compare(src, dst)
def test_copy_nbd_to_nbd(tmpdir, src_fmt, dst_fmt, zero):
# Make sure we have zero extents larger than MAX_ZERO_SIZE (1 GiB). It
# would be nice to have also data extents larger than MAX_COPY_SIZE (128
# MiB), but this is too slow for automated tests.
size = 2 * io.MAX_ZERO_SIZE
# Default cluser size with qcow2 format.
cluster_size = 64 * 1024
src = str(tmpdir.join("src." + src_fmt))
qemu_img.create(src, src_fmt, size=size)
with qemu_nbd.open(src, src_fmt) as c:
# Create first data extent.
c.write(0, b"data extent 1\n")
# Between the data extents we have a zero extent bigger than
# io.MAX_ZERO_SIZE.
# Create data extent larger than io.BUFFER_SIZE.
data = b"data extent 2\n" + b"x" * io.BUFFER_SIZE
c.write(io.MAX_ZERO_SIZE + 2 * cluster_size, data)
# Between the data extents we have a zero extent smaller than
# io.MAX_ZERO_SIZE.
# Create last data extent at the end of the image.
c.write(size - 4096, b"data extent 3\n")
c.flush()
src_sock = UnixAddress(tmpdir.join("src.sock"))
src_url = urlparse(src_sock.url())
dst = str(tmpdir.join("dst." + dst_fmt))
qemu_img.create(dst, dst_fmt, size=size)
dst_sock = UnixAddress(tmpdir.join("dst.sock"))
dst_url = urlparse(dst_sock.url())
with qemu_nbd.run(src, src_fmt, src_sock, read_only=True), \
qemu_nbd.run(dst, dst_fmt, dst_sock), \
nbd.open(src_url, "r") as src_backend, \
nbd.open(dst_url, "r+") as dst_backend:
# Because we copy to new image, we can alays use zero=False, but we
# test both to verify that the result is the same.
io.copy(src_backend, dst_backend, zero=zero)
qemu_img.compare(src, dst)
def full_backup(tmpdir, disk, fmt, sock, checkpoint=None):
"""
Start qemu internal nbd server using address sock, exposing disk for
full backup, creating temporary files in tmpdir.
"""
scratch_disk = str(tmpdir.join("scratch.qcow2"))
qmp_sock = nbd.UnixAddress(tmpdir.join("qmp.sock"))
disk_size = qemu_img.info(disk)["virtual-size"]
qemu_img.create(scratch_disk, "qcow2", size=disk_size)
with qemu.run(disk, fmt, qmp_sock, start_cpu=False, shutdown_timeout=10), \
qmp.Client(qmp_sock) as c, \
run(c, sock, scratch_disk, checkpoint=checkpoint):
yield
def test_add_bitmap(tmpdir):
# Test command with arguments. This is also interesting for incremental
# backup flows.
image = str(tmpdir.join("image.qcow2"))
qemu_img.create(image, "qcow2", size=1024**3)
qmp_sock = nbd.UnixAddress(tmpdir.join("qmp.sock"))
with qemu.run(image, "qcow2", qmp_sock, start_cpu=False):
with qmp.Client(qmp_sock) as c:
c.execute("block-dirty-bitmap-add", {
"node": "file0",
"name": "bitmap0",
})
b = qmp.find_node(c, image)
assert b["dirty-bitmaps"][0]["name"] == "bitmap0"
def test_incremental_backup_guest(tmpdir, base_image):
disk_size = qemu_img.info(base_image)["virtual-size"]
disk = str(tmpdir.join("disk.qcow2"))
qemu_img.create(disk, "qcow2", backing=base_image)
scratch_disk = str(tmpdir.join("scratch.qcow2"))
qemu_img.create(scratch_disk, "qcow2", size=disk_size)
full_backup_disk = str(tmpdir.join("full-backup.qcow2"))
qemu_img.create(full_backup_disk, "qcow2", size=disk_size)
incr_backup_disk = str(tmpdir.join("incr-backup.qcow2"))
qemu_img.create(incr_backup_disk, "qcow2", size=disk_size)
qmp_sock = nbd.UnixAddress(tmpdir.join("qmp.sock"))
nbd_sock = nbd.UnixAddress(tmpdir.join("nbd.sock"))
with qemu.run(disk, "qcow2", qmp_sock, shutdown_timeout=10) as guest, \
qmp.Client(qmp_sock) as qmp_client:
guest.login("root", "")
with backup.run(qmp_client,
nbd_sock,
scratch_disk,
checkpoint="check1"):
backup.copy_disk(nbd_sock.url("sda"), full_backup_disk)
qemu_img.create(scratch_disk, "qcow2", size=disk_size)
assert guest.run("touch before-backup; sync") == ""
with backup.run(qmp_client,
nbd_sock,
scratch_disk,
checkpoint="check2",
incremental="check1"):
assert guest.run("touch during-backup; sync") == ""
backup.copy_dirty(nbd_sock.url("sda"), incr_backup_disk)
qemu_img.unsafe_rebase(incr_backup_disk, full_backup_disk)
verify_backup(incr_backup_disk, ["before-backup"])
def test_shared(tmpdir, fmt):
size = 1024**2
chunk_size = size // 2
src = str(tmpdir.join("src." + fmt))
qemu_img.create(src, fmt, size=size)
with qemu_nbd.open(src, fmt) as c:
c.write(0, b"a" * chunk_size)
c.write(0, b"b" * chunk_size)
c.flush()
dst = str(tmpdir.join("dst." + fmt))
qemu_img.create(dst, fmt, size=size)
src_addr = nbd.UnixAddress(str(tmpdir.join("src.sock")))
dst_addr = nbd.UnixAddress(str(tmpdir.join("dst.sock")))
# Start 2 qemu-nbd servers, each with 2 clients that can read and write in
# parallel for higher throughput.
with qemu_nbd.run(src, fmt, src_addr, read_only=True, shared=2), \
qemu_nbd.run(dst, fmt, dst_addr, shared=2), \
nbd.Client(src_addr) as src_client1, \
nbd.Client(src_addr) as src_client2, \
nbd.Client(dst_addr) as dst_client1, \
nbd.Client(dst_addr) as dst_client2:
# Copy first half of the image with src_client1 and dst_client2 and
# second half with src_client2 and dst_client2. In a real application
# we would have more clients, running in multiple threads.
chunk1 = src_client1.read(0, chunk_size)
dst_client1.write(0, chunk1)
chunk2 = src_client2.read(chunk_size, chunk_size)
dst_client2.write(chunk_size, chunk2)
dst_client1.flush()
dst_client2.flush()
qemu_img.compare(src, dst)
def test_full_backup(tmpdir, fmt, transport):
disk_size = 1024**2
disk_part = disk_size // 4
disk = str(tmpdir.join("disk." + fmt))
backup_disk = str(tmpdir.join("backup.raw"))
# Create disk
qemu_img.create(disk, fmt, size=disk_size)
# Pupulate disk with data.
with qemu_nbd.open(disk, fmt) as d:
for i in range(0, disk_size, disk_part):
data = b"%d\n" % i
d.write(i, data.ljust(512))
d.flush()
if transport == "unix":
nbd_sock = nbd.UnixAddress(tmpdir.join("nbd.sock"))
else:
nbd_sock = nbd.TCPAddress("localhost", testutil.random_tcp_port())
# Backup using qemu-img convert.
with backup.full_backup(tmpdir, disk, fmt, nbd_sock):
log.debug("Backing up image with qemu-img")
qemu_img.convert(nbd_sock.url("sda"),
backup_disk,
src_fmt="raw",
dst_fmt="raw",
progress=True)
# Compare source and backup disks.
with qemu_nbd.open(disk, fmt, read_only=True) as d, \
io.open(backup_disk, "rb") as b:
for i in range(0, disk_size, disk_part):
b.seek(i)
assert d.read(i, 512) == b.read(512)