def test_full_backup_guest(tmpdir, base_image):
base = qemu_img.info(base_image)
disk_size = base["virtual-size"]
disk = str(tmpdir.join("disk.qcow2"))
qemu_img.create(disk,
"qcow2",
backing_file=base_image,
backing_format=base["format"])
scratch_disk = str(tmpdir.join("scratch.qcow2"))
qemu_img.create(scratch_disk, "qcow2", size=disk_size)
backup_disk = str(tmpdir.join("backup.qcow2"))
qemu_img.create(backup_disk, "qcow2", size=disk_size)
qmp_sock = nbd.UnixAddress(tmpdir.join("qmp.sock"))
nbd_sock = nbd.UnixAddress(tmpdir.join("nbd.sock"))
with qemu.run(disk, "qcow2", qmp_sock, shutdown_timeout=10) as guest, \
qmp.Client(qmp_sock) as qmp_client:
guest.login("root", "")
assert guest.run("touch before-backup; sync") == ""
with backup.run(qmp_client,
nbd_sock,
scratch_disk,
checkpoint="check1"):
assert guest.run("touch during-backup; sync") == ""
backup.copy_disk(nbd_sock.url("sda"), backup_disk)
verify_backup(backup_disk, ["before-backup"])
def test_options(tmpdir, fmt, options):
size = 4 * 1024**2
chunk_size = 128 * 1024
src = str(tmpdir.join("src." + fmt))
qemu_img.create(src, fmt, size=size)
with qemu_nbd.open(src, fmt) as c:
for offset in range(0, size, chunk_size):
c.write(offset, struct.pack(">Q", offset))
c.flush()
dst = str(tmpdir.join("dst." + fmt))
qemu_img.create(dst, fmt, size=size)
src_addr = nbd.UnixAddress(str(tmpdir.join("src.sock")))
dst_addr = nbd.UnixAddress(str(tmpdir.join("dst.sock")))
with qemu_nbd.run(
src, fmt, src_addr,
read_only=True,
**options), \
qemu_nbd.run(
dst, fmt, dst_addr,
**options), \
nbd.Client(src_addr) as src_client, \
nbd.Client(dst_addr) as dst_client:
nbdutil.copy(src_client, dst_client)
qemu_img.compare(src, dst)
def test_backing_chain(tmpdir):
size = 128 * 1024
base = str(tmpdir.join("base.raw"))
top = str(tmpdir.join("top.qcow2"))
base_data = b"data from base".ljust(32, b"\0")
# Add base image with some data.
qemu_img.create(base, "raw", size=size)
with qemu_nbd.open(base, "raw") as c:
c.write(0, base_data)
c.flush()
# Add empty overlay.
qemu_img.create(top, "qcow2", backing_file=base, backing_format="raw")
top_addr = nbd.UnixAddress(str(tmpdir.join("sock")))
# By default, we see data from base.
with qemu_nbd.run(top, "qcow2", top_addr), \
nbd.Client(top_addr) as c:
assert c.read(0, 32) == base_data
# With backing chain disabled, we see data only from top.
with qemu_nbd.run(top, "qcow2", top_addr, backing_chain=False), \
nbd.Client(top_addr) as c:
assert c.read(0, 32) == b"\0" * 32
def test_shared(tmpdir, fmt):
size = 1024**2
chunk_size = size // 2
src = str(tmpdir.join("src." + fmt))
qemu_img.create(src, fmt, size=size)
with qemu_nbd.open(src, fmt) as c:
c.write(0, b"a" * chunk_size)
c.write(0, b"b" * chunk_size)
c.flush()
dst = str(tmpdir.join("dst." + fmt))
qemu_img.create(dst, fmt, size=size)
src_addr = nbd.UnixAddress(str(tmpdir.join("src.sock")))
dst_addr = nbd.UnixAddress(str(tmpdir.join("dst.sock")))
# Start 2 qemu-nbd servers, each with 2 clients that can read and write in
# parallel for higher throughput.
with qemu_nbd.run(src, fmt, src_addr, read_only=True, shared=2), \
qemu_nbd.run(dst, fmt, dst_addr, shared=2), \
nbd.Client(src_addr) as src_client1, \
nbd.Client(src_addr) as src_client2, \
nbd.Client(dst_addr) as dst_client1, \
nbd.Client(dst_addr) as dst_client2:
# Copy first half of the image with src_client1 and dst_client2 and
# second half with src_client2 and dst_client2. In a real application
# we would have more clients, running in multiple threads.
chunk1 = src_client1.read(0, chunk_size)
dst_client1.write(0, chunk1)
chunk2 = src_client2.read(chunk_size, chunk_size)
dst_client2.write(chunk_size, chunk2)
dst_client1.flush()
dst_client2.flush()
qemu_img.compare(src, dst)
def test_open_unix(tmpdir, url, export):
image = str(tmpdir.join("image"))
with open(image, "wb") as f:
f.truncate(1024**3)
sock = nbd.UnixAddress(tmpdir.join("sock"))
url = url.replace("/path", sock)
log.debug("Trying url=%r export=%r", url, export)
with qemu_nbd.run(image, "raw", sock, export_name=export):
with nbd.open(urlparse(url)) as c:
assert c.export_size == 1024**3
def test_query_status(tmpdir):
# Simplest possible test.
image = str(tmpdir.join("image.raw"))
with open(image, "wb") as f:
f.truncate(1024**2)
qmp_sock = nbd.UnixAddress(tmpdir.join("qmp.sock"))
with qemu.run(image, "raw", qmp_sock, start_cpu=False):
with qmp.Client(qmp_sock) as c:
r = c.execute("query-status")
assert r["status"] == "prelaunch"
def test_raw_read(tmpdir):
image = str(tmpdir.join("image"))
sock = nbd.UnixAddress(tmpdir.join("sock"))
offset = 1024**2
data = b"can read from raw"
with open(image, "wb") as f:
f.truncate(1024**3)
f.seek(offset)
f.write(data)
with qemu_nbd.run(image, "raw", sock):
with nbd.Client(sock) as c:
assert c.read(offset, len(data)) == data
def test_get_nbd_backend(tmpdir, cfg, nbd_server, transport):
if transport == "unix":
nbd_server.sock = nbd.UnixAddress(tmpdir.join("sock"))
else:
nbd_server.sock = nbd.TCPAddress("localhost",
testutil.random_tcp_port())
nbd_server.start()
ticket = auth.Ticket(
testutil.create_ticket(url=urlunparse(nbd_server.url)))
req = Request()
b = backends.get(req, ticket, cfg).backend
assert b.name == "nbd"
def test_qcow2_write_read(tmpdir):
image = str(tmpdir.join("image"))
sock = nbd.UnixAddress(tmpdir.join("sock"))
offset = 1024**2
data = b"can read and write qcow2"
create_image(image, "qcow2", 1024**3)
with qemu_nbd.run(image, "qcow2", sock):
with nbd.Client(sock) as c:
c.write(offset, data)
c.flush()
with nbd.Client(sock) as c:
assert c.read(offset, len(data)) == data
def test_run_unix(tmpdir):
image = str(tmpdir.join("image"))
sock = str(tmpdir.join("sock"))
with io.open(image, "wb") as f:
f.truncate(1024**2)
addr = nbd.UnixAddress(sock)
with qemu_nbd.run(image, "raw", addr):
# The helper already waited for the NBD socket, not wait needed.
assert sockutil.wait_for_socket(addr, 0.0)
# The socket must be closed, no wait needed.
assert not sockutil.wait_for_socket(addr, 0.0)
def full_backup(tmpdir, disk, fmt, sock, checkpoint=None):
"""
Start qemu internal nbd server using address sock, exposing disk for
full backup, creating temporary files in tmpdir.
"""
scratch_disk = str(tmpdir.join("scratch.qcow2"))
qmp_sock = nbd.UnixAddress(tmpdir.join("qmp.sock"))
disk_size = qemu_img.info(disk)["virtual-size"]
qemu_img.create(scratch_disk, "qcow2", size=disk_size)
with qemu.run(disk, fmt, qmp_sock, start_cpu=False, shutdown_timeout=10), \
qmp.Client(qmp_sock) as c, \
run(c, sock, scratch_disk, checkpoint=checkpoint):
yield
def test_add_bitmap(tmpdir):
# Test command with arguments. This is also interesting for incremental
# backup flows.
image = str(tmpdir.join("image.qcow2"))
qemu_img.create(image, "qcow2", size=1024**3)
qmp_sock = nbd.UnixAddress(tmpdir.join("qmp.sock"))
with qemu.run(image, "qcow2", qmp_sock, start_cpu=False):
with qmp.Client(qmp_sock) as c:
c.execute("block-dirty-bitmap-add", {
"node": "file0",
"name": "bitmap0",
})
b = qmp.find_node(c, image)
assert b["dirty-bitmaps"][0]["name"] == "bitmap0"
def test_raw_write(tmpdir):
image = str(tmpdir.join("image"))
with open(image, "wb") as f:
f.truncate(1024**3)
sock = nbd.UnixAddress(tmpdir.join("sock"))
offset = 1024**2
data = b"can write to raw"
with qemu_nbd.run(image, "raw", sock):
with nbd.Client(sock) as c:
c.write(offset, data)
c.flush()
with open(image, "rb") as f:
f.seek(offset)
assert f.read(len(data)) == data
def test_handshake(tmpdir, export, fmt):
image = str(tmpdir.join("image"))
create_image(image, fmt, 1024**3)
sock = nbd.UnixAddress(tmpdir.join("sock"))
with qemu_nbd.run(image, fmt, sock, export_name=export):
if export:
c = nbd.Client(sock, export)
else:
c = nbd.Client(sock)
with c:
# TODO: test transmission_flags?
assert c.export_size == 1024**3
assert c.minimum_block_size == 1
assert c.preferred_block_size == 4096
assert c.maximum_block_size == 32 * 1024**2
assert c.base_allocation
def test_handshake(tmpdir, export, fmt):
image = str(tmpdir.join("image"))
create_image(image, fmt, 1024**3)
sock = nbd.UnixAddress(tmpdir.join("sock"))
with qemu_nbd.run(image, fmt, sock, export_name=export):
if export:
c = nbd.Client(sock, export)
else:
c = nbd.Client(sock)
with c:
# TODO: test transmission_flags?
assert c.export_size == 1024**3
assert c.minimum_block_size == 1
assert c.preferred_block_size == 4096
assert c.maximum_block_size == 32 * 1024**2
# Both available in in current qemu-nbd (>= 5.2.0).
assert c.has_base_allocation
assert c.has_allocation_depth == (fmt != "raw")
def test_zero_min_block_size(tmpdir, fmt):
offset = 1024**2
image = str(tmpdir.join("image"))
sock = nbd.UnixAddress(tmpdir.join("sock"))
create_image(image, fmt, 1024**3)
with qemu_nbd.run(image, fmt, sock):
# Fill range with data
with nbd.Client(sock) as c:
size = c.minimum_block_size
c.write(offset, b"x" * size)
c.flush()
# Zero range using minimum block size
with nbd.Client(sock) as c:
c.zero(offset, size)
c.flush()
with nbd.Client(sock) as c:
assert c.read(offset, size) == b"\0" * size
def test_zero(tmpdir, fmt):
size = 2 * 1024**2
offset = 1024**2
image = str(tmpdir.join("image"))
sock = nbd.UnixAddress(tmpdir.join("sock"))
create_image(image, fmt, size)
with qemu_nbd.run(image, fmt, sock):
# Fill image with data
with nbd.Client(sock) as c:
c.write(0, b"x" * size)
c.flush()
# Zero a range
with nbd.Client(sock) as c:
c.zero(offset, 4096)
c.flush()
with nbd.Client(sock) as c:
assert c.read(offset, 4096) == b"\0" * 4096
def nbd_server(tmpdir):
"""
Returns nbd_server exporting a temporary file.
The test may configure the server before starting it. Typical example is
setting the read_only flag to create a read only server.
The test must start the server. The test framework will stop the server
when the test ends.
"""
image = str(tmpdir.join("image"))
with io.open(image, "wb") as f:
f.truncate(10 * 1024**2)
sock = nbd.UnixAddress(tmpdir.join("sock"))
server = qemu_nbd.Server(image, "raw", sock)
try:
yield server
finally:
server.stop()
def test_add_bitmap(tmpdir):
# Test command with arguments. This is also interesting for incremental
# backup flows.
image = str(tmpdir.join("image.qcow2"))
qemu_img.create(image, "qcow2", size=1024**3)
qmp_sock = nbd.UnixAddress(tmpdir.join("qmp.sock"))
with qemu.run(image, "qcow2", qmp_sock, start_cpu=False):
with qmp.Client(qmp_sock) as c:
c.execute("block-dirty-bitmap-add", {
"node": "file0",
"name": "bitmap0",
})
node = qmp.find_node(c, image)
log.debug("Found node:\n%s", pprint.pformat(node))
if qemu.version() >= (6, 0, 0):
bitmaps = node["inserted"]["dirty-bitmaps"]
else:
bitmaps = node["dirty-bitmaps"]
assert bitmaps[0]["name"] == "bitmap0"
def test_full_backup(tmpdir, fmt, transport):
disk_size = 1024**2
disk_part = disk_size // 4
disk = str(tmpdir.join("disk." + fmt))
backup_disk = str(tmpdir.join("backup.raw"))
# Create disk
qemu_img.create(disk, fmt, size=disk_size)
# Pupulate disk with data.
with qemu_nbd.open(disk, fmt) as d:
for i in range(0, disk_size, disk_part):
data = b"%d\n" % i
d.write(i, data.ljust(512))
d.flush()
if transport == "unix":
nbd_sock = nbd.UnixAddress(tmpdir.join("nbd.sock"))
else:
nbd_sock = nbd.TCPAddress("localhost", testutil.random_tcp_port())
# Backup using qemu-img convert.
with backup.full_backup(tmpdir, disk, fmt, nbd_sock):
log.debug("Backing up image with qemu-img")
qemu_img.convert(nbd_sock.url("sda"),
backup_disk,
src_fmt="raw",
dst_fmt="raw",
progress=True)
# Compare source and backup disks.
with qemu_nbd.open(disk, fmt, read_only=True) as d, \
io.open(backup_disk, "rb") as b:
for i in range(0, disk_size, disk_part):
b.seek(i)
assert d.read(i, 512) == b.read(512)
import urllib.parse
import pytest
from ovirt_imageio._internal import nbd
from ovirt_imageio._internal import nbdutil
from ovirt_imageio._internal import qemu_img
from ovirt_imageio._internal import qemu_nbd
from ovirt_imageio._internal import sockutil
from . import testutil
from . marks import flaky_in_ovirt_ci
@pytest.mark.parametrize("addr,export,url", [
(nbd.UnixAddress("/sock"), None, "nbd:unix:/sock"),
(nbd.UnixAddress("/sock"), "", "nbd:unix:/sock"),
(nbd.UnixAddress("/sock"), "sda", "nbd:unix:/sock:exportname=sda"),
(nbd.TCPAddress("host", 10900), None, "nbd:host:10900"),
(nbd.TCPAddress("host", 10900), "", "nbd:host:10900"),
(nbd.TCPAddress("host", 10900), "sdb", "nbd:host:10900:exportname=sdb"),
])
def test_server_url(addr, export, url):
srv = qemu_nbd.Server("image", "raw", addr, export_name=export)
assert srv.url == urllib.parse.urlparse(url)
@pytest.mark.parametrize("fmt", ["raw", "qcow2"])
def test_open(tmpdir, fmt):
disk = str(tmpdir.join("disk." + fmt))
qemu_img.create(disk, fmt, size=1024**2)
def nbd_sock(request, tmpdir):
if request.param == "unix":
return nbd.UnixAddress(tmpdir.join("sock"))
else:
return nbd.TCPAddress("localhost", testutil.random_tcp_port())
