def test_download_qcow2_as_raw(tmpdir, srv):
src = str(tmpdir.join("src.qcow2"))
qemu_img.create(src, "qcow2", size=IMAGE_SIZE)
# Allocate one cluster in the middle of the image.
with qemu_nbd.open(src, "qcow2") as c:
c.write(CLUSTER_SIZE, b"a" * CLUSTER_SIZE)
c.flush()
actual_size = os.path.getsize(src)
url = prepare_transfer(srv, "file://" + src, size=actual_size)
dst = str(tmpdir.join("dst.qcow2"))
# When downloading qcow2 image using the nbd backend, we get raw data and
# we can convert it to any format we want. Howver when downloading using
# the file backend, we get qcow2 bytestream and we cannot convert it.
#
# To store the qcow2 bytestream, we must use fmt="raw". This instructs
# qemu-nbd on the client side to treat the data as raw bytes, storing them
# without any change on the local file.
#
# This is baisically like:
#
# qemu-img convert -f raw -O raw src.qcow2 dst.qcow2
#
client.download(url, dst, srv.config.tls.ca_file, fmt="raw")
# The result should be identical qcow2 image content. Allocation may
# differ but for this test we get identical allocation.
qemu_img.compare(src, dst, format1="qcow2", format2="qcow2", strict=True)
def test_upload_from_ova(tmpdir, srv, fmt, compressed):
offset = CLUSTER_SIZE
data = b"I can eat glass and it doesn't hurt me."
# Create raw disk with some data.
tmp = str(tmpdir.join("tmp"))
with open(tmp, "wb") as f:
f.truncate(IMAGE_SIZE)
f.seek(offset)
f.write(data)
# Create source disk.
src = str(tmpdir.join("src"))
qemu_img.convert(tmp, src, "raw", fmt, compressed=compressed)
# Create OVA package.
ova = str(tmpdir.join("src.ova"))
with tarfile.open(ova, "w") as tar:
tar.add(src, arcname=os.path.basename(src))
# Prepare destination file.
dst = str(tmpdir.join("dst"))
with open(dst, "wb") as f:
f.truncate(IMAGE_SIZE)
# Test uploading src from ova.
url = prepare_transfer(srv, "file://" + dst)
client.upload(ova,
url,
srv.config.tls.ca_file,
member=os.path.basename(src))
qemu_img.compare(src, dst)
def test_options(tmpdir, fmt, options):
size = 4 * 1024**2
chunk_size = 128 * 1024
src = str(tmpdir.join("src." + fmt))
qemu_img.create(src, fmt, size=size)
with qemu_nbd.open(src, fmt) as c:
for offset in range(0, size, chunk_size):
c.write(offset, struct.pack(">Q", offset))
c.flush()
dst = str(tmpdir.join("dst." + fmt))
qemu_img.create(dst, fmt, size=size)
src_addr = nbd.UnixAddress(str(tmpdir.join("src.sock")))
dst_addr = nbd.UnixAddress(str(tmpdir.join("dst.sock")))
with qemu_nbd.run(
src, fmt, src_addr,
read_only=True,
**options), \
qemu_nbd.run(
dst, fmt, dst_addr,
**options), \
nbd.Client(src_addr) as src_client, \
nbd.Client(dst_addr) as dst_client:
nbdutil.copy(src_client, dst_client)
qemu_img.compare(src, dst)
def test_compare_missing_file(tmpdir):
src = str(tmpdir.join("src.raw"))
dst = str(tmpdir.join("dst.raw"))
qemu_img.create(src, "raw", size=1024**2)
with pytest.raises(qemu_img.OpenImageError):
qemu_img.compare(src, dst)
def test_download_shallow(srv, nbd_server, tmpdir, base_fmt):
size = 10 * 1024**2
# Create source base image with some data in first clusters.
src_base = str(tmpdir.join("src_base." + base_fmt))
qemu_img.create(src_base, base_fmt, size=size)
with qemu_nbd.open(src_base, base_fmt) as c:
c.write(0 * CLUSTER_SIZE, b"a" * CLUSTER_SIZE)
c.write(1 * CLUSTER_SIZE, b"b" * CLUSTER_SIZE)
c.write(2 * CLUSTER_SIZE, b"c" * CLUSTER_SIZE)
c.flush()
# Create source top image with some data in second cluster and zero in the
# third cluster.
src_top = str(tmpdir.join("src_top.qcow2"))
qemu_img.create(src_top,
"qcow2",
backing_file=src_base,
backing_format=base_fmt)
with qemu_nbd.open(src_top, "qcow2") as c:
c.write(1 * CLUSTER_SIZE, b"B" * CLUSTER_SIZE)
c.zero(2 * CLUSTER_SIZE, CLUSTER_SIZE)
c.flush()
# Create empty backing file for downloding top image.
dst_base = str(tmpdir.join("dst_base." + base_fmt))
qemu_img.create(dst_base, base_fmt, size=size)
dst_top = str(tmpdir.join("dst_top.qcow2"))
# Start nbd server exporting top image without the backing file.
nbd_server.image = src_top
nbd_server.fmt = "qcow2"
nbd_server.backing_chain = False
nbd_server.shared = 8
nbd_server.start()
# Upload using nbd backend.
url = prepare_transfer(srv, nbd_server.sock.url(), size=size)
client.download(url,
dst_top,
srv.config.tls.ca_file,
backing_file=dst_base,
backing_format=base_fmt)
# Stop the server to allow comparing.
nbd_server.stop()
# To compare we need to remove its backing files.
qemu_img.unsafe_rebase(src_top, "")
qemu_img.unsafe_rebase(dst_top, "")
qemu_img.compare(src_top,
dst_top,
format1="qcow2",
format2="qcow2",
strict=True)
def test_copy_nbd_to_nbd(tmpdir, src_fmt, dst_fmt, zero, hole):
# Default cluser size with qcow2 format.
cluster_size = 64 * 1024
extents = [
("data", cluster_size),
("zero", cluster_size),
("data", cluster_size),
("hole", cluster_size + io.MAX_ZERO_SIZE),
("data", cluster_size + io.BUFFER_SIZE),
("hole", cluster_size),
("data", cluster_size),
]
size = sum(length for _, length in extents)
src = str(tmpdir.join("src." + src_fmt))
qemu_img.create(src, src_fmt, size=size)
populate_image(src, src_fmt, extents)
src_sock = UnixAddress(tmpdir.join("src.sock"))
src_url = urlparse(src_sock.url())
dst = str(tmpdir.join("dst." + dst_fmt))
qemu_img.create(dst, dst_fmt, size=size)
dst_sock = UnixAddress(tmpdir.join("dst.sock"))
dst_url = urlparse(dst_sock.url())
# Note: We need extra worker for reading extents for source.
max_workers = 2
with qemu_nbd.run(
src, src_fmt, src_sock,
read_only=True,
shared=max_workers + 1), \
qemu_nbd.run(
dst, dst_fmt, dst_sock,
shared=max_workers), \
nbd.open(src_url, "r") as src_backend, \
nbd.open(dst_url, "r+", sparse=True) as dst_backend:
# Because we copy to new image, we can always use zero=False, but we
# test both to verify that the result is the same.
io.copy(src_backend,
dst_backend,
max_workers=max_workers,
zero=zero,
hole=hole)
# Compare image content - must match.
qemu_img.compare(src, dst)
# Allocation can be compared only with qcow2 images when we write zeroes to
# zero extents and skip holes.
if src_fmt == "qcow2" and dst_fmt == "qcow2" and zero and not hole:
qemu_img.compare(src, dst, strict=True)
def test_compare_wrong_format(tmpdir):
size = 1024**2
src = str(tmpdir.join("src.raw"))
dst = str(tmpdir.join("dst.raw"))
qemu_img.create(src, "raw", size=size)
qemu_img.create(dst, "raw", size=size)
with pytest.raises(qemu_img.OpenImageError):
qemu_img.compare(src, dst, format1="qcow2")
with pytest.raises(qemu_img.OpenImageError):
qemu_img.compare(src, dst, format2="qcow2")
def test_upload_preallocated(tmpdir, srv, fmt):
src = str(tmpdir.join("src"))
qemu_img.create(src, fmt, size=IMAGE_SIZE)
dst = str(tmpdir.join("dst"))
with open(dst, "wb") as f:
f.write(b"a" * IMAGE_SIZE)
url = prepare_transfer(srv, "file://" + dst, sparse=False)
client.upload(src, url, srv.config.tls.ca_file)
qemu_img.compare(src, dst)
assert os.stat(dst).st_blocks * 512 == IMAGE_SIZE
def test_compare_identical_content(tmpdir, src_fmt, dst_fmt):
size = 1024**2
src = str(tmpdir.join("src." + src_fmt))
dst = str(tmpdir.join("dst." + dst_fmt))
qemu_img.create(src, src_fmt, size=size)
qemu_img.create(dst, dst_fmt, size=size)
# Destination image has different allocation.
with qemu_nbd.open(dst, dst_fmt) as c:
c.write(size // 2, b"\0")
c.flush()
qemu_img.compare(src, dst, format1=src_fmt, format2=dst_fmt)
def test_compare_different_content(tmpdir, src_fmt, dst_fmt):
size = 1024**2
src = str(tmpdir.join("src." + src_fmt))
dst = str(tmpdir.join("dst." + dst_fmt))
qemu_img.create(src, src_fmt, size=size)
qemu_img.create(dst, dst_fmt, size=size)
# Destination image has different content.
with qemu_nbd.open(dst, dst_fmt) as c:
c.write(size // 2, b"x")
c.flush()
with pytest.raises(qemu_img.ContentMismatch):
qemu_img.compare(src, dst, format1=src_fmt, format2=dst_fmt)
def test_download_proxy_url_unused(tmpdir, srv):
src = str(tmpdir.join("src"))
with open(src, "wb") as f:
f.truncate(IMAGE_SIZE)
dst = str(tmpdir.join("dst"))
# If transfer_url is accessible, proxy_url is not used.
transfer_url = prepare_transfer(srv, "file://" + src)
proxy_url = "https://no.proxy:54322/images/no-ticket"
client.download(transfer_url, dst, srv.config.tls.ca_file, fmt="raw",
proxy_url=proxy_url)
qemu_img.compare(src, dst, format1="raw", format2="raw")
def test_upload_empty_sparse(tmpdir, srv, fmt):
src = str(tmpdir.join("src"))
qemu_img.create(src, fmt, size=IMAGE_SIZE)
dst = str(tmpdir.join("dst"))
with open(dst, "wb") as f:
f.write(b"a" * IMAGE_SIZE)
url = prepare_transfer(srv, "file://" + dst)
client.upload(src, url, srv.config.tls.ca_file)
# TODO: Check why allocation differ when src is qcow2. Target image
# allocation is 0 bytes as expected, but comparing with strict=True fail at
# offset 0.
qemu_img.compare(src, dst, format1=fmt, format2="raw", strict=fmt == "raw")
def test_download_raw(tmpdir, srv, fmt):
src = str(tmpdir.join("src"))
with open(src, "wb") as f:
f.truncate(IMAGE_SIZE)
f.seek(IMAGE_SIZE // 2)
f.write(b"data")
url = prepare_transfer(srv, "file://" + src)
dst = str(tmpdir.join("dst"))
# When we download raw data, we can convert it on-the-fly to other format.
client.download(url, dst, srv.config.tls.ca_file, fmt=fmt)
# file backend does not support extents, so downloaded data is always
# fully allocated.
qemu_img.compare(src, dst, format1="raw", format2=fmt)
def test_compare_different_allocation(tmpdir, src_fmt, dst_fmt):
# Images has same content, but different allocation.
size = 1024**2
src = str(tmpdir.join("src." + src_fmt))
dst = str(tmpdir.join("dst." + dst_fmt))
qemu_img.create(src, src_fmt, size=size)
qemu_img.create(dst, dst_fmt, size=size)
with qemu_nbd.open(dst, dst_fmt) as c:
c.write(size // 2, b"\0")
c.flush()
with pytest.raises(qemu_img.ContentMismatch):
qemu_img.compare(
src, dst, format1=src_fmt, format2=dst_fmt, strict=True)
def test_upload_hole_at_start_sparse(tmpdir, srv, fmt):
src = str(tmpdir.join("src"))
qemu_img.create(src, fmt, size=IMAGE_SIZE)
with qemu_nbd.open(src, fmt) as c:
c.write(IMAGE_SIZE - 6, b"middle")
c.flush()
dst = str(tmpdir.join("dst"))
with open(dst, "wb") as f:
f.write(b"a" * IMAGE_SIZE)
url = prepare_transfer(srv, "file://" + dst)
client.upload(src, url, srv.config.tls.ca_file)
qemu_img.compare(src, dst, format1=fmt, format2="raw", strict=fmt == "raw")
def test_upload_proxy_url(tmpdir, srv):
src = str(tmpdir.join("src"))
with open(src, "wb") as f:
f.truncate(IMAGE_SIZE)
dst = str(tmpdir.join("dst"))
with open(dst, "wb") as f:
f.truncate(IMAGE_SIZE)
# If transfer_url is not accessible, proxy_url is used.
transfer_url = "https://no.server:54322/images/no-ticket"
proxy_url = prepare_transfer(srv, "file://" + dst)
client.upload(src, transfer_url, srv.config.tls.ca_file,
proxy_url=proxy_url)
qemu_img.compare(src, dst, format1="raw", format2="raw", strict=True)
def test_upload_full_sparse(tmpdir, srv, fmt):
src = str(tmpdir.join("src"))
qemu_img.create(src, fmt, size=IMAGE_SIZE)
with qemu_nbd.open(src, fmt) as c:
c.write(0, b"b" * IMAGE_SIZE)
c.flush()
dst = str(tmpdir.join("dst"))
with open(dst, "wb") as f:
f.write(b"a" * IMAGE_SIZE)
url = prepare_transfer(srv, "file://" + dst)
client.upload(src, url, srv.config.tls.ca_file)
qemu_img.compare(src, dst, strict=True)
def test_upload_hole_at_end_sparse(tmpdir, srv, fmt):
size = 3 * 1024**2
src = str(tmpdir.join("src"))
qemu_img.create(src, fmt, size=size)
with qemu_nbd.open(src, fmt) as c:
c.write(0, b"b" * 1024**2)
c.flush()
dst = str(tmpdir.join("dst"))
with open(dst, "wb") as f:
f.write(b"a" * size)
url = prepare_transfer(srv, "file://" + dst, size=size)
client.upload(src, url, srv.config.tls.ca_file)
qemu_img.compare(src, dst, format1=fmt, format2="raw", strict=fmt == "raw")
def test_shared(tmpdir, fmt):
size = 1024**2
chunk_size = size // 2
src = str(tmpdir.join("src." + fmt))
qemu_img.create(src, fmt, size=size)
with qemu_nbd.open(src, fmt) as c:
c.write(0, b"a" * chunk_size)
c.write(0, b"b" * chunk_size)
c.flush()
dst = str(tmpdir.join("dst." + fmt))
qemu_img.create(dst, fmt, size=size)
src_addr = nbd.UnixAddress(str(tmpdir.join("src.sock")))
dst_addr = nbd.UnixAddress(str(tmpdir.join("dst.sock")))
# Start 2 qemu-nbd servers, each with 2 clients that can read and write in
# parallel for higher throughput.
with qemu_nbd.run(src, fmt, src_addr, read_only=True, shared=2), \
qemu_nbd.run(dst, fmt, dst_addr, shared=2), \
nbd.Client(src_addr) as src_client1, \
nbd.Client(src_addr) as src_client2, \
nbd.Client(dst_addr) as dst_client1, \
nbd.Client(dst_addr) as dst_client2:
# Copy first half of the image with src_client1 and dst_client2 and
# second half with src_client2 and dst_client2. In a real application
# we would have more clients, running in multiple threads.
chunk1 = src_client1.read(0, chunk_size)
dst_client1.write(0, chunk1)
chunk2 = src_client2.read(chunk_size, chunk_size)
dst_client2.write(chunk_size, chunk2)
dst_client1.flush()
dst_client2.flush()
qemu_img.compare(src, dst)
def test_upload_hole_at_start_sparse(tmpdir, srv, fmt):
size = 3 * 1024**2
src = str(tmpdir.join("src"))
qemu_img.create(src, fmt, size=size)
with qemu_nbd.open(src, fmt) as c:
c.write(size - 1024**2, b"b" * 1024**2)
c.flush()
log.debug("src extents: %s", list(nbdutil.extents(c)))
dst = str(tmpdir.join("dst"))
with open(dst, "wb") as f:
f.write(b"a" * size)
url = prepare_transfer(srv, "file://" + dst, size=size)
client.upload(src, url, srv.config.tls.ca_file)
with qemu_nbd.open(dst, "raw", read_only=True) as c:
log.debug("dst extents: %s", list(nbdutil.extents(c)))
qemu_img.compare(src, dst, format1=fmt, format2="raw", strict=fmt == "raw")
def test_upload_shallow(srv, nbd_server, tmpdir, base_fmt):
size = 10 * 1024**2
# Create base image with some data in first 3 clusters.
src_base = str(tmpdir.join("src_base." + base_fmt))
qemu_img.create(src_base, base_fmt, size=size)
with qemu_nbd.open(src_base, base_fmt) as c:
c.write(0 * CLUSTER_SIZE, b"a" * CLUSTER_SIZE)
c.write(1 * CLUSTER_SIZE, b"b" * CLUSTER_SIZE)
c.write(2 * CLUSTER_SIZE, b"c" * CLUSTER_SIZE)
c.flush()
log.debug("src_base extents: %s", list(nbdutil.extents(c)))
# Create src image with some data in second cluster and zero in third
# cluster.
src_top = str(tmpdir.join("src_top.qcow2"))
qemu_img.create(
src_top, "qcow2", backing_file=src_base, backing_format=base_fmt)
with qemu_nbd.open(src_top, "qcow2") as c:
c.write(1 * CLUSTER_SIZE, b"B" * CLUSTER_SIZE)
c.zero(2 * CLUSTER_SIZE, CLUSTER_SIZE)
c.flush()
with qemu_nbd.open(
src_top, "qcow2", read_only=True, backing_chain=False) as c:
log.debug("src_top extents: %s", list(nbdutil.extents(c)))
# Create empty destination base image.
dst_base = str(tmpdir.join("dst_base." + base_fmt))
qemu_img.create(dst_base, base_fmt, size=size)
# Create empty destination top image.
dst_top = str(tmpdir.join("dst_top.qcow2"))
qemu_img.create(
dst_top, "qcow2", backing_file=dst_base, backing_format=base_fmt)
# Upload base image.
nbd_server.image = dst_base
nbd_server.fmt = base_fmt
nbd_server.start()
url = prepare_transfer(srv, nbd_server.sock.url(), size=size)
client.upload(
src_base,
url,
srv.config.tls.ca_file,
backing_chain=False)
nbd_server.stop()
with qemu_nbd.open(dst_base, base_fmt, read_only=True) as c:
log.debug("dst_base extents: %s", list(nbdutil.extents(c)))
# Compare image content - must match.
qemu_img.compare(
src_base, dst_base, format1=base_fmt, format2=base_fmt, strict=False)
# Comparing allocation is not possible with raw format since qemu-nbd does
# not report allocation for raw images, so we treat unallocated areas as
# zero area instead of a hole.
if base_fmt == "qcow2":
qemu_img.compare(
src_base, dst_base, format1=base_fmt, format2=base_fmt,
strict=True)
# Upload top image.
nbd_server.image = dst_top
nbd_server.fmt = "qcow2"
nbd_server.start()
url = prepare_transfer(srv, nbd_server.sock.url(), size=size)
client.upload(
src_top,
url,
srv.config.tls.ca_file,
backing_chain=False)
nbd_server.stop()
with qemu_nbd.open(
dst_top, "qcow2", read_only=True, backing_chain=False) as c:
log.debug("dst_top extents: %s", list(nbdutil.extents(c)))
# Test image content - must match.
qemu_img.compare(
src_top, dst_top, format1="qcow2", format2="qcow2", strict=False)
# Compare allocation for qcow2 chain - nice to have.
if base_fmt == "qcow2":
qemu_img.compare(
src_top, dst_top, format1="qcow2", format2="qcow2", strict=True)
def test_download_shallow(srv, nbd_server, tmpdir, base_fmt):
size = 10 * 1024**2
# Create source base image with some data in first clusters.
src_base = str(tmpdir.join("src_base." + base_fmt))
qemu_img.create(src_base, base_fmt, size=size)
with qemu_nbd.open(src_base, base_fmt) as c:
c.write(0 * CLUSTER_SIZE, b"a" * CLUSTER_SIZE)
c.write(1 * CLUSTER_SIZE, b"b" * CLUSTER_SIZE)
c.write(2 * CLUSTER_SIZE, b"c" * CLUSTER_SIZE)
c.flush()
log.debug("src_base extents: %s", list(nbdutil.extents(c)))
# Create source top image with some data in second cluster and zero in the
# third cluster.
src_top = str(tmpdir.join("src_top.qcow2"))
qemu_img.create(
src_top, "qcow2", backing_file=src_base, backing_format=base_fmt)
with qemu_nbd.open(src_top, "qcow2") as c:
c.write(1 * CLUSTER_SIZE, b"B" * CLUSTER_SIZE)
c.zero(2 * CLUSTER_SIZE, CLUSTER_SIZE)
c.flush()
with qemu_nbd.open(
src_top, "qcow2", read_only=True, backing_chain=False) as c:
log.debug("src_top extents: %s", list(nbdutil.extents(c)))
dst_base = str(tmpdir.join("dst_base." + base_fmt))
dst_top = str(tmpdir.join("dst_top.qcow2"))
# Download base image.
nbd_server.image = src_base
nbd_server.fmt = base_fmt
nbd_server.start()
url = prepare_transfer(srv, nbd_server.sock.url(), size=size)
client.download(
url,
dst_base,
srv.config.tls.ca_file,
fmt=base_fmt)
nbd_server.stop()
with qemu_nbd.open(dst_base, base_fmt, read_only=True) as c:
log.debug("dst_base extents: %s", list(nbdutil.extents(c)))
# Compare image content - must match.
qemu_img.compare(
src_base, dst_base, format1=base_fmt, format2=base_fmt, strict=False)
# And allocation - nice to have.
if base_fmt == "qcow2" or qemu_nbd.version() >= (6, 0, 0):
qemu_img.compare(
src_base, dst_base, format1=base_fmt, format2=base_fmt,
strict=True)
# Download top image.
nbd_server.image = src_top
nbd_server.fmt = "qcow2"
nbd_server.backing_chain = False
nbd_server.start()
url = prepare_transfer(srv, nbd_server.sock.url(), size=size)
client.download(
url,
dst_top,
srv.config.tls.ca_file,
backing_file=dst_base,
backing_format=base_fmt)
nbd_server.stop()
with qemu_nbd.open(
dst_top, "qcow2", read_only=True, backing_chain=False) as c:
log.debug("dst_top extents: %s", list(nbdutil.extents(c)))
# Compare both image content - must match.
qemu_img.compare(
src_top, dst_top, format1="qcow2", format2="qcow2", strict=False)
# And allocation - nice to have.
qemu_img.compare(
src_top, dst_top, format1="qcow2", format2="qcow2", strict=True)
