def test_download_qcow2_as_raw(tmpdir, srv): src = str(tmpdir.join("src.qcow2")) qemu_img.create(src, "qcow2", size=IMAGE_SIZE) # Allocate one cluster in the middle of the image. with qemu_nbd.open(src, "qcow2") as c: c.write(CLUSTER_SIZE, b"a" * CLUSTER_SIZE) c.flush() actual_size = os.path.getsize(src) url = prepare_transfer(srv, "file://" + src, size=actual_size) dst = str(tmpdir.join("dst.qcow2")) # When downloading qcow2 image using the nbd backend, we get raw data and # we can convert it to any format we want. Howver when downloading using # the file backend, we get qcow2 bytestream and we cannot convert it. # # To store the qcow2 bytestream, we must use fmt="raw". This instructs # qemu-nbd on the client side to treat the data as raw bytes, storing them # without any change on the local file. # # This is baisically like: # # qemu-img convert -f raw -O raw src.qcow2 dst.qcow2 # client.download(url, dst, srv.config.tls.ca_file, fmt="raw") # The result should be identical qcow2 image content. Allocation may # differ but for this test we get identical allocation. qemu_img.compare(src, dst, format1="qcow2", format2="qcow2", strict=True)
def test_upload_from_ova(tmpdir, srv, fmt, compressed): offset = CLUSTER_SIZE data = b"I can eat glass and it doesn't hurt me." # Create raw disk with some data. tmp = str(tmpdir.join("tmp")) with open(tmp, "wb") as f: f.truncate(IMAGE_SIZE) f.seek(offset) f.write(data) # Create source disk. src = str(tmpdir.join("src")) qemu_img.convert(tmp, src, "raw", fmt, compressed=compressed) # Create OVA package. ova = str(tmpdir.join("src.ova")) with tarfile.open(ova, "w") as tar: tar.add(src, arcname=os.path.basename(src)) # Prepare destination file. dst = str(tmpdir.join("dst")) with open(dst, "wb") as f: f.truncate(IMAGE_SIZE) # Test uploading src from ova. url = prepare_transfer(srv, "file://" + dst) client.upload(ova, url, srv.config.tls.ca_file, member=os.path.basename(src)) qemu_img.compare(src, dst)
def test_options(tmpdir, fmt, options): size = 4 * 1024**2 chunk_size = 128 * 1024 src = str(tmpdir.join("src." + fmt)) qemu_img.create(src, fmt, size=size) with qemu_nbd.open(src, fmt) as c: for offset in range(0, size, chunk_size): c.write(offset, struct.pack(">Q", offset)) c.flush() dst = str(tmpdir.join("dst." + fmt)) qemu_img.create(dst, fmt, size=size) src_addr = nbd.UnixAddress(str(tmpdir.join("src.sock"))) dst_addr = nbd.UnixAddress(str(tmpdir.join("dst.sock"))) with qemu_nbd.run( src, fmt, src_addr, read_only=True, **options), \ qemu_nbd.run( dst, fmt, dst_addr, **options), \ nbd.Client(src_addr) as src_client, \ nbd.Client(dst_addr) as dst_client: nbdutil.copy(src_client, dst_client) qemu_img.compare(src, dst)
def test_compare_missing_file(tmpdir): src = str(tmpdir.join("src.raw")) dst = str(tmpdir.join("dst.raw")) qemu_img.create(src, "raw", size=1024**2) with pytest.raises(qemu_img.OpenImageError): qemu_img.compare(src, dst)
def test_download_shallow(srv, nbd_server, tmpdir, base_fmt): size = 10 * 1024**2 # Create source base image with some data in first clusters. src_base = str(tmpdir.join("src_base." + base_fmt)) qemu_img.create(src_base, base_fmt, size=size) with qemu_nbd.open(src_base, base_fmt) as c: c.write(0 * CLUSTER_SIZE, b"a" * CLUSTER_SIZE) c.write(1 * CLUSTER_SIZE, b"b" * CLUSTER_SIZE) c.write(2 * CLUSTER_SIZE, b"c" * CLUSTER_SIZE) c.flush() # Create source top image with some data in second cluster and zero in the # third cluster. src_top = str(tmpdir.join("src_top.qcow2")) qemu_img.create(src_top, "qcow2", backing_file=src_base, backing_format=base_fmt) with qemu_nbd.open(src_top, "qcow2") as c: c.write(1 * CLUSTER_SIZE, b"B" * CLUSTER_SIZE) c.zero(2 * CLUSTER_SIZE, CLUSTER_SIZE) c.flush() # Create empty backing file for downloding top image. dst_base = str(tmpdir.join("dst_base." + base_fmt)) qemu_img.create(dst_base, base_fmt, size=size) dst_top = str(tmpdir.join("dst_top.qcow2")) # Start nbd server exporting top image without the backing file. nbd_server.image = src_top nbd_server.fmt = "qcow2" nbd_server.backing_chain = False nbd_server.shared = 8 nbd_server.start() # Upload using nbd backend. url = prepare_transfer(srv, nbd_server.sock.url(), size=size) client.download(url, dst_top, srv.config.tls.ca_file, backing_file=dst_base, backing_format=base_fmt) # Stop the server to allow comparing. nbd_server.stop() # To compare we need to remove its backing files. qemu_img.unsafe_rebase(src_top, "") qemu_img.unsafe_rebase(dst_top, "") qemu_img.compare(src_top, dst_top, format1="qcow2", format2="qcow2", strict=True)
def test_copy_nbd_to_nbd(tmpdir, src_fmt, dst_fmt, zero, hole): # Default cluser size with qcow2 format. cluster_size = 64 * 1024 extents = [ ("data", cluster_size), ("zero", cluster_size), ("data", cluster_size), ("hole", cluster_size + io.MAX_ZERO_SIZE), ("data", cluster_size + io.BUFFER_SIZE), ("hole", cluster_size), ("data", cluster_size), ] size = sum(length for _, length in extents) src = str(tmpdir.join("src." + src_fmt)) qemu_img.create(src, src_fmt, size=size) populate_image(src, src_fmt, extents) src_sock = UnixAddress(tmpdir.join("src.sock")) src_url = urlparse(src_sock.url()) dst = str(tmpdir.join("dst." + dst_fmt)) qemu_img.create(dst, dst_fmt, size=size) dst_sock = UnixAddress(tmpdir.join("dst.sock")) dst_url = urlparse(dst_sock.url()) # Note: We need extra worker for reading extents for source. max_workers = 2 with qemu_nbd.run( src, src_fmt, src_sock, read_only=True, shared=max_workers + 1), \ qemu_nbd.run( dst, dst_fmt, dst_sock, shared=max_workers), \ nbd.open(src_url, "r") as src_backend, \ nbd.open(dst_url, "r+", sparse=True) as dst_backend: # Because we copy to new image, we can always use zero=False, but we # test both to verify that the result is the same. io.copy(src_backend, dst_backend, max_workers=max_workers, zero=zero, hole=hole) # Compare image content - must match. qemu_img.compare(src, dst) # Allocation can be compared only with qcow2 images when we write zeroes to # zero extents and skip holes. if src_fmt == "qcow2" and dst_fmt == "qcow2" and zero and not hole: qemu_img.compare(src, dst, strict=True)
def test_compare_wrong_format(tmpdir): size = 1024**2 src = str(tmpdir.join("src.raw")) dst = str(tmpdir.join("dst.raw")) qemu_img.create(src, "raw", size=size) qemu_img.create(dst, "raw", size=size) with pytest.raises(qemu_img.OpenImageError): qemu_img.compare(src, dst, format1="qcow2") with pytest.raises(qemu_img.OpenImageError): qemu_img.compare(src, dst, format2="qcow2")
def test_upload_preallocated(tmpdir, srv, fmt): src = str(tmpdir.join("src")) qemu_img.create(src, fmt, size=IMAGE_SIZE) dst = str(tmpdir.join("dst")) with open(dst, "wb") as f: f.write(b"a" * IMAGE_SIZE) url = prepare_transfer(srv, "file://" + dst, sparse=False) client.upload(src, url, srv.config.tls.ca_file) qemu_img.compare(src, dst) assert os.stat(dst).st_blocks * 512 == IMAGE_SIZE
def test_compare_identical_content(tmpdir, src_fmt, dst_fmt): size = 1024**2 src = str(tmpdir.join("src." + src_fmt)) dst = str(tmpdir.join("dst." + dst_fmt)) qemu_img.create(src, src_fmt, size=size) qemu_img.create(dst, dst_fmt, size=size) # Destination image has different allocation. with qemu_nbd.open(dst, dst_fmt) as c: c.write(size // 2, b"\0") c.flush() qemu_img.compare(src, dst, format1=src_fmt, format2=dst_fmt)
def test_compare_different_content(tmpdir, src_fmt, dst_fmt): size = 1024**2 src = str(tmpdir.join("src." + src_fmt)) dst = str(tmpdir.join("dst." + dst_fmt)) qemu_img.create(src, src_fmt, size=size) qemu_img.create(dst, dst_fmt, size=size) # Destination image has different content. with qemu_nbd.open(dst, dst_fmt) as c: c.write(size // 2, b"x") c.flush() with pytest.raises(qemu_img.ContentMismatch): qemu_img.compare(src, dst, format1=src_fmt, format2=dst_fmt)
def test_download_proxy_url_unused(tmpdir, srv): src = str(tmpdir.join("src")) with open(src, "wb") as f: f.truncate(IMAGE_SIZE) dst = str(tmpdir.join("dst")) # If transfer_url is accessible, proxy_url is not used. transfer_url = prepare_transfer(srv, "file://" + src) proxy_url = "https://no.proxy:54322/images/no-ticket" client.download(transfer_url, dst, srv.config.tls.ca_file, fmt="raw", proxy_url=proxy_url) qemu_img.compare(src, dst, format1="raw", format2="raw")
def test_upload_empty_sparse(tmpdir, srv, fmt): src = str(tmpdir.join("src")) qemu_img.create(src, fmt, size=IMAGE_SIZE) dst = str(tmpdir.join("dst")) with open(dst, "wb") as f: f.write(b"a" * IMAGE_SIZE) url = prepare_transfer(srv, "file://" + dst) client.upload(src, url, srv.config.tls.ca_file) # TODO: Check why allocation differ when src is qcow2. Target image # allocation is 0 bytes as expected, but comparing with strict=True fail at # offset 0. qemu_img.compare(src, dst, format1=fmt, format2="raw", strict=fmt == "raw")
def test_download_raw(tmpdir, srv, fmt): src = str(tmpdir.join("src")) with open(src, "wb") as f: f.truncate(IMAGE_SIZE) f.seek(IMAGE_SIZE // 2) f.write(b"data") url = prepare_transfer(srv, "file://" + src) dst = str(tmpdir.join("dst")) # When we download raw data, we can convert it on-the-fly to other format. client.download(url, dst, srv.config.tls.ca_file, fmt=fmt) # file backend does not support extents, so downloaded data is always # fully allocated. qemu_img.compare(src, dst, format1="raw", format2=fmt)
def test_compare_different_allocation(tmpdir, src_fmt, dst_fmt): # Images has same content, but different allocation. size = 1024**2 src = str(tmpdir.join("src." + src_fmt)) dst = str(tmpdir.join("dst." + dst_fmt)) qemu_img.create(src, src_fmt, size=size) qemu_img.create(dst, dst_fmt, size=size) with qemu_nbd.open(dst, dst_fmt) as c: c.write(size // 2, b"\0") c.flush() with pytest.raises(qemu_img.ContentMismatch): qemu_img.compare( src, dst, format1=src_fmt, format2=dst_fmt, strict=True)
def test_upload_hole_at_start_sparse(tmpdir, srv, fmt): src = str(tmpdir.join("src")) qemu_img.create(src, fmt, size=IMAGE_SIZE) with qemu_nbd.open(src, fmt) as c: c.write(IMAGE_SIZE - 6, b"middle") c.flush() dst = str(tmpdir.join("dst")) with open(dst, "wb") as f: f.write(b"a" * IMAGE_SIZE) url = prepare_transfer(srv, "file://" + dst) client.upload(src, url, srv.config.tls.ca_file) qemu_img.compare(src, dst, format1=fmt, format2="raw", strict=fmt == "raw")
def test_upload_proxy_url(tmpdir, srv): src = str(tmpdir.join("src")) with open(src, "wb") as f: f.truncate(IMAGE_SIZE) dst = str(tmpdir.join("dst")) with open(dst, "wb") as f: f.truncate(IMAGE_SIZE) # If transfer_url is not accessible, proxy_url is used. transfer_url = "https://no.server:54322/images/no-ticket" proxy_url = prepare_transfer(srv, "file://" + dst) client.upload(src, transfer_url, srv.config.tls.ca_file, proxy_url=proxy_url) qemu_img.compare(src, dst, format1="raw", format2="raw", strict=True)
def test_upload_full_sparse(tmpdir, srv, fmt): src = str(tmpdir.join("src")) qemu_img.create(src, fmt, size=IMAGE_SIZE) with qemu_nbd.open(src, fmt) as c: c.write(0, b"b" * IMAGE_SIZE) c.flush() dst = str(tmpdir.join("dst")) with open(dst, "wb") as f: f.write(b"a" * IMAGE_SIZE) url = prepare_transfer(srv, "file://" + dst) client.upload(src, url, srv.config.tls.ca_file) qemu_img.compare(src, dst, strict=True)
def test_upload_hole_at_end_sparse(tmpdir, srv, fmt): size = 3 * 1024**2 src = str(tmpdir.join("src")) qemu_img.create(src, fmt, size=size) with qemu_nbd.open(src, fmt) as c: c.write(0, b"b" * 1024**2) c.flush() dst = str(tmpdir.join("dst")) with open(dst, "wb") as f: f.write(b"a" * size) url = prepare_transfer(srv, "file://" + dst, size=size) client.upload(src, url, srv.config.tls.ca_file) qemu_img.compare(src, dst, format1=fmt, format2="raw", strict=fmt == "raw")
def test_shared(tmpdir, fmt): size = 1024**2 chunk_size = size // 2 src = str(tmpdir.join("src." + fmt)) qemu_img.create(src, fmt, size=size) with qemu_nbd.open(src, fmt) as c: c.write(0, b"a" * chunk_size) c.write(0, b"b" * chunk_size) c.flush() dst = str(tmpdir.join("dst." + fmt)) qemu_img.create(dst, fmt, size=size) src_addr = nbd.UnixAddress(str(tmpdir.join("src.sock"))) dst_addr = nbd.UnixAddress(str(tmpdir.join("dst.sock"))) # Start 2 qemu-nbd servers, each with 2 clients that can read and write in # parallel for higher throughput. with qemu_nbd.run(src, fmt, src_addr, read_only=True, shared=2), \ qemu_nbd.run(dst, fmt, dst_addr, shared=2), \ nbd.Client(src_addr) as src_client1, \ nbd.Client(src_addr) as src_client2, \ nbd.Client(dst_addr) as dst_client1, \ nbd.Client(dst_addr) as dst_client2: # Copy first half of the image with src_client1 and dst_client2 and # second half with src_client2 and dst_client2. In a real application # we would have more clients, running in multiple threads. chunk1 = src_client1.read(0, chunk_size) dst_client1.write(0, chunk1) chunk2 = src_client2.read(chunk_size, chunk_size) dst_client2.write(chunk_size, chunk2) dst_client1.flush() dst_client2.flush() qemu_img.compare(src, dst)
def test_upload_hole_at_start_sparse(tmpdir, srv, fmt): size = 3 * 1024**2 src = str(tmpdir.join("src")) qemu_img.create(src, fmt, size=size) with qemu_nbd.open(src, fmt) as c: c.write(size - 1024**2, b"b" * 1024**2) c.flush() log.debug("src extents: %s", list(nbdutil.extents(c))) dst = str(tmpdir.join("dst")) with open(dst, "wb") as f: f.write(b"a" * size) url = prepare_transfer(srv, "file://" + dst, size=size) client.upload(src, url, srv.config.tls.ca_file) with qemu_nbd.open(dst, "raw", read_only=True) as c: log.debug("dst extents: %s", list(nbdutil.extents(c))) qemu_img.compare(src, dst, format1=fmt, format2="raw", strict=fmt == "raw")
def test_upload_shallow(srv, nbd_server, tmpdir, base_fmt): size = 10 * 1024**2 # Create base image with some data in first 3 clusters. src_base = str(tmpdir.join("src_base." + base_fmt)) qemu_img.create(src_base, base_fmt, size=size) with qemu_nbd.open(src_base, base_fmt) as c: c.write(0 * CLUSTER_SIZE, b"a" * CLUSTER_SIZE) c.write(1 * CLUSTER_SIZE, b"b" * CLUSTER_SIZE) c.write(2 * CLUSTER_SIZE, b"c" * CLUSTER_SIZE) c.flush() log.debug("src_base extents: %s", list(nbdutil.extents(c))) # Create src image with some data in second cluster and zero in third # cluster. src_top = str(tmpdir.join("src_top.qcow2")) qemu_img.create( src_top, "qcow2", backing_file=src_base, backing_format=base_fmt) with qemu_nbd.open(src_top, "qcow2") as c: c.write(1 * CLUSTER_SIZE, b"B" * CLUSTER_SIZE) c.zero(2 * CLUSTER_SIZE, CLUSTER_SIZE) c.flush() with qemu_nbd.open( src_top, "qcow2", read_only=True, backing_chain=False) as c: log.debug("src_top extents: %s", list(nbdutil.extents(c))) # Create empty destination base image. dst_base = str(tmpdir.join("dst_base." + base_fmt)) qemu_img.create(dst_base, base_fmt, size=size) # Create empty destination top image. dst_top = str(tmpdir.join("dst_top.qcow2")) qemu_img.create( dst_top, "qcow2", backing_file=dst_base, backing_format=base_fmt) # Upload base image. nbd_server.image = dst_base nbd_server.fmt = base_fmt nbd_server.start() url = prepare_transfer(srv, nbd_server.sock.url(), size=size) client.upload( src_base, url, srv.config.tls.ca_file, backing_chain=False) nbd_server.stop() with qemu_nbd.open(dst_base, base_fmt, read_only=True) as c: log.debug("dst_base extents: %s", list(nbdutil.extents(c))) # Compare image content - must match. qemu_img.compare( src_base, dst_base, format1=base_fmt, format2=base_fmt, strict=False) # Comparing allocation is not possible with raw format since qemu-nbd does # not report allocation for raw images, so we treat unallocated areas as # zero area instead of a hole. if base_fmt == "qcow2": qemu_img.compare( src_base, dst_base, format1=base_fmt, format2=base_fmt, strict=True) # Upload top image. nbd_server.image = dst_top nbd_server.fmt = "qcow2" nbd_server.start() url = prepare_transfer(srv, nbd_server.sock.url(), size=size) client.upload( src_top, url, srv.config.tls.ca_file, backing_chain=False) nbd_server.stop() with qemu_nbd.open( dst_top, "qcow2", read_only=True, backing_chain=False) as c: log.debug("dst_top extents: %s", list(nbdutil.extents(c))) # Test image content - must match. qemu_img.compare( src_top, dst_top, format1="qcow2", format2="qcow2", strict=False) # Compare allocation for qcow2 chain - nice to have. if base_fmt == "qcow2": qemu_img.compare( src_top, dst_top, format1="qcow2", format2="qcow2", strict=True)
def test_download_shallow(srv, nbd_server, tmpdir, base_fmt): size = 10 * 1024**2 # Create source base image with some data in first clusters. src_base = str(tmpdir.join("src_base." + base_fmt)) qemu_img.create(src_base, base_fmt, size=size) with qemu_nbd.open(src_base, base_fmt) as c: c.write(0 * CLUSTER_SIZE, b"a" * CLUSTER_SIZE) c.write(1 * CLUSTER_SIZE, b"b" * CLUSTER_SIZE) c.write(2 * CLUSTER_SIZE, b"c" * CLUSTER_SIZE) c.flush() log.debug("src_base extents: %s", list(nbdutil.extents(c))) # Create source top image with some data in second cluster and zero in the # third cluster. src_top = str(tmpdir.join("src_top.qcow2")) qemu_img.create( src_top, "qcow2", backing_file=src_base, backing_format=base_fmt) with qemu_nbd.open(src_top, "qcow2") as c: c.write(1 * CLUSTER_SIZE, b"B" * CLUSTER_SIZE) c.zero(2 * CLUSTER_SIZE, CLUSTER_SIZE) c.flush() with qemu_nbd.open( src_top, "qcow2", read_only=True, backing_chain=False) as c: log.debug("src_top extents: %s", list(nbdutil.extents(c))) dst_base = str(tmpdir.join("dst_base." + base_fmt)) dst_top = str(tmpdir.join("dst_top.qcow2")) # Download base image. nbd_server.image = src_base nbd_server.fmt = base_fmt nbd_server.start() url = prepare_transfer(srv, nbd_server.sock.url(), size=size) client.download( url, dst_base, srv.config.tls.ca_file, fmt=base_fmt) nbd_server.stop() with qemu_nbd.open(dst_base, base_fmt, read_only=True) as c: log.debug("dst_base extents: %s", list(nbdutil.extents(c))) # Compare image content - must match. qemu_img.compare( src_base, dst_base, format1=base_fmt, format2=base_fmt, strict=False) # And allocation - nice to have. if base_fmt == "qcow2" or qemu_nbd.version() >= (6, 0, 0): qemu_img.compare( src_base, dst_base, format1=base_fmt, format2=base_fmt, strict=True) # Download top image. nbd_server.image = src_top nbd_server.fmt = "qcow2" nbd_server.backing_chain = False nbd_server.start() url = prepare_transfer(srv, nbd_server.sock.url(), size=size) client.download( url, dst_top, srv.config.tls.ca_file, backing_file=dst_base, backing_format=base_fmt) nbd_server.stop() with qemu_nbd.open( dst_top, "qcow2", read_only=True, backing_chain=False) as c: log.debug("dst_top extents: %s", list(nbdutil.extents(c))) # Compare both image content - must match. qemu_img.compare( src_top, dst_top, format1="qcow2", format2="qcow2", strict=False) # And allocation - nice to have. qemu_img.compare( src_top, dst_top, format1="qcow2", format2="qcow2", strict=True)