def test_options(tmpdir, fmt, options): size = 4 * 1024**2 chunk_size = 128 * 1024 src = str(tmpdir.join("src." + fmt)) qemu_img.create(src, fmt, size=size) with qemu_nbd.open(src, fmt) as c: for offset in range(0, size, chunk_size): c.write(offset, struct.pack(">Q", offset)) c.flush() dst = str(tmpdir.join("dst." + fmt)) qemu_img.create(dst, fmt, size=size) src_addr = nbd.UnixAddress(str(tmpdir.join("src.sock"))) dst_addr = nbd.UnixAddress(str(tmpdir.join("dst.sock"))) with qemu_nbd.run( src, fmt, src_addr, read_only=True, **options), \ qemu_nbd.run( dst, fmt, dst_addr, **options), \ nbd.Client(src_addr) as src_client, \ nbd.Client(dst_addr) as dst_client: nbdutil.copy(src_client, dst_client) qemu_img.compare(src, dst)
def test_zero_extents_raw(tmpdir): size = 10 * 1024**2 # Create image with some data, zero and holes. image = str(tmpdir.join("image.raw")) qemu_img.create(image, "raw", size=size) with qemu_nbd.open(image, "raw") as c: c.write(0 * CLUSTER_SIZE, b"A" * CLUSTER_SIZE) c.zero(1 * CLUSTER_SIZE, CLUSTER_SIZE) c.write(2 * CLUSTER_SIZE, b"B" * CLUSTER_SIZE) c.flush() extents = list(client.extents(image)) # Note: raw files report unallocated as zero, not a a hole. assert extents == [ ZeroExtent(start=0 * CLUSTER_SIZE, length=CLUSTER_SIZE, zero=False, hole=False), ZeroExtent(start=1 * CLUSTER_SIZE, length=CLUSTER_SIZE, zero=True, hole=False), ZeroExtent(start=2 * CLUSTER_SIZE, length=CLUSTER_SIZE, zero=False, hole=False), ZeroExtent(start=3 * CLUSTER_SIZE, length=size - 3 * CLUSTER_SIZE, zero=True, hole=False), ]
def test_ova(tmpdir, fmt): size = 1024**2 offset = 64 * 1024 disks = [] # Created disks with unique content. for i in range(2): disk = str(tmpdir.join("disk{}.{}".format(i, fmt))) qemu_img.create(disk, fmt, size=size) with qemu_nbd.open(disk, fmt) as d: d.write(offset, disk.encode("utf-8")) d.flush() disks.append(disk) ova = str(tmpdir.join("vm.ova")) # Create a ova file. with tarfile.open(ova, "w") as tar: for disk in disks: tar.add(disk, arcname=os.path.basename(disk)) # Read disks contents from the ova file. with tarfile.open(ova) as tar: for disk in disks: member = tar.getmember(os.path.basename(disk)) with qemu_nbd.open( ova, fmt=fmt, read_only=True, offset=member.offset_data, size=member.size) as d: assert d.export_size == size data = disk.encode("utf-8") assert d.read(offset, len(data)) == data
def test_full_backup_complete_chain(tmpdir, nbd_sock, checkpoint): depth = 3 chunk_size = 1024**2 disk_size = depth * chunk_size for i in range(depth): # Create disk based on previous one. disk = str(tmpdir.join("disk.{}".format(i))) if i == 0: qemu_img.create(disk, "qcow2", size=disk_size) else: qemu_img.create(disk, "qcow2", backing_file="disk.{}".format(i - 1), backing_format="qcow2") # This data can be read only from this disk. with qemu_nbd.open(disk, "qcow2") as d: offset = i * chunk_size d.write(offset, b"%d\n" % offset) d.flush() # Start full backup and copy the data, veifying what we read. with backup.full_backup(tmpdir, disk, "qcow2", nbd_sock, checkpoint=checkpoint): verify_full_backup(nbd_sock, "sda") if checkpoint: bitmaps = list_bitmaps(disk) assert len(bitmaps) == 1 assert bitmaps[0]["name"] == checkpoint
def test_zero_extents_from_ova(tmpdir): size = 10 * 1024**2 # Create image with data, zero and hole clusters. disk = str(tmpdir.join("disk.qcow2")) qemu_img.create(disk, "qcow2", size=size) with qemu_nbd.open(disk, "qcow2") as c: c.write(0 * CLUSTER_SIZE, b"A" * CLUSTER_SIZE) c.zero(1 * CLUSTER_SIZE, CLUSTER_SIZE) c.flush() # Create OVA whith this image. ova = str(tmpdir.join("vm.ova")) with tarfile.open(ova, "w") as tar: tar.add(disk, arcname=os.path.basename(disk)) extents = list(client.extents(ova, member="disk.qcow2")) assert extents == [ ZeroExtent(start=0 * CLUSTER_SIZE, length=CLUSTER_SIZE, zero=False, hole=False), ZeroExtent(start=1 * CLUSTER_SIZE, length=CLUSTER_SIZE, zero=True, hole=False), ZeroExtent(start=2 * CLUSTER_SIZE, length=size - 2 * CLUSTER_SIZE, zero=True, hole=True), ]
def test_download_qcow2_as_raw(tmpdir, srv): src = str(tmpdir.join("src.qcow2")) qemu_img.create(src, "qcow2", size=IMAGE_SIZE) # Allocate one cluster in the middle of the image. with qemu_nbd.open(src, "qcow2") as c: c.write(CLUSTER_SIZE, b"a" * CLUSTER_SIZE) c.flush() actual_size = os.path.getsize(src) url = prepare_transfer(srv, "file://" + src, size=actual_size) dst = str(tmpdir.join("dst.qcow2")) # When downloading qcow2 image using the nbd backend, we get raw data and # we can convert it to any format we want. Howver when downloading using # the file backend, we get qcow2 bytestream and we cannot convert it. # # To store the qcow2 bytestream, we must use fmt="raw". This instructs # qemu-nbd on the client side to treat the data as raw bytes, storing them # without any change on the local file. # # This is baisically like: # # qemu-img convert -f raw -O raw src.qcow2 dst.qcow2 # client.download(url, dst, srv.config.tls.ca_file, fmt="raw") # The result should be identical qcow2 image content. Allocation may # differ but for this test we get identical allocation. qemu_img.compare(src, dst, format1="qcow2", format2="qcow2", strict=True)
def test_dirty_bitmap(tmpdir): size = 1024**2 # Create image with empty bitmap. img = str(tmpdir.join("img.qcow2")) qemu_img.create(img, "qcow2", size=size) qemu_img.bitmap_add(img, "b0") # Write data to image, modifying the bitmap. with qemu_nbd.open(img, "qcow2") as c: # This will allocate one cluster. By default bitmap granularity is also # one cluster, so this will make the first extent dirty. c.write(0, b"a") c.flush() # Read dirty extents. with qemu_nbd.open(img, "qcow2", read_only=True, bitmap="b0") as c: extents = c.extents(0, size)[nbd.QEMU_DIRTY_BITMAP + "b0"] bitmap = qemu_img.info(img)["format-specific"]["data"]["bitmaps"][0] assert extents == [ nbd.Extent(length=bitmap["granularity"], flags=nbd.EXTENT_DIRTY), nbd.Extent(length=size - bitmap["granularity"], flags=0), ]
def test_backing_chain(tmpdir): size = 128 * 1024 base = str(tmpdir.join("base.raw")) top = str(tmpdir.join("top.qcow2")) base_data = b"data from base".ljust(32, b"\0") # Add base image with some data. qemu_img.create(base, "raw", size=size) with qemu_nbd.open(base, "raw") as c: c.write(0, base_data) c.flush() # Add empty overlay. qemu_img.create(top, "qcow2", backing_file=base, backing_format="raw") top_addr = nbd.UnixAddress(str(tmpdir.join("sock"))) # By default, we see data from base. with qemu_nbd.run(top, "qcow2", top_addr), \ nbd.Client(top_addr) as c: assert c.read(0, 32) == base_data # With backing chain disabled, we see data only from top. with qemu_nbd.run(top, "qcow2", top_addr, backing_chain=False), \ nbd.Client(top_addr) as c: assert c.read(0, 32) == b"\0" * 32
def test_extents_zero(nbd_server, user_file, fmt): size = 6 * 1024**3 qemu_img.create(user_file.path, fmt, size=size) nbd_server.image = user_file.path nbd_server.fmt = fmt nbd_server.start() with nbd.open(nbd_server.url, "r+") as b: # qcow2 extents resolution is cluster size. data = b"x" * 64 * 1024 b.write(data) # The second extent length is bigger than NBD maximum length, testing # that our extent length is not limited by NBD limits. The backend # sends multiple block status commands and merge the returned extents. b.seek(5 * 1024**3) b.write(data) # Holes can be reported only for qcow2 images. hole = fmt == "qcow2" assert list(b.extents()) == [ extent.ZeroExtent(0, len(data), False, False), extent.ZeroExtent(len(data), 5 * 1024**3 - len(data), True, hole), extent.ZeroExtent(5 * 1024**3, len(data), False, False), extent.ZeroExtent(5 * 1024**3 + len(data), 1024**3 - len(data), True, hole), ]
def test_size(nbd_server, fmt): size = 150 * 1024**2 nbd_server.fmt = fmt qemu_img.create(nbd_server.image, fmt, size=size) nbd_server.start() with nbd.open(nbd_server.url) as b: assert b.size() == size
def test_zero_extents_qcow2(tmpdir): size = 10 * 1024**2 # Create base image with one data and one zero cluster. base = str(tmpdir.join("base.qcow2")) qemu_img.create(base, "qcow2", size=size) with qemu_nbd.open(base, "qcow2") as c: c.write(0 * CLUSTER_SIZE, b"A" * CLUSTER_SIZE) c.zero(1 * CLUSTER_SIZE, CLUSTER_SIZE) c.flush() # Create top image with one data and one zero cluster. top = str(tmpdir.join("top.qcow2")) qemu_img.create( top, "qcow2", backing_file=base, backing_format="qcow2") with qemu_nbd.open(top, "qcow2") as c: c.write(3 * CLUSTER_SIZE, b"B" * CLUSTER_SIZE) c.zero(4 * CLUSTER_SIZE, CLUSTER_SIZE) c.flush() extents = list(client.extents(top)) assert extents == [ # Extents from base... ZeroExtent( start=0 * CLUSTER_SIZE, length=CLUSTER_SIZE, zero=False, hole=False), ZeroExtent( start=1 * CLUSTER_SIZE, length=CLUSTER_SIZE, zero=True, hole=False), ZeroExtent( start=2 * CLUSTER_SIZE, length=CLUSTER_SIZE, zero=True, hole=True), # Extents from top... ZeroExtent( start=3 * CLUSTER_SIZE, length=CLUSTER_SIZE, zero=False, hole=False), ZeroExtent( start=4 * CLUSTER_SIZE, length=CLUSTER_SIZE, zero=True, hole=False), # Rest of unallocated data... ZeroExtent( start=5 * CLUSTER_SIZE, length=size - 5 * CLUSTER_SIZE, zero=True, hole=True), ]
def test_compare_missing_file(tmpdir): src = str(tmpdir.join("src.raw")) dst = str(tmpdir.join("dst.raw")) qemu_img.create(src, "raw", size=1024**2) with pytest.raises(qemu_img.OpenImageError): qemu_img.compare(src, dst)
def test_checksum_algorithm(tmpdir, algorithm, digest_size): img = str(tmpdir.join("img")) qemu_img.create(img, "raw", size="2m") expected = blkhash.checksum( img, block_size=1024**2, algorithm=algorithm, digest_size=digest_size) actual = client.checksum(img, block_size=1024**2, algorithm=algorithm) assert actual == expected
def test_extents_dirty_not_availabe(nbd_server, fmt): qemu_img.create(nbd_server.image, fmt, 65536) nbd_server.fmt = fmt nbd_server.start() with nbd.open(nbd_server.url, "r+", dirty=True) as b: with pytest.raises(errors.UnsupportedOperation): list(b.extents(context="dirty"))
def create_image(path, fmt, size): if fmt == "raw": # qemu-img allocates the first block on Fedora, but not on CentOS 8.0. # Allocate manually for consistent results. # TODO: Use qemu-img when we have CentOS 8.1 AV. with io.open(path, "wb") as f: f.truncate(size) else: qemu_img.create(path, "qcow2", size=size)
def test_add_bitmap(tmpdir): size = 10 * 1024**2 img = str(tmpdir.join("img.qcow2")) qemu_img.create(img, "qcow2", size=size) qemu_img.bitmap_add(img, "b0") bitmaps = qemu_img.info(img)["format-specific"]["data"]["bitmaps"] assert bitmaps == [ {"name": "b0", "flags": ["auto"], "granularity": 65536} ]
def test_create_info(tmpdir, fmt): size = 1024**2 image = str(tmpdir.join("image." + fmt)) qemu_img.create(image, fmt, size=size) info = qemu_img.info(image) assert info["filename"] == image assert info["virtual-size"] == size assert info["format"] == fmt
def test_zero_sparse(nbd_server, user_file, sparse): size = 10 * 1024**2 qemu_img.create(user_file.path, "raw", size=size) nbd_server.image = user_file.path nbd_server.start() with nbd.open(nbd_server.url, "r+", sparse=sparse) as b: b.zero(b.size()) b.flush() actual_size = os.stat(user_file.path).st_blocks * 512 assert actual_size == 0 if sparse else b.size()
def test_copy_nbd_to_nbd(tmpdir, src_fmt, dst_fmt, zero, hole): # Default cluser size with qcow2 format. cluster_size = 64 * 1024 extents = [ ("data", cluster_size), ("zero", cluster_size), ("data", cluster_size), ("hole", cluster_size + io.MAX_ZERO_SIZE), ("data", cluster_size + io.BUFFER_SIZE), ("hole", cluster_size), ("data", cluster_size), ] size = sum(length for _, length in extents) src = str(tmpdir.join("src." + src_fmt)) qemu_img.create(src, src_fmt, size=size) populate_image(src, src_fmt, extents) src_sock = UnixAddress(tmpdir.join("src.sock")) src_url = urlparse(src_sock.url()) dst = str(tmpdir.join("dst." + dst_fmt)) qemu_img.create(dst, dst_fmt, size=size) dst_sock = UnixAddress(tmpdir.join("dst.sock")) dst_url = urlparse(dst_sock.url()) # Note: We need extra worker for reading extents for source. max_workers = 2 with qemu_nbd.run( src, src_fmt, src_sock, read_only=True, shared=max_workers + 1), \ qemu_nbd.run( dst, dst_fmt, dst_sock, shared=max_workers), \ nbd.open(src_url, "r") as src_backend, \ nbd.open(dst_url, "r+", sparse=True) as dst_backend: # Because we copy to new image, we can always use zero=False, but we # test both to verify that the result is the same. io.copy(src_backend, dst_backend, max_workers=max_workers, zero=zero, hole=hole) # Compare image content - must match. qemu_img.compare(src, dst) # Allocation can be compared only with qcow2 images when we write zeroes to # zero extents and skip holes. if src_fmt == "qcow2" and dst_fmt == "qcow2" and zero and not hole: qemu_img.compare(src, dst, strict=True)
def test_open(tmpdir, fmt): disk = str(tmpdir.join("disk." + fmt)) qemu_img.create(disk, fmt, size=1024**2) offset = 64 * 1024 data = b"it works" with qemu_nbd.open(disk, fmt) as d: d.write(offset, data) d.flush() with qemu_nbd.open(disk, fmt, read_only=True) as d: assert d.read(offset, len(data)) == data
def verify_backup(backup_disk, expected_files): log.info("Verifying backup") preview_disk = backup_disk + ".preview" qemu_img.create(preview_disk, "qcow2", backing_file=backup_disk, backing_format="qcow2") with qemu.run(preview_disk, "qcow2") as guest: guest.login("root", "") out = guest.run("ls -1 --color=never") assert out.splitlines() == expected_files
def test_compare_wrong_format(tmpdir): size = 1024**2 src = str(tmpdir.join("src.raw")) dst = str(tmpdir.join("dst.raw")) qemu_img.create(src, "raw", size=size) qemu_img.create(dst, "raw", size=size) with pytest.raises(qemu_img.OpenImageError): qemu_img.compare(src, dst, format1="qcow2") with pytest.raises(qemu_img.OpenImageError): qemu_img.compare(src, dst, format2="qcow2")
def test_compare_identical_content(tmpdir, src_fmt, dst_fmt): size = 1024**2 src = str(tmpdir.join("src." + src_fmt)) dst = str(tmpdir.join("dst." + dst_fmt)) qemu_img.create(src, src_fmt, size=size) qemu_img.create(dst, dst_fmt, size=size) # Destination image has different allocation. with qemu_nbd.open(dst, dst_fmt) as c: c.write(size // 2, b"\0") c.flush() qemu_img.compare(src, dst, format1=src_fmt, format2=dst_fmt)
def test_dirty_extents(tmpdir): size = 1024**2 # Create base image with empty dirty bitmap. base = str(tmpdir.join("base.qcow2")) qemu_img.create(base, "qcow2", size=size) qemu_img.bitmap_add(base, "b0") # Write data, modifying the dirty bitmap. with qemu_nbd.open(base, "qcow2") as c: c.write(0 * CLUSTER_SIZE, b"A" * CLUSTER_SIZE) c.zero(1 * CLUSTER_SIZE, CLUSTER_SIZE) c.flush() # Create top image with empty dirty bitmap. top = str(tmpdir.join("top.qcow2")) qemu_img.create(top, "qcow2", backing_file=base, backing_format="qcow2") qemu_img.bitmap_add(top, "b0") # Write data, modifying the dirty bitmap. with qemu_nbd.open(top, "qcow2") as c: c.write(3 * CLUSTER_SIZE, b"B" * CLUSTER_SIZE) c.zero(4 * CLUSTER_SIZE, CLUSTER_SIZE) c.flush() dirty_extents = list(client.extents(base, bitmap="b0")) assert dirty_extents == [ DirtyExtent(start=0 * CLUSTER_SIZE, length=2 * CLUSTER_SIZE, dirty=True), DirtyExtent(start=2 * CLUSTER_SIZE, length=size - 2 * CLUSTER_SIZE, dirty=False), ] dirty_extents = list(client.extents(top, bitmap="b0")) # Note: qemu-nbd reports dirty extents only for the top image. assert dirty_extents == [ DirtyExtent(start=0 * CLUSTER_SIZE, length=3 * CLUSTER_SIZE, dirty=False), DirtyExtent(start=3 * CLUSTER_SIZE, length=2 * CLUSTER_SIZE, dirty=True), DirtyExtent(start=5 * CLUSTER_SIZE, length=size - 5 * CLUSTER_SIZE, dirty=False), ]
def test_upload_preallocated(tmpdir, srv, fmt): src = str(tmpdir.join("src")) qemu_img.create(src, fmt, size=IMAGE_SIZE) dst = str(tmpdir.join("dst")) with open(dst, "wb") as f: f.write(b"a" * IMAGE_SIZE) url = prepare_transfer(srv, "file://" + dst, sparse=False) client.upload(src, url, srv.config.tls.ca_file) qemu_img.compare(src, dst) assert os.stat(dst).st_blocks * 512 == IMAGE_SIZE
def test_incremental_backup_guest(tmpdir, base_image): base = qemu_img.info(base_image) disk_size = base["virtual-size"] disk = str(tmpdir.join("disk.qcow2")) qemu_img.create(disk, "qcow2", backing_file=base_image, backing_format=base["format"]) scratch_disk = str(tmpdir.join("scratch.qcow2")) qemu_img.create(scratch_disk, "qcow2", size=disk_size) full_backup_disk = str(tmpdir.join("full-backup.qcow2")) qemu_img.create(full_backup_disk, "qcow2", size=disk_size) incr_backup_disk = str(tmpdir.join("incr-backup.qcow2")) qemu_img.create(incr_backup_disk, "qcow2", size=disk_size) qmp_sock = nbd.UnixAddress(tmpdir.join("qmp.sock")) nbd_sock = nbd.UnixAddress(tmpdir.join("nbd.sock")) with qemu.run(disk, "qcow2", qmp_sock, shutdown_timeout=10) as guest, \ qmp.Client(qmp_sock) as qmp_client: guest.login("root", "") with backup.run(qmp_client, nbd_sock, scratch_disk, checkpoint="check1"): backup.copy_disk(nbd_sock.url("sda"), full_backup_disk) qemu_img.create(scratch_disk, "qcow2", size=disk_size) assert guest.run("touch before-backup; sync") == "" with backup.run(qmp_client, nbd_sock, scratch_disk, checkpoint="check2", incremental="check1"): assert guest.run("touch during-backup; sync") == "" backup.copy_dirty(nbd_sock.url("sda"), incr_backup_disk) qemu_img.unsafe_rebase(incr_backup_disk, full_backup_disk) verify_backup(incr_backup_disk, ["before-backup"])
def full_backup(tmpdir, disk, fmt, sock, checkpoint=None): """ Start qemu internal nbd server using address sock, exposing disk for full backup, creating temporary files in tmpdir. """ scratch_disk = str(tmpdir.join("scratch.qcow2")) qmp_sock = nbd.UnixAddress(tmpdir.join("qmp.sock")) disk_size = qemu_img.info(disk)["virtual-size"] qemu_img.create(scratch_disk, "qcow2", size=disk_size) with qemu.run(disk, fmt, qmp_sock, start_cpu=False, shutdown_timeout=10), \ qmp.Client(qmp_sock) as c, \ run(c, sock, scratch_disk, checkpoint=checkpoint): yield
def test_compare_different_content(tmpdir, src_fmt, dst_fmt): size = 1024**2 src = str(tmpdir.join("src." + src_fmt)) dst = str(tmpdir.join("dst." + dst_fmt)) qemu_img.create(src, src_fmt, size=size) qemu_img.create(dst, dst_fmt, size=size) # Destination image has different content. with qemu_nbd.open(dst, dst_fmt) as c: c.write(size // 2, b"x") c.flush() with pytest.raises(qemu_img.ContentMismatch): qemu_img.compare(src, dst, format1=src_fmt, format2=dst_fmt)
def test_compare_different_allocation(tmpdir, src_fmt, dst_fmt): # Images has same content, but different allocation. size = 1024**2 src = str(tmpdir.join("src." + src_fmt)) dst = str(tmpdir.join("dst." + dst_fmt)) qemu_img.create(src, src_fmt, size=size) qemu_img.create(dst, dst_fmt, size=size) with qemu_nbd.open(dst, dst_fmt) as c: c.write(size // 2, b"\0") c.flush() with pytest.raises(qemu_img.ContentMismatch): qemu_img.compare( src, dst, format1=src_fmt, format2=dst_fmt, strict=True)
def test_upload_empty_sparse(tmpdir, srv, fmt): src = str(tmpdir.join("src")) qemu_img.create(src, fmt, size=IMAGE_SIZE) dst = str(tmpdir.join("dst")) with open(dst, "wb") as f: f.write(b"a" * IMAGE_SIZE) url = prepare_transfer(srv, "file://" + dst) client.upload(src, url, srv.config.tls.ca_file) # TODO: Check why allocation differ when src is qcow2. Target image # allocation is 0 bytes as expected, but comparing with strict=True fail at # offset 0. qemu_img.compare(src, dst, format1=fmt, format2="raw", strict=fmt == "raw")