def test_upload_from_ova(tmpdir, srv, fmt, compressed):
offset = CLUSTER_SIZE
data = b"I can eat glass and it doesn't hurt me."
# Create raw disk with some data.
tmp = str(tmpdir.join("tmp"))
with open(tmp, "wb") as f:
f.truncate(IMAGE_SIZE)
f.seek(offset)
f.write(data)
# Create source disk.
src = str(tmpdir.join("src"))
qemu_img.convert(tmp, src, "raw", fmt, compressed=compressed)
# Create OVA package.
ova = str(tmpdir.join("src.ova"))
with tarfile.open(ova, "w") as tar:
tar.add(src, arcname=os.path.basename(src))
# Prepare destination file.
dst = str(tmpdir.join("dst"))
with open(dst, "wb") as f:
f.truncate(IMAGE_SIZE)
# Test uploading src from ova.
url = prepare_transfer(srv, "file://" + dst)
client.upload(ova,
url,
srv.config.tls.ca_file,
member=os.path.basename(src))
qemu_img.compare(src, dst)
def test_progress(tmpdir, srv):
src = str(tmpdir.join("src"))
with open(src, "wb") as f:
f.write(b"b" * 4096)
f.seek(IMAGE_SIZE // 2)
f.write(b"b" * 4096)
f.truncate(IMAGE_SIZE)
dst = str(tmpdir.join("dst"))
with open(dst, "wb") as f:
f.truncate(IMAGE_SIZE)
url = prepare_transfer(srv, "file://" + dst, sparse=True)
progress = FakeProgress()
client.upload(src, url, srv.config.tls.ca_file, progress=progress)
assert progress.size == IMAGE_SIZE
# Note: when using multiple connections order of updates is not
# predictable.
assert set(progress.updates) == {
# First write.
4096,
# First zero.
IMAGE_SIZE // 2 - 4096,
# Second write.
4096,
# Second zero
IMAGE_SIZE // 2 - 4096,
}
def test_upload_shallow(srv, nbd_server, tmpdir, base_fmt):
size = 10 * 1024**2
# Create base image with some data in first 3 clusters.
src_base = str(tmpdir.join("src_base." + base_fmt))
qemu_img.create(src_base, base_fmt, size=size)
with qemu_nbd.open(src_base, base_fmt) as c:
c.write(0 * CLUSTER_SIZE, b"a" * CLUSTER_SIZE)
c.write(1 * CLUSTER_SIZE, b"b" * CLUSTER_SIZE)
c.write(2 * CLUSTER_SIZE, b"c" * CLUSTER_SIZE)
c.flush()
# Create src image with some data in second cluster and zero in third
# cluster.
src_top = str(tmpdir.join("src_top.qcow2"))
qemu_img.create(src_top,
"qcow2",
backing_file=src_base,
backing_format=base_fmt)
with qemu_nbd.open(src_top, "qcow2") as c:
c.write(1 * CLUSTER_SIZE, b"B" * CLUSTER_SIZE)
c.zero(2 * CLUSTER_SIZE, CLUSTER_SIZE)
c.flush()
# Create empty destination base image.
dst_base = str(tmpdir.join("dst_base." + base_fmt))
qemu_img.create(dst_base, base_fmt, size=size)
# Create empty destination top image.
dst_top = str(tmpdir.join("dst_top.qcow2"))
qemu_img.create(dst_top,
"qcow2",
backing_file=dst_base,
backing_format=base_fmt)
# Start nbd server for for destination image.
nbd_server.image = dst_top
nbd_server.fmt = "qcow2"
nbd_server.start()
# Upload using nbd backend.
url = prepare_transfer(srv, nbd_server.sock.url(), size=size)
client.upload(src_top, url, srv.config.tls.ca_file, backing_chain=False)
# Stop the server to allow comparing.
nbd_server.stop()
# To compare top, we need to remove the backing files.
qemu_img.unsafe_rebase(src_top, "")
qemu_img.unsafe_rebase(dst_top, "")
qemu_img.compare(src_top,
dst_top,
format1="qcow2",
format2="qcow2",
strict=True)
def test_upload_unix_socket(tmpdir, srv, use_unix_socket):
src = str(tmpdir.join("src"))
with open(src, "wb") as f:
f.write(b"b" * IMAGE_SIZE)
dst = str(tmpdir.join("dst"))
url = prepare_upload(srv, dst)
client.upload(src, url, srv.config.tls.ca_file)
check_content(src, dst)
def test_progress_callback(tmpdir, srv):
src = str(tmpdir.join("src"))
with open(src, "wb") as f:
f.truncate(IMAGE_SIZE)
dst = str(tmpdir.join("dst"))
url = prepare_upload(srv, dst, size=IMAGE_SIZE, sparse=True)
progress = []
client.upload(src, url, srv.config.tls.ca_file, progress=progress.append)
assert progress == [IMAGE_SIZE]
def test_upload_empty_sparse(tmpdir, srv):
src = str(tmpdir.join("src"))
with open(src, "wb") as f:
f.truncate(IMAGE_SIZE)
dst = str(tmpdir.join("dst"))
url = prepare_upload(srv, dst)
client.upload(src, url, srv.config.tls.ca_file)
check_content(src, dst)
assert os.stat(dst).st_blocks == os.stat(src).st_blocks
def test_upload_preallocated(tmpdir, srv, fmt):
src = str(tmpdir.join("src"))
qemu_img.create(src, fmt, size=IMAGE_SIZE)
dst = str(tmpdir.join("dst"))
with open(dst, "wb") as f:
f.write(b"a" * IMAGE_SIZE)
url = prepare_transfer(srv, "file://" + dst, sparse=False)
client.upload(src, url, srv.config.tls.ca_file)
qemu_img.compare(src, dst)
assert os.stat(dst).st_blocks * 512 == IMAGE_SIZE
def test_upload_hole_at_middle_sparse(tmpdir, srv):
src = str(tmpdir.join("src"))
with open(src, "wb") as f:
f.truncate(IMAGE_SIZE)
f.write(b"b" * (IMAGE_SIZE // 4))
f.seek(IMAGE_SIZE // 2, os.SEEK_CUR)
f.write(b"b" * (IMAGE_SIZE // 4))
dst = str(tmpdir.join("dst"))
url = prepare_upload(srv, dst)
client.upload(src, url, srv.config.tls.ca_file)
check_content(src, dst)
assert os.stat(dst).st_blocks == os.stat(src).st_blocks
def test_upload_empty_sparse(tmpdir, srv, fmt):
src = str(tmpdir.join("src"))
qemu_img.create(src, fmt, size=IMAGE_SIZE)
dst = str(tmpdir.join("dst"))
with open(dst, "wb") as f:
f.write(b"a" * IMAGE_SIZE)
url = prepare_transfer(srv, "file://" + dst)
client.upload(src, url, srv.config.tls.ca_file)
# TODO: Check why allocation differ when src is qcow2. Target image
# allocation is 0 bytes as expected, but comparing with strict=True fail at
# offset 0.
qemu_img.compare(src, dst, format1=fmt, format2="raw", strict=fmt == "raw")
def test_upload_full_sparse(tmpdir, srv, fmt):
src = str(tmpdir.join("src"))
qemu_img.create(src, fmt, size=IMAGE_SIZE)
with qemu_nbd.open(src, fmt) as c:
c.write(0, b"b" * IMAGE_SIZE)
c.flush()
dst = str(tmpdir.join("dst"))
with open(dst, "wb") as f:
f.write(b"a" * IMAGE_SIZE)
url = prepare_transfer(srv, "file://" + dst)
client.upload(src, url, srv.config.tls.ca_file)
qemu_img.compare(src, dst, strict=True)
def test_upload_proxy_url_unused(tmpdir, srv):
src = str(tmpdir.join("src"))
with open(src, "wb") as f:
f.truncate(IMAGE_SIZE)
dst = str(tmpdir.join("dst"))
with open(dst, "wb") as f:
f.truncate(IMAGE_SIZE)
# If transfer_url is accessible, proxy_url is not used.
transfer_url = prepare_transfer(srv, "file://" + dst)
proxy_url = "https://no.proxy:54322/images/no-ticket"
client.upload(src, transfer_url, srv.config.tls.ca_file,
proxy_url=proxy_url)
qemu_img.compare(src, dst, format1="raw", format2="raw", strict=True)
def test_upload_hole_at_start_sparse(tmpdir, srv, fmt):
src = str(tmpdir.join("src"))
qemu_img.create(src, fmt, size=IMAGE_SIZE)
with qemu_nbd.open(src, fmt) as c:
c.write(IMAGE_SIZE - 6, b"middle")
c.flush()
dst = str(tmpdir.join("dst"))
with open(dst, "wb") as f:
f.write(b"a" * IMAGE_SIZE)
url = prepare_transfer(srv, "file://" + dst)
client.upload(src, url, srv.config.tls.ca_file)
qemu_img.compare(src, dst, format1=fmt, format2="raw", strict=fmt == "raw")
def test_upload_hole_at_end_sparse(tmpdir, srv, fmt):
size = 3 * 1024**2
src = str(tmpdir.join("src"))
qemu_img.create(src, fmt, size=size)
with qemu_nbd.open(src, fmt) as c:
c.write(0, b"b" * 1024**2)
c.flush()
dst = str(tmpdir.join("dst"))
with open(dst, "wb") as f:
f.write(b"a" * size)
url = prepare_transfer(srv, "file://" + dst, size=size)
client.upload(src, url, srv.config.tls.ca_file)
qemu_img.compare(src, dst, format1=fmt, format2="raw", strict=fmt == "raw")
def test_upload_hole_at_start_sparse(tmpdir, srv, fmt):
size = 3 * 1024**2
src = str(tmpdir.join("src"))
qemu_img.create(src, fmt, size=size)
with qemu_nbd.open(src, fmt) as c:
c.write(size - 1024**2, b"b" * 1024**2)
c.flush()
log.debug("src extents: %s", list(nbdutil.extents(c)))
dst = str(tmpdir.join("dst"))
with open(dst, "wb") as f:
f.write(b"a" * size)
url = prepare_transfer(srv, "file://" + dst, size=size)
client.upload(src, url, srv.config.tls.ca_file)
with qemu_nbd.open(dst, "raw", read_only=True) as c:
log.debug("dst extents: %s", list(nbdutil.extents(c)))
qemu_img.compare(src, dst, format1=fmt, format2="raw", strict=fmt == "raw")
def test_stress(srv, nbd_server, tmpdir, fmt):
size = 10 * 1024**2
# Create empty source and destination images.
src = str(tmpdir.join("src." + fmt))
qemu_img.create(src, fmt, size=size)
dst = str(tmpdir.join("dst." + fmt))
qemu_img.create(dst, fmt, size=size)
# Upload and download the image multiple times. This used to fail randomly
# when the executor closed the destination backend before it was cloned by
# the workers.
nbd_server.image = dst
nbd_server.fmt = fmt
nbd_server.start()
url = prepare_transfer(srv, nbd_server.sock.url(), size=size)
for i in range(20):
client.upload(src, url, srv.config.tls.ca_file)
client.download(url, src, srv.config.tls.ca_file, fmt=fmt)
nbd_server.stop()
def test_progress(tmpdir, srv):
src = str(tmpdir.join("src"))
with open(src, "wb") as f:
f.write(b"b" * 4096)
f.seek(IMAGE_SIZE // 2)
f.write(b"b" * 4096)
f.truncate(IMAGE_SIZE)
dst = str(tmpdir.join("dst"))
url = prepare_upload(srv, dst, sparse=True)
progress = FakeProgress(IMAGE_SIZE)
client.upload(src, url, srv.config.tls.ca_file, progress=progress)
assert progress.updates == [
# First write.
4096,
# First zero.
IMAGE_SIZE // 2 - 4096,
# Second write.
4096,
# Second zero
IMAGE_SIZE // 2 - 4096,
]
def upload_disk_image(connection, module):
transfers_service = connection.system_service().image_transfers_service()
hosts_service = connection.system_service().hosts_service()
transfer = start_transfer(connection, module, otypes.ImageTransferDirection.UPLOAD)
try:
extra_args = {}
parameters = inspect.signature(client.upload).parameters
if "proxy_url" in parameters:
extra_args["proxy_url"] = transfer.proxy_url
if module.params.get('max_workers') and "max_workers" in parameters:
extra_args["max_workers"] = module.params.get('max_workers')
client.upload(
module.params.get('upload_image_path'),
transfer.transfer_url,
module.params.get('auth').get('ca_file'),
secure=not module.params.get('auth').get('insecure'),
buffer_size=client.BUFFER_SIZE,
**extra_args
)
except Exception as e:
cancel_transfer(connection, transfer.id)
raise e
finalize_transfer(connection, module, transfer.id)
return True
# Use multiple workers to speed up the upload.
if "max_workers" in parameters:
extra_args["max_workers"] = args.max_workers
if args.use_proxy:
upload_url = transfer.proxy_url
else:
upload_url = transfer.transfer_url
# Use fallback to proxy_url if feature is available. Upload will use the
# proxy_url if transfer_url is not accessible.
if "proxy_url" in parameters:
extra_args["proxy_url"] = transfer.proxy_url
progress("Uploading image...")
with client.ProgressBar() as pb:
client.upload(args.filename,
upload_url,
args.cafile,
secure=args.secure,
buffer_size=args.buffer_size,
progress=pb,
**extra_args)
progress("Finalizing image transfer...")
imagetransfer.finalize_transfer(connection, transfer, disk)
connection.close()
progress("Upload completed successfully")
host_service = hosts_service.host_service(transfer.host.id)
transfer_host = host_service.get()
print("Transfer host: %s" % transfer_host.name)
# At this stage, the SDK granted the permission to start transferring the disk, and the
# user should choose its preferred tool for doing it. We use the recommended
# way, ovirt-imageio client library.
if args.use_proxy:
destination_url = transfer.proxy_url
else:
destination_url = transfer.transfer_url
print("Uploading image...")
with client.ProgressBar() as pb:
client.upload(
args.filename,
destination_url,
args.cafile,
secure=args.secure,
progress=pb)
print("Finalizing image transfer...")
# Successful cleanup
transfer_service.finalize()
connection.close()
print("Upload completed successfully")
def test_upload_shallow(srv, nbd_server, tmpdir, base_fmt):
size = 10 * 1024**2
# Create base image with some data in first 3 clusters.
src_base = str(tmpdir.join("src_base." + base_fmt))
qemu_img.create(src_base, base_fmt, size=size)
with qemu_nbd.open(src_base, base_fmt) as c:
c.write(0 * CLUSTER_SIZE, b"a" * CLUSTER_SIZE)
c.write(1 * CLUSTER_SIZE, b"b" * CLUSTER_SIZE)
c.write(2 * CLUSTER_SIZE, b"c" * CLUSTER_SIZE)
c.flush()
log.debug("src_base extents: %s", list(nbdutil.extents(c)))
# Create src image with some data in second cluster and zero in third
# cluster.
src_top = str(tmpdir.join("src_top.qcow2"))
qemu_img.create(
src_top, "qcow2", backing_file=src_base, backing_format=base_fmt)
with qemu_nbd.open(src_top, "qcow2") as c:
c.write(1 * CLUSTER_SIZE, b"B" * CLUSTER_SIZE)
c.zero(2 * CLUSTER_SIZE, CLUSTER_SIZE)
c.flush()
with qemu_nbd.open(
src_top, "qcow2", read_only=True, backing_chain=False) as c:
log.debug("src_top extents: %s", list(nbdutil.extents(c)))
# Create empty destination base image.
dst_base = str(tmpdir.join("dst_base." + base_fmt))
qemu_img.create(dst_base, base_fmt, size=size)
# Create empty destination top image.
dst_top = str(tmpdir.join("dst_top.qcow2"))
qemu_img.create(
dst_top, "qcow2", backing_file=dst_base, backing_format=base_fmt)
# Upload base image.
nbd_server.image = dst_base
nbd_server.fmt = base_fmt
nbd_server.start()
url = prepare_transfer(srv, nbd_server.sock.url(), size=size)
client.upload(
src_base,
url,
srv.config.tls.ca_file,
backing_chain=False)
nbd_server.stop()
with qemu_nbd.open(dst_base, base_fmt, read_only=True) as c:
log.debug("dst_base extents: %s", list(nbdutil.extents(c)))
# Compare image content - must match.
qemu_img.compare(
src_base, dst_base, format1=base_fmt, format2=base_fmt, strict=False)
# Comparing allocation is not possible with raw format since qemu-nbd does
# not report allocation for raw images, so we treat unallocated areas as
# zero area instead of a hole.
if base_fmt == "qcow2":
qemu_img.compare(
src_base, dst_base, format1=base_fmt, format2=base_fmt,
strict=True)
# Upload top image.
nbd_server.image = dst_top
nbd_server.fmt = "qcow2"
nbd_server.start()
url = prepare_transfer(srv, nbd_server.sock.url(), size=size)
client.upload(
src_top,
url,
srv.config.tls.ca_file,
backing_chain=False)
nbd_server.stop()
with qemu_nbd.open(
dst_top, "qcow2", read_only=True, backing_chain=False) as c:
log.debug("dst_top extents: %s", list(nbdutil.extents(c)))
# Test image content - must match.
qemu_img.compare(
src_top, dst_top, format1="qcow2", format2="qcow2", strict=False)
# Compare allocation for qcow2 chain - nice to have.
if base_fmt == "qcow2":
qemu_img.compare(
src_top, dst_top, format1="qcow2", format2="qcow2", strict=True)
host = imagetransfer.find_host(connection, args.sd_name)
progress("Creating image transfer...")
transfer = imagetransfer.create_transfer(
connection, disk, types.ImageTransferDirection.UPLOAD, host=host)
try:
progress("Transfer ID: {}".format(transfer.id))
progress("Transfer host name: {}".format(transfer.host.name))
progress("Uploading disk {} from {}...".format(
args.ova_disk_name, os.path.basename(args.ova_file)))
with client.ProgressBar() as pb:
client.upload(
args.ova_file,
transfer.transfer_url,
args.cafile,
secure=args.secure,
progress=pb,
proxy_url=transfer.proxy_url,
member=args.ova_disk_name)
except Exception:
progress("Upload failed, cancelling image transfer...")
imagetransfer.cancel_transfer(connection, transfer)
raise
progress("Finalizing image transfer...")
imagetransfer.finalize_transfer(connection, transfer, disk)
progress("Upload completed successfully")
