def test_full_backup_single_image(tmpdir, fmt, transport):
chunk_size = 1024**2
disk_size = 5 * chunk_size
# Create disk
disk = str(tmpdir.join("disk." + fmt))
subprocess.check_call([
"qemu-img",
"create",
"-f",
fmt,
disk,
str(disk_size),
])
# Pupulate disk with data.
with qemu_nbd.open(disk, fmt) as d:
for i in range(0, disk_size, chunk_size):
d.write(i, b"%d\n" % i)
d.flush()
if transport == "unix":
sock = nbd.UnixAddress(tmpdir.join("sock"))
else:
sock = nbd.TCPAddress("localhost", testutil.random_tcp_port())
# Start full backup and copy the data, veifying what we read.
with backup.full_backup(tmpdir, disk, fmt, sock):
with nbd.Client(sock, "sda") as c:
log.debug("Backing up data with nbd client")
for i in range(0, disk_size, chunk_size):
data = c.read(i, chunk_size)
assert data.startswith(b"%d\n\0" % i)
def test_size(nbd_server, fmt):
nbd_server.fmt = fmt
subprocess.check_call(
["qemu-img", "create", "-f", fmt, nbd_server.image, "150m"])
nbd_server.start()
with nbd.open(nbd_server.url, "r") as b:
assert b.size() == 150 * 1024**2
def loop_device(tmpdir):
backing_file = str(tmpdir.join("backing_file"))
with util.open(backing_file, "w") as f:
buf = util.aligned_buffer(BLOCKSIZE * 3)
with closing(buf):
buf[:] = b"x" * BLOCKSIZE * 3
f.write(buf)
out = subprocess.check_output(
["losetup", "--find", backing_file, "--show"])
try:
loop = out.strip().decode("ascii")
yield loop
finally:
subprocess.check_call(["losetup", "--detach", loop])
def test_qcow2_write_read(tmpdir):
image = str(tmpdir.join("image"))
sock = nbd.UnixAddress(tmpdir.join("sock"))
offset = 1024**2
data = b"can read and write qcow2"
subprocess.check_call(["qemu-img", "create", "-f", "qcow2", image, "1g"])
with qemu_nbd.run(image, "qcow2", sock):
with nbd.Client(sock) as c:
c.write(offset, data)
c.flush()
with nbd.Client(sock) as c:
assert c.read(offset, len(data)) == data
def test_full_backup(tmpdir, fmt, transport):
disk_size = 1024**2
disk_part = disk_size // 4
disk = str(tmpdir.join("disk." + fmt))
backup_disk = str(tmpdir.join("backup.raw"))
# Create disk
subprocess.check_call([
"qemu-img",
"create",
"-f",
fmt,
disk,
str(disk_size),
])
# Pupulate disk with data.
with qemu_nbd.open(disk, fmt) as d:
for i in range(0, disk_size, disk_part):
data = b"%d\n" % i
d.write(i, data.ljust(512))
d.flush()
if transport == "unix":
nbd_sock = nbd.UnixAddress(tmpdir.join("nbd.sock"))
else:
nbd_sock = nbd.TCPAddress("localhost", testutil.random_tcp_port())
# Backup using qemu-img convert.
with backup.full_backup(tmpdir, disk, fmt, nbd_sock):
log.debug("Backing up image with qemu-img")
subprocess.check_call([
"qemu-img",
"convert",
"-p",
"-f",
"raw",
"-O",
"raw",
nbd_sock.url("sda"),
backup_disk,
])
# Compare source and backup disks.
with qemu_nbd.open(disk, fmt, read_only=True) as d, \
io.open(backup_disk, "rb") as b:
for i in range(0, disk_size, disk_part):
b.seek(i)
assert d.read(i, 512) == b.read(512)
def test_add_bitmap(tmpdir):
# Test command with arguments. This is also interesting for incremental
# backup flows.
image = str(tmpdir.join("image.qcow2"))
subprocess.check_call(["qemu-img", "create", "-f", "qcow2", image, "1g"])
qmp_sock = nbd.UnixAddress(tmpdir.join("qmp.sock"))
with qemu.run(image, "qcow2", qmp_sock, start_cpu=False):
with qmp.Client(qmp_sock) as c:
b = qmp.find_node(c, image)
c.execute("block-dirty-bitmap-add", {
"node": b["device"],
"name": "bitmap0",
})
b = qmp.find_node(c, image)
assert b["dirty-bitmaps"][0]["name"] == "bitmap0"
def test_handshake(tmpdir, export, fmt):
image = str(tmpdir.join("image"))
subprocess.check_call(["qemu-img", "create", "-f", fmt, image, "1g"])
sock = nbd.UnixAddress(tmpdir.join("sock"))
with qemu_nbd.run(image, fmt, sock, export_name=export):
if export:
c = nbd.Client(sock, export)
else:
c = nbd.Client(sock)
with c:
# TODO: test transmission_flags?
assert c.export_size == 1024**3
assert c.minimum_block_size == 1
assert c.preferred_block_size == 4096
assert c.maximum_block_size == 32 * 1024**2
assert c.base_allocation
def test_full_backup_handshake(tmpdir, fmt, transport):
image = str(tmpdir.join("image"))
subprocess.check_call(["qemu-img", "create", "-f", fmt, image, "1g"])
if transport == "unix":
sock = nbd.UnixAddress(tmpdir.join("sock"))
else:
sock = nbd.TCPAddress("localhost", testutil.random_tcp_port())
with backup.full_backup(tmpdir, image, fmt, sock):
with nbd.Client(sock, "sda") as c:
# TODO: test transmission_flags?
assert c.export_size == 1024**3
assert c.minimum_block_size == 1
assert c.preferred_block_size == 4096
assert c.maximum_block_size == 32 * 1024**2
assert c.base_allocation
def test_zero_min_block_size(tmpdir, format):
offset = 1024**2
image = str(tmpdir.join("image"))
sock = nbd.UnixAddress(tmpdir.join("sock"))
subprocess.check_call(["qemu-img", "create", "-f", format, image, "1g"])
with qemu_nbd.run(image, format, sock):
# Fill range with data
with nbd.Client(sock) as c:
size = c.minimum_block_size
c.write(offset, b"x" * size)
c.flush()
# Zero range using minimum block size
with nbd.Client(sock) as c:
c.zero(offset, size)
c.flush()
with nbd.Client(sock) as c:
assert c.read(offset, size) == b"\0" * size
def test_open(tmpdir, fmt):
disk = str(tmpdir.join("disk." + fmt))
subprocess.check_call([
"qemu-img",
"create",
"-f",
fmt,
disk,
"1m",
])
offset = 64 * 1024
data = b"it works"
with qemu_nbd.open(disk, fmt) as d:
d.write(offset, data)
d.flush()
with qemu_nbd.open(disk, fmt, read_only=True) as d:
assert d.read(offset, len(data)) == data
def test_zero(tmpdir, format):
size = 2 * 1024**2
offset = 1024**2
image = str(tmpdir.join("image"))
sock = nbd.UnixAddress(tmpdir.join("sock"))
subprocess.check_call(
["qemu-img", "create", "-f", format, image,
str(size)])
with qemu_nbd.run(image, format, sock):
# Fill image with data
with nbd.Client(sock) as c:
c.write(0, b"x" * size)
c.flush()
# Zero a range
with nbd.Client(sock) as c:
c.zero(offset, 4096)
c.flush()
with nbd.Client(sock) as c:
assert c.read(offset, 4096) == b"\0" * 4096
def test_full_backup_complete_chain(tmpdir, transport):
depth = 3
chunk_size = 1024**2
disk_size = depth * chunk_size
for i in range(depth):
# Create disk based on previous one.
disk = str(tmpdir.join("disk.%d" % i))
cmd = ["qemu-img", "create", "-f", "qcow2"]
if i > 0:
cmd.append("-b")
cmd.append("disk.%d" % (i - 1))
cmd.append(disk)
cmd.append(str(disk_size))
subprocess.check_call(cmd)
# This data can be read only from this disk.
with qemu_nbd.open(disk, "qcow2") as d:
d.write(i * chunk_size, b"%d\n" % i)
d.flush()
if transport == "unix":
sock = nbd.UnixAddress(tmpdir.join("sock"))
else:
sock = nbd.TCPAddress("localhost", testutil.random_tcp_port())
# Start full backup and copy the data, veifying what we read.
with backup.full_backup(tmpdir, disk, "qcow2", sock):
with nbd.Client(sock, "sda") as c:
log.debug("Backing up data with nbd client")
for i in range(depth):
# Every chunk comes from different image.
data = c.read(i * chunk_size, chunk_size)
assert data.startswith(b"%d\n\0" % i)
def full_backup(tmpdir, disk, fmt, nbd_sock):
"""
Start qemu internal nbd server using address nbd_sock, exposing disk for
full backup, creating temporary files in tmpdir.
"""
log.debug("Creating scratch disk")
scratch_disk = str(tmpdir.join("scratch.qcow2"))
subprocess.check_call(
["qemu-img", "create", "-f", "qcow2", "-b", disk, scratch_disk])
qmp_sock = nbd.UnixAddress(tmpdir.join("qmp.sock"))
with qemu.run(disk, fmt, qmp_sock, start_cpu=False), \
qmp.Client(qmp_sock) as c:
log.debug("Starting nbd server")
if nbd_sock.transport == "unix":
addr = {"type": "unix", "data": {"path": nbd_sock.path}}
elif nbd_sock.transport == "tcp":
addr = {
"type": "inet",
"data": {
"host": nbd_sock.host,
# Qemu wants port as string.
"port": str(nbd_sock.port),
}
}
else:
raise RuntimeError("Unsupported transport: {}".format(nbd_sock))
c.execute("nbd-server-start", {"addr": addr})
node = qmp.find_node(c, disk)
log.debug("Adding backup node for %s", node)
c.execute(
"blockdev-add", {
"driver": "qcow2",
"node-name": "backup-sda",
"file": {
"driver": "file",
"filename": scratch_disk,
},
"backing": node["device"],
})
log.debug("Starting backup job")
c.execute(
"transaction", {
'actions': [
{
'data': {
'device': node["device"],
'job-id': 'backup-sda',
'sync': 'none',
'target': 'backup-sda'
},
'type': 'blockdev-backup',
},
]
})
log.debug("Adding node to nbd server")
c.execute("nbd-server-add", {"device": "backup-sda", "name": "sda"})
try:
yield
finally:
# Give qemu time to detect that the NBD client disconnected before
# we tear down the nbd server.
log.debug("Waiting before tearing down nbd server")
time.sleep(0.1)
log.debug("Removing disk sda from nbd server")
c.execute("nbd-server-remove", {"name": "sda"})
log.debug("Stopping nbd server")
c.execute("nbd-server-stop")
log.debug("Cancelling block job")
c.execute("block-job-cancel", {"device": "backup-sda"})
log.debug("Removing backup node")
c.execute("blockdev-del", {"node-name": "backup-sda"})