def test_vsock_epipe(test_microvm_with_api, network_config, bin_vsock_path,
test_fc_session_root_path):
"""
Vsock negative test to validate SIGPIPE/EPIPE handling.
@type: negative
"""
vm = test_microvm_with_api
vm.spawn()
vm.basic_config()
_tap, _, _ = vm.ssh_network_config(network_config, '1')
vm.vsock.put(vsock_id="vsock0",
guest_cid=3,
uds_path="/{}".format(VSOCK_UDS_PATH))
# Configure metrics to assert against `sigpipe` count.
metrics_fifo_path = os.path.join(vm.path, 'metrics_fifo')
metrics_fifo = log_tools.Fifo(metrics_fifo_path)
response = vm.metrics.put(
metrics_path=vm.create_jailed_resource(metrics_fifo.path))
assert vm.api_session.is_status_no_content(response.status_code)
vm.start()
# Generate the random data blob file.
blob_path, blob_hash = make_blob(test_fc_session_root_path)
vm_blob_path = "/tmp/vsock/test.blob"
# Set up a tmpfs drive on the guest, so we can copy the blob there.
# Guest-initiated connections (echo workers) will use this blob.
conn = SSHConnection(vm.ssh_config)
cmd = "mkdir -p /tmp/vsock"
cmd += " && mount -t tmpfs tmpfs -o size={} /tmp/vsock".format(BLOB_SIZE +
1024 * 1024)
ecode, _, _ = conn.execute_command(cmd)
assert ecode == 0
# Copy `vsock_helper` and the random blob to the guest.
vsock_helper = bin_vsock_path
conn.scp_file(vsock_helper, '/bin/vsock_helper')
conn.scp_file(blob_path, vm_blob_path)
path = os.path.join(vm.jailer.chroot_path(), VSOCK_UDS_PATH)
# Negative test for host-initiated connections that
# are closed with in flight data.
negative_test_host_connections(vm, path, blob_path, blob_hash)
metrics = vm.flush_metrics(metrics_fifo)
# Validate that at least 1 `SIGPIPE` signal was received.
# Since we are reusing the existing echo server which triggers
# reads/writes on the UDS backend connections, these might be closed
# before a read() or a write() is about to be performed by the emulation.
# The test uses 100 connections it is enough to close at least one
# before write().
#
# If this ever fails due to 100 closes before read() we must
# add extra tooling that will trigger only writes().
assert metrics['signals']['sigpipe'] > 0
def test_vsock(
test_microvm_with_ssh,
network_config,
bin_vsock_path,
test_session_root_path
):
"""Vsock tests. See the module docstring for a high-level description."""
vm = test_microvm_with_ssh
vm.spawn()
vm.basic_config()
_tap, _, _ = vm.ssh_network_config(network_config, '1')
vm.vsock.put(
vsock_id="vsock0",
guest_cid=3,
uds_path="/{}".format(VSOCK_UDS_PATH)
)
vm.start()
# Generate the random data blob file.
blob_path, blob_hash = make_blob(test_session_root_path)
vm_blob_path = "/tmp/vsock/test.blob"
# Set up a tmpfs drive on the guest, so we can copy the blob there.
# Guest-initiated connections (echo workers) will use this blob.
conn = SSHConnection(vm.ssh_config)
cmd = "mkdir -p /tmp/vsock"
cmd += " && mount -t tmpfs tmpfs -o size={} /tmp/vsock".format(
BLOB_SIZE + 1024*1024
)
ecode, _, _ = conn.execute_command(cmd)
assert ecode == 0
# Copy `vsock_helper` and the random blob to the guest.
vsock_helper = bin_vsock_path
conn.scp_file(vsock_helper, '/bin/vsock_helper')
conn.scp_file(blob_path, vm_blob_path)
# Test guest-initiated connections.
path = os.path.join(
vm.path,
_make_host_port_path(VSOCK_UDS_PATH, ECHO_SERVER_PORT)
)
check_guest_connections(vm, path, vm_blob_path, blob_hash)
# Test host-initiated connections.
path = os.path.join(vm.jailer.chroot_path(), VSOCK_UDS_PATH)
check_host_connections(vm, path, blob_path, blob_hash)
def _test_seq_snapshots(context):
logger = context.custom['logger']
seq_len = context.custom['seq_len']
vm_builder = context.custom['builder']
snapshot_type = context.custom['snapshot_type']
bin_vsock_path = context.custom['bin_vsock_path']
test_session_root_path = context.custom['test_session_root_path']
enable_diff_snapshots = snapshot_type == SnapshotType.DIFF
logger.info("Testing {} with microvm: \"{}\", kernel {}, disk {} ".format(
snapshot_type, context.microvm.name(), context.kernel.name(),
context.disk.name()))
# Create a rw copy artifact.
root_disk = context.disk.copy()
# Get ssh key from read-only artifact.
ssh_key = context.disk.ssh_key()
# Create a fresh microvm from aftifacts.
basevm = vm_builder.build(kernel=context.kernel,
disks=[root_disk],
ssh_key=ssh_key,
config=context.microvm,
enable_diff_snapshots=enable_diff_snapshots)
# Configure vsock device.
basevm.vsock.put(vsock_id="vsock0",
guest_cid=3,
uds_path="/{}".format(VSOCK_UDS_PATH))
# Generate a random data file for vsock.
blob_path, blob_hash = make_blob(test_session_root_path)
basevm.start()
ssh_connection = net_tools.SSHConnection(basevm.ssh_config)
# Verify if guest can run commands.
exit_code, _, _ = ssh_connection.execute_command("sync")
assert exit_code == 0
# Copy the data file and a vsock helper to the guest.
cmd = "mkdir -p /tmp/vsock && mount -t tmpfs tmpfs /tmp/vsock"
ecode, _, _ = ssh_connection.execute_command(cmd)
assert ecode == 0, "Failed to set up tmpfs drive on the guest."
vsock_helper = bin_vsock_path
ssh_connection.scp_file(vsock_helper, '/bin/vsock_helper')
vm_blob_path = "/tmp/vsock/test.blob"
ssh_connection.scp_file(blob_path, vm_blob_path)
logger.info("Create {} #0.".format(snapshot_type))
# Create a snapshot builder from a microvm.
snapshot_builder = SnapshotBuilder(basevm)
# Create base snapshot.
snapshot = snapshot_builder.create([root_disk.local_path()], ssh_key,
snapshot_type)
base_snapshot = snapshot
basevm.kill()
for i in range(seq_len):
logger.info("Load snapshot #{}, mem {}".format(i, snapshot.mem))
microvm, _ = vm_builder.build_from_snapshot(snapshot, True,
enable_diff_snapshots)
# Attempt to connect to resumed microvm.
ssh_connection = net_tools.SSHConnection(microvm.ssh_config)
# Test vsock guest-initiated connections.
path = os.path.join(microvm.path,
"{}_{}".format(VSOCK_UDS_PATH, ECHO_SERVER_PORT))
check_guest_connections(microvm, path, vm_blob_path, blob_hash)
# Test vsock host-initiated connections.
path = os.path.join(microvm.jailer.chroot_path(), VSOCK_UDS_PATH)
check_host_connections(microvm, path, blob_path, blob_hash)
# Start a new instance of fio on each iteration.
_guest_run_fio_iteration(ssh_connection, i)
logger.info("Create snapshot #{}.".format(i + 1))
# Create a snapshot builder from the currently running microvm.
snapshot_builder = SnapshotBuilder(microvm)
snapshot = snapshot_builder.create([root_disk.local_path()], ssh_key,
snapshot_type)
# If we are testing incremental snapshots we must merge the base with
# current layer.
if snapshot_type == SnapshotType.DIFF:
logger.info("Base: {}, Layer: {}".format(base_snapshot.mem,
snapshot.mem))
snapshot.rebase_snapshot(base_snapshot)
# Update the base for next iteration.
base_snapshot = snapshot
microvm.kill()
def test_vsock_transport_reset(bin_cloner_path, bin_vsock_path,
test_fc_session_root_path):
"""
Vsock transport reset test.
Steps:
1. Start echo server on the guest
2. Start host workers that ping-pong data between guest and host,
without closing any of them
3. Pause VM -> Create snapshot -> Resume VM
4. Check that worker sockets no longer work by setting a timeout
so the sockets won't block and do a recv operation.
5. If the recv operation timeouts, the connection was closed.
Else, the connection was not closed and the test fails.
6. Close VM -> Load VM from Snapshot -> check that vsock
device is still working.
@type: functional
"""
vm_builder = MicrovmBuilder(bin_cloner_path)
vm_instance = vm_builder.build_vm_nano()
test_vm = vm_instance.vm
root_disk = vm_instance.disks[0]
ssh_key = vm_instance.ssh_key
test_vm.vsock.put(vsock_id="vsock0",
guest_cid=3,
uds_path="/{}".format(VSOCK_UDS_PATH))
test_vm.start()
snapshot_builder = SnapshotBuilder(test_vm)
disks = [root_disk.local_path()]
# Generate the random data blob file.
blob_path, blob_hash = make_blob(test_fc_session_root_path)
vm_blob_path = "/tmp/vsock/test.blob"
conn = SSHConnection(test_vm.ssh_config)
# Set up a tmpfs drive on the guest, so we can copy the blob there.
# Guest-initiated connections (echo workers) will use this blob.
_copy_vsock_data_to_guest(conn, blob_path, vm_blob_path, bin_vsock_path)
# Start guest echo server.
path = os.path.join(test_vm.jailer.chroot_path(), VSOCK_UDS_PATH)
conn = SSHConnection(test_vm.ssh_config)
cmd = "vsock_helper echosrv -d {}".format(ECHO_SERVER_PORT)
ecode, _, _ = conn.execute_command(cmd)
assert ecode == 0
# Start host workers that connect to the guest server.
workers = []
for _ in range(TEST_WORKER_COUNT):
worker = HostEchoWorker(path, blob_path)
workers.append(worker)
worker.start()
for wrk in workers:
wrk.join()
# Create snapshot.
snapshot = snapshot_builder.create(disks, ssh_key, SnapshotType.FULL)
response = test_vm.vm.patch(state="Resumed")
assert test_vm.api_session.is_status_no_content(response.status_code)
# Check that sockets are no longer working on workers.
for worker in workers:
# Whatever we send to the server, it should return the same
# value.
buf = bytearray("TEST\n".encode("utf-8"))
worker.sock.send(buf)
try:
# Arbitrary timeout, we set this so the socket won't block as
# it shouldn't receive anything.
worker.sock.settimeout(0.25)
response = worker.sock.recv(32)
# If we reach here, it means the connection did not close.
assert False, "Connection not closed: {}".format(
response.decode("utf-8"))
except SocketTimeout as exc:
assert True, exc
# Terminate VM.
test_vm.kill()
# Load snapshot.
test_vm, _ = vm_builder.build_from_snapshot(snapshot,
resume=True,
diff_snapshots=False)
# Check that vsock device still works.
# Test guest-initiated connections.
path = os.path.join(test_vm.path,
make_host_port_path(VSOCK_UDS_PATH, ECHO_SERVER_PORT))
check_guest_connections(test_vm, path, vm_blob_path, blob_hash)
# Test host-initiated connections.
path = os.path.join(test_vm.jailer.chroot_path(), VSOCK_UDS_PATH)
check_host_connections(test_vm, path, blob_path, blob_hash)
def _test_seq_snapshots(context):
logger = context.custom["logger"]
seq_len = context.custom["seq_len"]
vm_builder = context.custom["builder"]
snapshot_type = context.custom["snapshot_type"]
diff_snapshots = snapshot_type == SnapshotType.DIFF
logger.info('Testing {} with microvm: "{}", kernel {}, disk {} '.format(
snapshot_type,
context.microvm.name(),
context.kernel.name(),
context.disk.name(),
))
# Create a rw copy artifact.
root_disk = context.disk.copy()
# Get ssh key from read-only artifact.
ssh_key = context.disk.ssh_key()
# Create a fresh microvm from artifacts.
vm_instance = vm_builder.build(
kernel=context.kernel,
disks=[root_disk],
ssh_key=ssh_key,
config=context.microvm,
diff_snapshots=diff_snapshots,
)
basevm = vm_instance.vm
basevm.vsock.put(vsock_id="vsock0",
guest_cid=3,
uds_path="/{}".format(VSOCK_UDS_PATH))
basevm.start()
ssh_connection = net_tools.SSHConnection(basevm.ssh_config)
# Verify if guest can run commands.
exit_code, _, _ = ssh_connection.execute_command("sync")
assert exit_code == 0
test_fc_session_root_path = context.custom["test_fc_session_root_path"]
vsock_helper = context.custom["bin_vsock_path"]
vm_blob_path = "/tmp/vsock/test.blob"
# Generate a random data file for vsock.
blob_path, blob_hash = make_blob(test_fc_session_root_path)
# Copy the data file and a vsock helper to the guest.
_copy_vsock_data_to_guest(ssh_connection, blob_path, vm_blob_path,
vsock_helper)
logger.info("Create {} #0.".format(snapshot_type))
# Create a snapshot builder from a microvm.
snapshot_builder = SnapshotBuilder(basevm)
# Create base snapshot.
snapshot = snapshot_builder.create([root_disk.local_path()], ssh_key,
snapshot_type)
base_snapshot = snapshot
basevm.kill()
for i in range(seq_len):
logger.info("Load snapshot #{}, mem {}".format(i, snapshot.mem))
microvm, _ = vm_builder.build_from_snapshot(
snapshot, resume=True, diff_snapshots=diff_snapshots)
# Attempt to connect to resumed microvm.
ssh_connection = net_tools.SSHConnection(microvm.ssh_config)
# Test vsock guest-initiated connections.
path = os.path.join(
microvm.path, make_host_port_path(VSOCK_UDS_PATH,
ECHO_SERVER_PORT))
check_guest_connections(microvm, path, vm_blob_path, blob_hash)
# Test vsock host-initiated connections.
path = os.path.join(microvm.jailer.chroot_path(), VSOCK_UDS_PATH)
check_host_connections(microvm, path, blob_path, blob_hash)
# Start a new instance of fio on each iteration.
_guest_run_fio_iteration(ssh_connection, i)
logger.info("Create snapshot #{}.".format(i + 1))
# Create a snapshot builder from the currently running microvm.
snapshot_builder = SnapshotBuilder(microvm)
snapshot = snapshot_builder.create([root_disk.local_path()], ssh_key,
snapshot_type)
# If we are testing incremental snapshots we must merge the base with
# current layer.
if snapshot_type == SnapshotType.DIFF:
logger.info("Base: {}, Layer: {}".format(base_snapshot.mem,
snapshot.mem))
snapshot.rebase_snapshot(base_snapshot)
# Update the base for next iteration.
base_snapshot = snapshot
microvm.kill()
