def snapshot_create_producer(
logger,
vm,
disks,
ssh_key,
target_version,
metrics_fifo,
snapshot_type):
"""Produce results for snapshot create tests."""
snapshot_builder = SnapshotBuilder(vm)
snapshot_builder.create(disks=disks,
ssh_key=ssh_key,
snapshot_type=snapshot_type,
target_version=target_version,
use_ramdisk=True)
metrics = vm.flush_metrics(metrics_fifo)
if snapshot_type == SnapshotType.FULL:
value = metrics['latencies_us']['full_create_snapshot'] / USEC_IN_MSEC
else:
value = metrics['latencies_us']['diff_create_snapshot'] / USEC_IN_MSEC
logger.info("Latency {} ms".format(value))
return value
def _test_snapshot_create_latency(context):
logger = context.custom['logger']
vm_builder = context.custom['builder']
snapshot_type = context.custom['snapshot_type']
enable_diff_snapshots = snapshot_type == SnapshotType.DIFF
logger.info("""Measuring snapshot create({}) latency for microvm: \"{}\",
kernel {}, disk {} """.format(snapshot_type,
context.microvm.name(),
context.kernel.name(),
context.disk.name()))
# Create a rw copy artifact.
rw_disk = context.disk.copy()
# Get ssh key from read-only artifact.
ssh_key = context.disk.ssh_key()
# Measure a burst of snapshot create calls.
for i in range(SAMPLE_COUNT):
# Create a fresh microvm from aftifacts.
vm = vm_builder.build(kernel=context.kernel,
disks=[rw_disk],
ssh_key=ssh_key,
config=context.microvm,
enable_diff_snapshots=enable_diff_snapshots)
# Configure metrics system.
metrics_fifo_path = os.path.join(vm.path, 'metrics_fifo')
metrics_fifo = log_tools.Fifo(metrics_fifo_path)
response = vm.metrics.put(
metrics_path=vm.create_jailed_resource(metrics_fifo.path)
)
assert vm.api_session.is_status_no_content(response.status_code)
vm.start()
# Create a snapshot builder from a microvm.
snapshot_builder = SnapshotBuilder(vm)
snapshot_builder.create([rw_disk],
ssh_key,
snapshot_type)
metrics = vm.flush_metrics(metrics_fifo)
if snapshot_type == SnapshotType.FULL:
value = metrics['latencies_us']['full_create_snapshot']
baseline = CREATE_LATENCY_BASELINES[context.microvm.name()]['FULL']
else:
value = metrics['latencies_us']['diff_create_snapshot']
baseline = CREATE_LATENCY_BASELINES[context.microvm.name()]['DIFF']
value = value / USEC_IN_MSEC
assert baseline > value, "CreateSnapshot latency degraded."
logger.info("Latency {}/3: {} ms".format(i + 1, value))
vm.kill()
def _test_snapshot_compatibility(context):
logger = context.custom['logger']
vm_builder = context.custom['builder']
snapshot_type = context.custom['snapshot_type']
enable_diff_snapshots = snapshot_type == SnapshotType.DIFF
logger.info("Testing {} with microvm: \"{}\", kernel {}, disk {} ".format(
snapshot_type, context.microvm.name(), context.kernel.name(),
context.disk.name()))
# Create a rw copy artifact.
root_disk = context.disk.copy()
# Get ssh key from read-only artifact.
ssh_key = context.disk.ssh_key()
# Create a fresh microvm from aftifacts.
microvm = vm_builder.build(kernel=context.kernel,
disks=[root_disk],
ssh_key=ssh_key,
config=context.microvm,
enable_diff_snapshots=enable_diff_snapshots)
# Add a memory balloon with stats enabled.
response = microvm.balloon.put(amount_mib=0,
deflate_on_oom=True,
stats_polling_interval_s=1)
assert microvm.api_session.is_status_no_content(response.status_code)
microvm.start()
logger.info("Create {} #0.".format(snapshot_type))
# Pause the microVM in order to allow snapshots
response = microvm.vm.patch(state='Paused')
assert microvm.api_session.is_status_no_content(response.status_code)
# Try to create a snapshot with a balloon on version 0.23.0.
# This is skipped for aarch64, since the snapshotting feature
# was introduced in v0.24.0.
if platform.machine() == "x86_64":
response = microvm.snapshot.create(mem_file_path='memfile',
snapshot_path='dummy',
diff=False,
version='0.23.0')
# This should fail as the balloon was introduced in 0.24.0.
assert microvm.api_session.is_status_bad_request(response.status_code)
assert ('Target version does not implement the '
'virtio-balloon device') in response.json()['fault_message']
# Create a snapshot builder from a microvm.
snapshot_builder = SnapshotBuilder(microvm)
# Check we can create a snapshot with a balloon on current version.
snapshot_builder.create([root_disk.local_path()], ssh_key, snapshot_type)
microvm.kill()
def create_snapshot(bin_cloner_path):
"""Create a snapshot of a microVM."""
vm_builder = MicrovmBuilder(bin_cloner_path)
vm_instance = vm_builder.build_vm_nano()
basevm = vm_instance.vm
root_disk = vm_instance.disks[0]
ssh_key = vm_instance.ssh_key
# Add a memory balloon.
response = basevm.balloon.put(amount_mib=0,
deflate_on_oom=True,
stats_polling_interval_s=0)
assert basevm.api_session.is_status_no_content(response.status_code)
basevm.start()
ssh_connection = net_tools.SSHConnection(basevm.ssh_config)
# Verify if guest can run commands.
exit_code, _, _ = ssh_connection.execute_command("sync")
assert exit_code == 0
# Create a snapshot builder from a microvm.
snapshot_builder = SnapshotBuilder(basevm)
# Create base snapshot.
snapshot = snapshot_builder.create([root_disk.local_path()], ssh_key)
basevm.kill()
return snapshot
def create_512mb_full_snapshot(bin_cloner_path,
target_version: str = None,
fc_binary=None,
jailer_binary=None):
"""Create a snapshoft from a 2vcpu 512MB microvm."""
vm_instance = C3micro.spawn(bin_cloner_path, True, fc_binary,
jailer_binary)
# Attempt to connect to the fresh microvm.
ssh_connection = net_tools.SSHConnection(vm_instance.vm.ssh_config)
# Run a fio workload and validate succesfull execution.
fio = """fio --filename=/dev/vda --direct=1 --rw=randread --bs=4k \
--ioengine=libaio --iodepth=16 --runtime=2 --numjobs=4 --time_based \
--group_reporting --name=iops-test-job --eta-newline=1 --readonly"""
exit_code, _, _ = ssh_connection.execute_command(fio)
assert exit_code == 0
# Create a snapshot builder from a microvm.
snapshot_builder = SnapshotBuilder(vm_instance.vm)
# The snapshot builder expects disks as paths, not artifacts.
disks = []
for disk in vm_instance.disks:
disks.append(disk.local_path())
snapshot = snapshot_builder.create(disks, vm_instance.ssh_key,
SnapshotType.FULL, target_version)
vm_instance.vm.kill()
return snapshot
def test_patch_drive_snapshot(bin_cloner_path):
"""
Test that a patched drive is correctly used by guests loaded from snapshot.
@type: functional
"""
logger = logging.getLogger("snapshot_sequence")
vm_builder = MicrovmBuilder(bin_cloner_path)
snapshot_type = SnapshotType.FULL
diff_snapshots = False
# Use a predefined vm instance.
vm_instance = vm_builder.build_vm_nano()
basevm = vm_instance.vm
root_disk = vm_instance.disks[0]
ssh_key = vm_instance.ssh_key
# Add a scratch 128MB RW non-root block device.
scratchdisk1 = drive_tools.FilesystemFile(tempfile.mktemp(), size=128)
basevm.add_drive("scratch", scratchdisk1.path)
basevm.start()
ssh_connection = net_tools.SSHConnection(basevm.ssh_config)
# Verify if guest can run commands.
exit_code, _, _ = ssh_connection.execute_command("sync")
assert exit_code == 0
# Update drive to have another backing file, double in size.
new_file_size_mb = 2 * int(scratchdisk1.size() / (1024 * 1024))
logger.info("Patch drive, new file: size %sMB.", new_file_size_mb)
scratchdisk1 = drive_tools.FilesystemFile(tempfile.mktemp(),
new_file_size_mb)
basevm.patch_drive("scratch", scratchdisk1)
logger.info("Create %s #0.", snapshot_type)
# Create a snapshot builder from a microvm.
snapshot_builder = SnapshotBuilder(basevm)
disks = [root_disk.local_path(), scratchdisk1.path]
# Create base snapshot.
snapshot = snapshot_builder.create(disks, ssh_key, snapshot_type)
basevm.kill()
# Load snapshot in a new Firecracker microVM.
logger.info("Load snapshot, mem %s", snapshot.mem)
microvm, _ = vm_builder.build_from_snapshot(snapshot,
resume=True,
diff_snapshots=diff_snapshots)
# Attempt to connect to resumed microvm.
ssh_connection = net_tools.SSHConnection(microvm.ssh_config)
# Verify the new microVM has the right scratch drive.
guest_drive_size = _get_guest_drive_size(ssh_connection)
assert guest_drive_size == str(scratchdisk1.size())
microvm.kill()
def create_snapshots(context):
"""Snapshot microVM built from vm configuration file."""
vm = setup_vm(context)
# Get ssh key from read-only artifact.
ssh_key = context.disk.ssh_key()
ssh_key.download(vm.path)
vm.ssh_config['ssh_key_path'] = ssh_key.local_path()
os.chmod(vm.ssh_config['ssh_key_path'], 0o400)
cpu_template = context.custom['cpu_template']
fn = partial(add_cpu_template, cpu_template)
_configure_vm_from_json(vm, VM_CONFIG_FILE, json_xform=fn)
configure_network_interfaces(vm)
vm.spawn()
# Ensure the microVM has started.
response = vm.machine_cfg.get()
assert vm.api_session.is_status_ok(response.status_code)
assert vm.state == "Running"
# Populate MMDS.
data_store = {
'latest': {
'meta-data': {
'ami-id': 'ami-12345678',
'reservation-id': 'r-fea54097',
'local-hostname': 'ip-10-251-50-12.ec2.internal',
'public-hostname': 'ec2-203-0-113-25.compute-1.amazonaws.com'
}
}
}
populate_mmds(vm, data_store)
# Iterate and validate connectivity on all ifaces after boot.
for iface in net_ifaces:
vm.ssh_config['hostname'] = iface.guest_ip
ssh_connection = net_tools.SSHConnection(vm.ssh_config)
exit_code, _, _ = ssh_connection.execute_command("sync")
assert exit_code == 0
# Validate MMDS.
validate_mmds(ssh_connection, data_store)
# Create a snapshot builder from a microVM.
snapshot_builder = SnapshotBuilder(vm)
# Snapshot the microVM.
snapshot = snapshot_builder.create([vm.rootfs_file],
ssh_key,
SnapshotType.DIFF,
net_ifaces=net_ifaces)
copy_snapshot_artifacts(snapshot, vm.rootfs_file, context.kernel.name(),
ssh_key, cpu_template)
vm.kill()
def _test_compare_mem_files(context):
logger = context.custom["logger"]
vm_builder = context.custom["builder"]
# Create a rw copy artifact.
root_disk = context.disk.copy()
# Get ssh key from read-only artifact.
ssh_key = context.disk.ssh_key()
# Create a fresh microvm from artifacts.
vm_instance = vm_builder.build(
kernel=context.kernel,
disks=[root_disk],
ssh_key=ssh_key,
config=context.microvm,
diff_snapshots=True,
)
basevm = vm_instance.vm
basevm.start()
ssh_connection = net_tools.SSHConnection(basevm.ssh_config)
# Verify if guest can run commands.
exit_code, _, _ = ssh_connection.execute_command("sync")
assert exit_code == 0
# Create a snapshot builder from a microvm.
snapshot_builder = SnapshotBuilder(basevm)
logger.info("Create full snapshot.")
# Create full snapshot.
full_snapshot = snapshot_builder.create([root_disk.local_path()], ssh_key,
SnapshotType.FULL)
logger.info("Create diff snapshot.")
# Create diff snapshot.
diff_snapshot = snapshot_builder.create(
[root_disk.local_path()],
ssh_key,
SnapshotType.DIFF,
mem_file_name="diff_vm.mem",
snapshot_name="diff_vm.vmstate",
)
assert filecmp.cmp(full_snapshot.mem, diff_snapshot.mem)
basevm.kill()
def test_negative_postload_api(bin_cloner_path):
"""Test APIs fail after loading from snapshot."""
logger = logging.getLogger("snapshot_api_fail")
vm_builder = MicrovmBuilder(bin_cloner_path)
vm_instance = VMNano.spawn(bin_cloner_path, diff_snapshots=True)
basevm = vm_instance.vm
root_disk = vm_instance.disks[0]
ssh_key = vm_instance.ssh_key
basevm.start()
ssh_connection = net_tools.SSHConnection(basevm.ssh_config)
# Verify if guest can run commands.
exit_code, _, _ = ssh_connection.execute_command("sync")
assert exit_code == 0
logger.info("Create snapshot")
# Create a snapshot builder from a microvm.
snapshot_builder = SnapshotBuilder(basevm)
# Create base snapshot.
snapshot = snapshot_builder.create([root_disk.local_path()],
ssh_key,
SnapshotType.DIFF)
basevm.kill()
logger.info("Load snapshot, mem %s", snapshot.mem)
# Do not resume, just load, so we can still call APIs that work.
microvm, _ = vm_builder.build_from_snapshot(snapshot,
False,
True)
fail_msg = "The requested operation is not supported after starting " \
"the microVM"
try:
microvm.start()
except AssertionError as error:
assert fail_msg in str(error)
else:
assert False, "Negative test failed"
try:
microvm.basic_config()
except AssertionError as error:
assert fail_msg in str(error)
else:
assert False, "Negative test failed"
microvm.kill()
def test_serial_after_snapshot(bin_cloner_path):
"""
Serial I/O after restoring from a snapshot.
@type: functional
"""
vm_builder = MicrovmBuilder(bin_cloner_path)
vm_instance = vm_builder.build_vm_nano(
diff_snapshots=False,
daemonize=False,
)
microvm = vm_instance.vm
root_disk = vm_instance.disks[0]
ssh_key = vm_instance.ssh_key
microvm.start()
serial = Serial(microvm)
serial.open()
# Image used for tests on aarch64 has autologon
if platform.machine() == "x86_64":
serial.rx(token="login: "******"root")
serial.rx("Password: "******"root")
# Make sure that at the time we snapshot the vm, the user is logged in.
serial.rx("#")
snapshot_builder = SnapshotBuilder(microvm)
disks = [root_disk.local_path()]
# Create diff snapshot.
snapshot = snapshot_builder.create(disks, ssh_key, SnapshotType.FULL)
# Kill base microVM.
microvm.kill()
# Load microVM clone from snapshot.
test_microvm, _ = vm_builder.build_from_snapshot(snapshot,
resume=True,
diff_snapshots=False,
daemonize=False)
serial = Serial(test_microvm)
serial.open()
# We need to send a newline to signal the serial to flush
# the login content.
serial.tx("")
serial.rx("#")
serial.tx("pwd")
res = serial.rx("#")
assert "/root" in res
def test_mmds_snapshot(bin_cloner_path):
"""
Exercise MMDS behavior with snapshots.
Ensures that MMDS V2 behavior is not affected by taking a snapshot
and that MMDS V2 is not available after snapshot load.
@type: functional
"""
vm_builder = MicrovmBuilder(bin_cloner_path)
net_iface = NetIfaceConfig()
vm_instance = vm_builder.build_vm_nano(
net_ifaces=[net_iface],
diff_snapshots=True
)
test_microvm = vm_instance.vm
root_disk = vm_instance.disks[0]
ssh_key = vm_instance.ssh_key
ipv4_address = '169.254.169.250'
# Configure MMDS version with custom IPv4 address.
_configure_mmds(
test_microvm,
version='V2',
iface_id=DEFAULT_DEV_NAME,
ipv4_address=ipv4_address
)
data_store = {
'latest': {
'meta-data': {
'ami-id': 'ami-12345678'
}
}
}
_populate_data_store(test_microvm, data_store)
test_microvm.start()
snapshot_builder = SnapshotBuilder(test_microvm)
disks = [root_disk.local_path()]
ssh_connection = net_tools.SSHConnection(test_microvm.ssh_config)
cmd = 'ip route add {} dev eth0'.format(ipv4_address)
_, stdout, stderr = ssh_connection.execute_command(cmd)
_assert_out(stdout, stderr, '')
# Generate token.
token = generate_mmds_session_token(
ssh_connection,
ipv4_address=ipv4_address,
token_ttl=60
)
pre = 'curl -m 2 -s'
pre += ' -X GET'
pre += ' -H "X-metadata-token: {}"'.format(token)
pre += ' http://{}/'.format(ipv4_address)
# Fetch metadata.
cmd = pre + 'latest/meta-data/'
_, stdout, stderr = ssh_connection.execute_command(cmd)
_assert_out(stdout, stderr, "ami-id")
# Create diff snapshot.
snapshot = snapshot_builder.create(disks,
ssh_key,
SnapshotType.DIFF)
# Resume microVM and ensure session token is still valid on the base.
response = test_microvm.vm.patch(state='Resumed')
assert test_microvm.api_session.is_status_no_content(response.status_code)
_, stdout, stderr = ssh_connection.execute_command(
pre + 'latest/meta-data/'
)
_assert_out(stdout, stderr, "ami-id")
# Kill base microVM.
test_microvm.kill()
# Load microVM clone from snapshot.
test_microvm, _ = vm_builder.build_from_snapshot(snapshot,
resume=True,
diff_snapshots=True)
_populate_data_store(test_microvm, data_store)
ssh_connection = net_tools.SSHConnection(test_microvm.ssh_config)
# Mmds V2 is not available with snapshots.
# Test that `PUT` requests are not allowed.
cmd = 'curl -m 2 -s'
cmd += ' -X PUT'
cmd += ' -H "X-metadata-token-ttl-seconds: 1"'
cmd += ' http://{}/latest/api/token'.format(ipv4_address)
_, stdout, stderr = ssh_connection.execute_command(cmd)
expected = "Not allowed HTTP method."
_assert_out(stdout, stderr, expected)
# Fetch metadata using V1 requests and ensure IPv4 configuration
# is persistent between snapshots.
cmd = 'curl -s http://{}/latest/meta-data/ami-id/'.format(ipv4_address)
_, stdout, stderr = ssh_connection.execute_command(cmd)
_assert_out(stdout, stderr, 'ami-12345678')
def test_vsock_transport_reset(bin_cloner_path, bin_vsock_path,
test_fc_session_root_path):
"""
Vsock transport reset test.
Steps:
1. Start echo server on the guest
2. Start host workers that ping-pong data between guest and host,
without closing any of them
3. Pause VM -> Create snapshot -> Resume VM
4. Check that worker sockets no longer work by setting a timeout
so the sockets won't block and do a recv operation.
5. If the recv operation timeouts, the connection was closed.
Else, the connection was not closed and the test fails.
6. Close VM -> Load VM from Snapshot -> check that vsock
device is still working.
@type: functional
"""
vm_builder = MicrovmBuilder(bin_cloner_path)
vm_instance = vm_builder.build_vm_nano()
test_vm = vm_instance.vm
root_disk = vm_instance.disks[0]
ssh_key = vm_instance.ssh_key
test_vm.vsock.put(vsock_id="vsock0",
guest_cid=3,
uds_path="/{}".format(VSOCK_UDS_PATH))
test_vm.start()
snapshot_builder = SnapshotBuilder(test_vm)
disks = [root_disk.local_path()]
# Generate the random data blob file.
blob_path, blob_hash = make_blob(test_fc_session_root_path)
vm_blob_path = "/tmp/vsock/test.blob"
conn = SSHConnection(test_vm.ssh_config)
# Set up a tmpfs drive on the guest, so we can copy the blob there.
# Guest-initiated connections (echo workers) will use this blob.
_copy_vsock_data_to_guest(conn, blob_path, vm_blob_path, bin_vsock_path)
# Start guest echo server.
path = os.path.join(test_vm.jailer.chroot_path(), VSOCK_UDS_PATH)
conn = SSHConnection(test_vm.ssh_config)
cmd = "vsock_helper echosrv -d {}".format(ECHO_SERVER_PORT)
ecode, _, _ = conn.execute_command(cmd)
assert ecode == 0
# Start host workers that connect to the guest server.
workers = []
for _ in range(TEST_WORKER_COUNT):
worker = HostEchoWorker(path, blob_path)
workers.append(worker)
worker.start()
for wrk in workers:
wrk.join()
# Create snapshot.
snapshot = snapshot_builder.create(disks, ssh_key, SnapshotType.FULL)
response = test_vm.vm.patch(state="Resumed")
assert test_vm.api_session.is_status_no_content(response.status_code)
# Check that sockets are no longer working on workers.
for worker in workers:
# Whatever we send to the server, it should return the same
# value.
buf = bytearray("TEST\n".encode("utf-8"))
worker.sock.send(buf)
try:
# Arbitrary timeout, we set this so the socket won't block as
# it shouldn't receive anything.
worker.sock.settimeout(0.25)
response = worker.sock.recv(32)
# If we reach here, it means the connection did not close.
assert False, "Connection not closed: {}".format(
response.decode("utf-8"))
except SocketTimeout as exc:
assert True, exc
# Terminate VM.
test_vm.kill()
# Load snapshot.
test_vm, _ = vm_builder.build_from_snapshot(snapshot,
resume=True,
diff_snapshots=False)
# Check that vsock device still works.
# Test guest-initiated connections.
path = os.path.join(test_vm.path,
make_host_port_path(VSOCK_UDS_PATH, ECHO_SERVER_PORT))
check_guest_connections(test_vm, path, vm_blob_path, blob_hash)
# Test host-initiated connections.
path = os.path.join(test_vm.jailer.chroot_path(), VSOCK_UDS_PATH)
check_host_connections(test_vm, path, blob_path, blob_hash)
def _test_older_snapshot_resume_latency(context):
builder = context.custom["builder"]
logger = context.custom["logger"]
snapshot_type = context.custom['snapshot_type']
file_dumper = context.custom["results_file_dumper"]
firecracker = context.firecracker
jailer = firecracker.jailer()
jailer.download()
fc_version = firecracker.base_name()[1:]
logger.info("Firecracker version: %s", fc_version)
logger.info("Source Firecracker: %s", firecracker.local_path())
logger.info("Source Jailer: %s", jailer.local_path())
# Create a fresh microvm with the binary artifacts.
vm_instance = builder.build_vm_micro(firecracker.local_path(),
jailer.local_path())
basevm = vm_instance.vm
basevm.start()
ssh_connection = net_tools.SSHConnection(basevm.ssh_config)
# Check if guest works.
exit_code, _, _ = ssh_connection.execute_command("ls")
assert exit_code == 0
# The snapshot builder expects disks as paths, not artifacts.
disks = []
for disk in vm_instance.disks:
disks.append(disk.local_path())
# Create a snapshot builder from a microvm.
snapshot_builder = SnapshotBuilder(basevm)
snapshot = snapshot_builder.create(disks, vm_instance.ssh_key,
snapshot_type)
basevm.kill()
st_core = core.Core(name="older_snapshot_resume_latency",
iterations=SAMPLE_COUNT)
prod = producer.LambdaProducer(func=snapshot_resume_producer,
func_kwargs={
"logger": logger,
"vm_builder": builder,
"snapshot": snapshot,
"snapshot_type": snapshot_type,
"use_ramdisk": False
})
cons = consumer.LambdaConsumer(func=lambda cons, result: cons.consume_stat(
st_name="max", ms_name="latency", value=result),
func_kwargs={})
eager_map(cons.set_measurement_def,
snapshot_resume_measurements(context.microvm.name()))
st_core.add_pipe(producer=prod, consumer=cons, tag=context.microvm.name())
# Gather results and verify pass criteria.
try:
result = st_core.run_exercise()
except core.CoreException as err:
handle_failure(file_dumper, err)
dump_test_result(file_dumper, result)
def _test_snapshot_resume_latency(context):
logger = context.custom['logger']
vm_builder = context.custom['builder']
snapshot_type = context.custom['snapshot_type']
file_dumper = context.custom['results_file_dumper']
diff_snapshots = snapshot_type == SnapshotType.DIFF
logger.info("""Measuring snapshot resume({}) latency for microvm: \"{}\",
kernel {}, disk {} """.format(snapshot_type, context.microvm.name(),
context.kernel.name(), context.disk.name()))
# Create a rw copy artifact.
rw_disk = context.disk.copy()
# Get ssh key from read-only artifact.
ssh_key = context.disk.ssh_key()
# Create a fresh microvm from aftifacts.
vm_instance = vm_builder.build(kernel=context.kernel,
disks=[rw_disk],
ssh_key=ssh_key,
config=context.microvm,
diff_snapshots=diff_snapshots,
use_ramdisk=True)
basevm = vm_instance.vm
basevm.start()
ssh_connection = net_tools.SSHConnection(basevm.ssh_config)
# Check if guest works.
exit_code, _, _ = ssh_connection.execute_command("ls")
assert exit_code == 0
logger.info("Create {}.".format(snapshot_type))
# Create a snapshot builder from a microvm.
snapshot_builder = SnapshotBuilder(basevm)
snapshot = snapshot_builder.create([rw_disk.local_path()],
ssh_key,
snapshot_type,
use_ramdisk=True)
basevm.kill()
st_core = core.Core(name="snapshot_resume_latency",
iterations=SAMPLE_COUNT)
prod = producer.LambdaProducer(func=snapshot_resume_producer,
func_kwargs={
"logger": logger,
"vm_builder": vm_builder,
"snapshot": snapshot,
"snapshot_type": snapshot_type,
"use_ramdisk": True
})
cons = consumer.LambdaConsumer(func=lambda cons, result: cons.consume_stat(
st_name="max", ms_name="latency", value=result),
func_kwargs={})
eager_map(cons.set_measurement_def,
snapshot_resume_measurements(context.microvm.name()))
st_core.add_pipe(producer=prod, consumer=cons, tag=context.microvm.name())
# Gather results and verify pass criteria.
try:
result = st_core.run_exercise()
except core.CoreException as err:
handle_failure(file_dumper, err)
dump_test_result(file_dumper, result)
def create_snapshot_helper(builder, logger, target_version=None,
drives=None, ifaces=None,
balloon=False, diff_snapshots=False,
fc_binary=None, jailer_binary=None):
"""Create a snapshot with many devices."""
vm_instance = builder.build_vm_nano(net_ifaces=ifaces,
diff_snapshots=diff_snapshots,
fc_binary=fc_binary,
jailer_binary=jailer_binary)
vm = vm_instance.vm
if diff_snapshots is False:
snapshot_type = SnapshotType.FULL
else:
# Version 0.24 and greater has Diff and balloon support.
snapshot_type = SnapshotType.DIFF
if balloon:
# Add a memory balloon with stats enabled.
response = vm.balloon.put(
amount_mib=0,
deflate_on_oom=True,
stats_polling_interval_s=1
)
assert vm.api_session.is_status_no_content(response.status_code)
# Disk path array needed when creating the snapshot later.
disks = [vm_instance.disks[0].local_path()]
test_drives = [] if drives is None else drives
# Add disks.
for scratch in test_drives:
# Add a scratch 64MB RW non-root block device.
scratchdisk = drive_tools.FilesystemFile(tempfile.mktemp(), size=64)
vm.add_drive(scratch, scratchdisk.path)
disks.append(scratchdisk.path)
# Workaround FilesystemFile destructor removal of file.
scratchdisk.path = None
vm.start()
# Iterate and validate connectivity on all ifaces after boot.
for iface in ifaces:
vm.ssh_config['hostname'] = iface.guest_ip
ssh_connection = net_tools.SSHConnection(vm.ssh_config)
exit_code, _, _ = ssh_connection.execute_command("sync")
assert exit_code == 0
# Mount scratch drives in guest.
for blk in test_drives:
# Create mount point and mount each device.
cmd = "mkdir -p /mnt/{blk} && mount /dev/{blk} /mnt/{blk}".format(
blk=blk
)
exit_code, _, _ = ssh_connection.execute_command(cmd)
assert exit_code == 0
# Create file using dd using O_DIRECT.
# After resume we will compute md5sum on these files.
dd = "dd if=/dev/zero of=/mnt/{}/test bs=4096 count=10 oflag=direct"
exit_code, _, _ = ssh_connection.execute_command(dd.format(blk))
assert exit_code == 0
# Unmount the device.
cmd = "umount /dev/{}".format(blk)
exit_code, _, _ = ssh_connection.execute_command(cmd)
assert exit_code == 0
# Create a snapshot builder from a microvm.
snapshot_builder = SnapshotBuilder(vm)
snapshot = snapshot_builder.create(disks,
vm_instance.ssh_key,
target_version=target_version,
snapshot_type=snapshot_type,
net_ifaces=ifaces)
logger.debug("========== Firecracker create snapshot log ==========")
logger.debug(vm.log_data)
vm.kill()
return snapshot
def _test_patch_drive_snapshot(context):
logger = context.custom['logger']
vm_builder = context.custom['builder']
snapshot_type = SnapshotType.FULL
enable_diff_snapshots = False
logger.info(
"Testing patch drive snapshot on \"{}\", kernel {}, disk {} ".format(
context.microvm.name(), context.kernel.name(),
context.disk.name()))
# Create a rw copy artifact.
root_disk = context.disk.copy()
# Get ssh key from read-only artifact.
ssh_key = context.disk.ssh_key()
# Create a fresh microvm from aftifacts.
basevm = vm_builder.build(kernel=context.kernel,
disks=[root_disk],
ssh_key=ssh_key,
config=context.microvm,
enable_diff_snapshots=enable_diff_snapshots)
network_config = net_tools.UniqueIPv4Generator.instance()
_, host_ip, guest_ip = basevm.ssh_network_config(network_config,
'1',
tapname="tap0")
logger.debug("Host IP: {}, Guest IP: {}".format(host_ip, guest_ip))
# Add a scratch 128MB RW non-root block device.
scratchdisk1 = drive_tools.FilesystemFile(tempfile.mktemp(), size=128)
basevm.add_drive('scratch', scratchdisk1.path)
# We will need netmask_len in build_from_snapshot() call later.
netmask_len = network_config.get_netmask_len()
basevm.start()
ssh_connection = net_tools.SSHConnection(basevm.ssh_config)
# Verify if guest can run commands.
exit_code, _, _ = ssh_connection.execute_command("sync")
assert exit_code == 0
# Update drive to have another backing file, double in size.
new_file_size_mb = 2 * int(scratchdisk1.size() / (1024 * 1024))
logger.info("Patch drive, new file: size {}MB.".format(new_file_size_mb))
scratchdisk1 = drive_tools.FilesystemFile(tempfile.mktemp(),
new_file_size_mb)
basevm.patch_drive('scratch', scratchdisk1)
logger.info("Create {} #0.".format(snapshot_type))
# Create a snapshot builder from a microvm.
snapshot_builder = SnapshotBuilder(basevm)
disks = [root_disk.local_path(), scratchdisk1.path]
# Create base snapshot.
snapshot = snapshot_builder.create(disks, ssh_key, snapshot_type)
basevm.kill()
# Load snapshot in a new Firecracker microVM.
logger.info("Load snapshot, mem {}".format(snapshot.mem))
microvm, _ = vm_builder.build_from_snapshot(snapshot, host_ip, guest_ip,
netmask_len, True,
enable_diff_snapshots)
# Attempt to connect to resumed microvm.
ssh_connection = net_tools.SSHConnection(microvm.ssh_config)
# Verify the new microVM has the right scratch drive.
guest_drive_size = _get_guest_drive_size(ssh_connection)
assert guest_drive_size == str(scratchdisk1.size())
microvm.kill()
def _test_snapshot_resume_latency(context):
logger = context.custom['logger']
vm_builder = context.custom['builder']
snapshot_type = context.custom['snapshot_type']
enable_diff_snapshots = snapshot_type == SnapshotType.DIFF
logger.info("""Measuring snapshot resume({}) latency for microvm: \"{}\",
kernel {}, disk {} """.format(snapshot_type, context.microvm.name(),
context.kernel.name(), context.disk.name()))
# Create a rw copy artifact.
rw_disk = context.disk.copy()
# Get ssh key from read-only artifact.
ssh_key = context.disk.ssh_key()
# Create a fresh microvm from aftifacts.
basevm = vm_builder.build(kernel=context.kernel,
disks=[rw_disk],
ssh_key=ssh_key,
config=context.microvm,
enable_diff_snapshots=enable_diff_snapshots)
basevm.start()
ssh_connection = net_tools.SSHConnection(basevm.ssh_config)
# Check if guest works.
exit_code, _, _ = ssh_connection.execute_command("sync")
assert exit_code == 0
logger.info("Create {}.".format(snapshot_type))
# Create a snapshot builder from a microvm.
snapshot_builder = SnapshotBuilder(basevm)
snapshot = snapshot_builder.create([rw_disk.local_path()], ssh_key,
snapshot_type)
basevm.kill()
for i in range(SAMPLE_COUNT):
microvm, metrics_fifo = vm_builder.build_from_snapshot(
snapshot, True, enable_diff_snapshots)
# Attempt to connect to resumed microvm.
ssh_connection = net_tools.SSHConnection(microvm.ssh_config)
# Verify if guest can run commands.
exit_code, _, _ = ssh_connection.execute_command("sync")
assert exit_code == 0
value = 0
# Parse all metric data points in search of load_snapshot time.
metrics = microvm.get_all_metrics(metrics_fifo)
for data_point in metrics:
metrics = json.loads(data_point)
cur_value = metrics['latencies_us']['load_snapshot'] / USEC_IN_MSEC
if cur_value > 0:
value = cur_value
break
baseline = LOAD_LATENCY_BASELINES[context.microvm.name()]
logger.info("Latency {}/{}: {} ms".format(i + 1, SAMPLE_COUNT, value))
assert baseline > value, "LoadSnapshot latency degraded."
microvm.kill()
def _validate_mmds_snapshot(
vm_instance,
vm_builder,
version,
target_fc_version=None,
fc_path=None,
jailer_path=None
):
"""Test MMDS behaviour across snap-restore."""
basevm = vm_instance.vm
root_disk = vm_instance.disks[0]
disks = [root_disk.local_path()]
ssh_key = vm_instance.ssh_key
ipv4_address = '169.254.169.250'
# Configure MMDS version with custom IPv4 address.
configure_mmds(
basevm,
version=version,
iface_ids=[DEFAULT_DEV_NAME],
ipv4_address=ipv4_address,
fc_version=target_fc_version
)
# Check if the FC version supports the latest format for mmds-config.
# If target_fc_version is None, we assume the current version is used.
if target_fc_version is None or \
(target_fc_version is not None and
compare_versions(target_fc_version, "1.0.0") >= 0):
expected_mmds_config = {
"version": version,
"ipv4_address": ipv4_address,
"network_interfaces": [DEFAULT_DEV_NAME]
}
response = basevm.full_cfg.get()
assert basevm.api_session.is_status_ok(response.status_code)
assert response.json()["mmds-config"] == expected_mmds_config
data_store = {
'latest': {
'meta-data': {
'ami-id': 'ami-12345678'
}
}
}
_populate_data_store(basevm, data_store)
basevm.start()
snapshot_builder = SnapshotBuilder(basevm)
ssh_connection = net_tools.SSHConnection(basevm.ssh_config)
_run_guest_cmd(ssh_connection, f'ip route add {ipv4_address} dev eth0', '')
# Generate token if needed.
token = None
if version == "V2":
token = generate_mmds_session_token(
ssh_connection,
ipv4_address,
token_ttl=60
)
# Fetch metadata.
cmd = generate_mmds_get_request(
ipv4_address,
token=token,
)
_run_guest_cmd(ssh_connection, cmd, data_store, use_json=True)
# Create snapshot.
snapshot = snapshot_builder.create(disks,
ssh_key,
SnapshotType.FULL,
target_version=target_fc_version)
# Resume microVM and ensure session token is still valid on the base.
response = basevm.vm.patch(state='Resumed')
assert basevm.api_session.is_status_no_content(response.status_code)
# Fetch metadata again using the same token.
_run_guest_cmd(ssh_connection, cmd, data_store, use_json=True)
# Kill base microVM.
basevm.kill()
# Load microVM clone from snapshot.
microvm, _ = vm_builder.build_from_snapshot(snapshot,
resume=True,
fc_binary=fc_path,
jailer_binary=jailer_path)
ssh_connection = net_tools.SSHConnection(microvm.ssh_config)
# Check the reported mmds config. In versions up to (including) v1.0.0 this
# was not populated after restore.
if target_fc_version is not None and \
compare_versions("1.0.0", target_fc_version) < 0:
response = microvm.full_cfg.get()
assert microvm.api_session.is_status_ok(response.status_code)
assert response.json()["mmds-config"] == expected_mmds_config
if version == 'V1':
# Verify that V2 requests don't work
assert generate_mmds_session_token(
ssh_connection,
ipv4_address,
token_ttl=60
) == "Not allowed HTTP method."
token = None
else:
# Attempting to reuse the token across a restore must fail.
cmd = generate_mmds_get_request(ipv4_address, token=token)
_run_guest_cmd(ssh_connection, cmd, 'MMDS token not valid.')
# Generate token.
token = generate_mmds_session_token(
ssh_connection,
ipv4_address,
token_ttl=60
)
# Data store is empty after a restore.
cmd = generate_mmds_get_request(ipv4_address, token=token)
_run_guest_cmd(ssh_connection, cmd, 'null')
# Now populate the store.
_populate_data_store(microvm, data_store)
# Fetch metadata.
_run_guest_cmd(ssh_connection, cmd, data_store, use_json=True)
def get_snap_restore_latency(context,
vcpus,
mem_size,
nets=1,
blocks=1,
all_devices=False,
iterations=10):
"""Restore snapshots with various configs to measure latency."""
vm_builder = context.custom['builder']
# Create a rw copy artifact.
rw_disk = context.disk.copy()
# Get ssh key from read-only artifact.
ssh_key = context.disk.ssh_key()
ifaces = None
if nets > 1:
ifaces = net_ifaces[:nets]
# Create a fresh microvm from artifacts.
vm_instance = vm_builder.build(kernel=context.kernel,
disks=[rw_disk],
ssh_key=ssh_key,
config=context.microvm,
net_ifaces=ifaces,
use_ramdisk=True)
basevm = vm_instance.vm
response = basevm.machine_cfg.put(vcpu_count=vcpus,
mem_size_mib=mem_size,
ht_enabled=False)
assert basevm.api_session.is_status_no_content(response.status_code)
extra_disk_paths = []
if blocks > 1:
for (name, diskfile) in list(scratch_drives)[:(blocks - 1)]:
basevm.add_drive(name, diskfile.path, use_ramdisk=True)
extra_disk_paths.append(diskfile.path)
assert len(extra_disk_paths) > 0
if all_devices:
response = basevm.balloon.put(amount_mib=0,
deflate_on_oom=True,
stats_polling_interval_s=1)
assert basevm.api_session.is_status_no_content(response.status_code)
response = basevm.vsock.put(vsock_id="vsock0",
guest_cid=3,
uds_path="/v.sock")
assert basevm.api_session.is_status_no_content(response.status_code)
basevm.start()
ssh_connection = net_tools.SSHConnection(basevm.ssh_config)
# Create a snapshot builder from a microvm.
snapshot_builder = SnapshotBuilder(basevm)
full_snapshot = snapshot_builder.create([rw_disk.local_path()] +
extra_disk_paths,
ssh_key,
SnapshotType.FULL,
net_ifaces=ifaces)
basevm.kill()
values = []
for _ in range(iterations):
microvm, metrics_fifo = vm_builder.build_from_snapshot(
full_snapshot, resume=True, use_ramdisk=True)
# Attempt to connect to resumed microvm.
ssh_connection = net_tools.SSHConnection(microvm.ssh_config)
# Check if guest still runs commands.
exit_code, _, _ = ssh_connection.execute_command("dmesg")
assert exit_code == 0
value = 0
# Parse all metric data points in search of load_snapshot time.
metrics = microvm.get_all_metrics(metrics_fifo)
for data_point in metrics:
metrics = json.loads(data_point)
cur_value = metrics['latencies_us']['load_snapshot']
if cur_value > 0:
value = cur_value / USEC_IN_MSEC
break
values.append(value)
microvm.kill()
full_snapshot.cleanup()
result = dict()
result[RESTORE_LATENCY] = values
return result
def _test_balloon_snapshot(context):
logger = context.custom['logger']
vm_builder = context.custom['builder']
snapshot_type = context.custom['snapshot_type']
enable_diff_snapshots = snapshot_type == SnapshotType.DIFF
logger.info("Testing {} with microvm: \"{}\", kernel {}, disk {} ".format(
snapshot_type, context.microvm.name(), context.kernel.name(),
context.disk.name()))
# Create a rw copy artifact.
root_disk = context.disk.copy()
# Get ssh key from read-only artifact.
ssh_key = context.disk.ssh_key()
# Create a fresh microvm from aftifacts.
basevm = vm_builder.build(kernel=context.kernel,
disks=[root_disk],
ssh_key=ssh_key,
config=context.microvm,
enable_diff_snapshots=enable_diff_snapshots)
copy_util_to_rootfs(root_disk.local_path(), 'fillmem')
# Add a memory balloon with stats enabled.
response = basevm.balloon.put(amount_mib=0,
deflate_on_oom=True,
stats_polling_interval_s=1)
assert basevm.api_session.is_status_no_content(response.status_code)
basevm.start()
ssh_connection = net_tools.SSHConnection(basevm.ssh_config)
# Dirty 60MB of pages.
make_guest_dirty_memory(ssh_connection, amount=(60 * MB_TO_PAGES))
time.sleep(1)
# Get the firecracker pid, and open an ssh connection.
firecracker_pid = basevm.jailer_clone_pid
# Check memory usage.
first_reading = get_stable_rss_mem_by_pid(firecracker_pid)
# Now inflate the balloon with 20MB of pages.
response = basevm.balloon.patch(amount_mib=20)
assert basevm.api_session.is_status_no_content(response.status_code)
# Check memory usage again.
second_reading = get_stable_rss_mem_by_pid(firecracker_pid)
# There should be a reduction in RSS, but it's inconsistent.
# We only test that the reduction happens.
assert first_reading > second_reading
logger.info("Create {} #0.".format(snapshot_type))
# Create a snapshot builder from a microvm.
snapshot_builder = SnapshotBuilder(basevm)
# Create base snapshot.
snapshot = snapshot_builder.create([root_disk.local_path()], ssh_key,
snapshot_type)
basevm.kill()
logger.info("Load snapshot #{}, mem {}".format(1, snapshot.mem))
microvm, _ = vm_builder.build_from_snapshot(snapshot, True,
enable_diff_snapshots)
# Attempt to connect to resumed microvm.
ssh_connection = net_tools.SSHConnection(microvm.ssh_config)
# Get the firecracker from snapshot pid, and open an ssh connection.
firecracker_pid = microvm.jailer_clone_pid
# Check memory usage.
third_reading = get_stable_rss_mem_by_pid(firecracker_pid)
# Dirty 60MB of pages.
make_guest_dirty_memory(ssh_connection, amount=(60 * MB_TO_PAGES))
# Check memory usage.
fourth_reading = get_stable_rss_mem_by_pid(firecracker_pid)
assert fourth_reading > third_reading
# Inflate the balloon with another 20MB of pages.
response = microvm.balloon.patch(amount_mib=40)
assert microvm.api_session.is_status_no_content(response.status_code)
fifth_reading = get_stable_rss_mem_by_pid(firecracker_pid)
# There should be a reduction in RSS, but it's inconsistent.
# We only test that the reduction happens.
assert fourth_reading > fifth_reading
microvm.kill()
def _test_seq_snapshots(context):
logger = context.custom['logger']
seq_len = context.custom['seq_len']
vm_builder = context.custom['builder']
snapshot_type = context.custom['snapshot_type']
enable_diff_snapshots = snapshot_type == SnapshotType.DIFF
logger.info("Testing {} with microvm: \"{}\", kernel {}, disk {} ".format(
snapshot_type, context.microvm.name(), context.kernel.name(),
context.disk.name()))
# Create a rw copy artifact.
root_disk = context.disk.copy()
# Get ssh key from read-only artifact.
ssh_key = context.disk.ssh_key()
# Create a fresh microvm from aftifacts.
basevm = vm_builder.build(kernel=context.kernel,
disks=[root_disk],
ssh_key=ssh_key,
config=context.microvm,
enable_diff_snapshots=enable_diff_snapshots)
network_config = net_tools.UniqueIPv4Generator.instance()
_, host_ip, guest_ip = basevm.ssh_network_config(network_config,
'1',
tapname="tap0")
logger.debug("Host IP: {}, Guest IP: {}".format(host_ip, guest_ip))
# We will need netmask_len in build_from_snapshot() call later.
netmask_len = network_config.get_netmask_len()
basevm.start()
ssh_connection = net_tools.SSHConnection(basevm.ssh_config)
# Verify if guest can run commands.
exit_code, _, _ = ssh_connection.execute_command("sync")
assert exit_code == 0
logger.info("Create {} #0.".format(snapshot_type))
# Create a snapshot builder from a microvm.
snapshot_builder = SnapshotBuilder(basevm)
# Create base snapshot.
snapshot = snapshot_builder.create([root_disk.local_path()], ssh_key,
snapshot_type)
base_snapshot = snapshot
basevm.kill()
for i in range(seq_len):
logger.info("Load snapshot #{}, mem {}".format(i, snapshot.mem))
microvm, _ = vm_builder.build_from_snapshot(snapshot, host_ip,
guest_ip, netmask_len,
True,
enable_diff_snapshots)
# Attempt to connect to resumed microvm.
ssh_connection = net_tools.SSHConnection(microvm.ssh_config)
# Start a new instance of fio on each iteration.
_guest_run_fio_iteration(ssh_connection, i)
logger.info("Create snapshot #{}.".format(i + 1))
# Create a snapshot builder from the currently running microvm.
snapshot_builder = SnapshotBuilder(microvm)
snapshot = snapshot_builder.create([root_disk.local_path()], ssh_key,
snapshot_type)
# If we are testing incremental snapshots we must merge the base with
# current layer.
if snapshot_type == SnapshotType.DIFF:
logger.info("Base: {}, Layer: {}".format(base_snapshot.mem,
snapshot.mem))
snapshot.rebase_snapshot(base_snapshot)
# Update the base for next iteration.
base_snapshot = snapshot
microvm.kill()
def test_describe_snapshot_all_versions(bin_cloner_path):
"""
Test `--describe-snapshot` correctness for all snapshot versions.
@type: functional
"""
logger = logging.getLogger("describe_snapshot")
builder = MicrovmBuilder(bin_cloner_path)
artifacts = ArtifactCollection(_test_images_s3_bucket())
# Fetch all firecracker binaries.
# For each binary create a snapshot and verify the data version
# of the snapshot state file.
firecracker_artifacts = artifacts.firecrackers(
max_version=get_firecracker_version_from_toml())
for firecracker in firecracker_artifacts:
firecracker.download()
jailer = firecracker.jailer()
jailer.download()
target_version = firecracker.base_name()[1:]
# Skip for aarch64, since the snapshotting feature
# was introduced in v0.24.0.
if platform.machine() == "aarch64" and "v0.23" in target_version:
continue
logger.info("Creating snapshot with Firecracker: %s",
firecracker.local_path())
logger.info("Using Jailer: %s", jailer.local_path())
logger.info("Using target version: %s", target_version)
# v0.23 does not support creating diff snapshots.
diff_snapshots = "0.23" not in target_version
vm_instance = builder.build_vm_nano(fc_binary=firecracker.local_path(),
jailer_binary=jailer.local_path(),
diff_snapshots=diff_snapshots)
vm = vm_instance.vm
vm.start()
# Create a snapshot builder from a microvm.
snapshot_builder = SnapshotBuilder(vm)
disks = [vm_instance.disks[0].local_path()]
# Version 0.24 and greater have Diff support.
snap_type = SnapshotType.DIFF if diff_snapshots else SnapshotType.FULL
snapshot = snapshot_builder.create(disks,
vm_instance.ssh_key,
target_version=target_version,
snapshot_type=snap_type)
logger.debug("========== Firecracker create snapshot log ==========")
logger.debug(vm.log_data)
vm.kill()
# Fetch Firecracker binary for the latest version
fc_binary, _ = get_firecracker_binaries()
# Verify the output of `--describe-snapshot` command line parameter
cmd = [fc_binary] + ["--describe-snapshot", snapshot.vmstate]
code, stdout, stderr = run_cmd(cmd)
assert code == 0
assert stderr == ''
assert target_version in stdout
def test_negative_snapshot_permissions(bin_cloner_path):
"""
Test missing permission error scenarios.
@type: negative
"""
logger = logging.getLogger("snapshot_negative")
vm_builder = MicrovmBuilder(bin_cloner_path)
# Use a predefined vm instance.
vm_instance = vm_builder.build_vm_nano()
basevm = vm_instance.vm
root_disk = vm_instance.disks[0]
ssh_key = vm_instance.ssh_key
basevm.start()
logger.info("Create snapshot")
# Create a snapshot builder from a microvm.
snapshot_builder = SnapshotBuilder(basevm)
disks = [root_disk.local_path()]
# Remove write permissions.
os.chmod(basevm.jailer.chroot_path(), 0o444)
try:
_ = snapshot_builder.create(disks, ssh_key, SnapshotType.FULL)
except AssertionError as error:
# Check if proper error is returned.
assert "Permission denied" in str(error)
else:
assert False, "Negative test failed"
# Restore proper permissions.
os.chmod(basevm.jailer.chroot_path(), 0o744)
# Create base snapshot.
snapshot = snapshot_builder.create(disks, ssh_key, SnapshotType.FULL)
logger.info("Load snapshot, mem %s", snapshot.mem)
basevm.kill()
# Remove permissions for mem file.
os.chmod(snapshot.mem, 0o000)
try:
_, _ = vm_builder.build_from_snapshot(snapshot,
resume=True,
diff_snapshots=True)
except AssertionError as error:
# Check if proper error is returned.
assert "Cannot open the memory file: Permission denied" in str(error)
else:
assert False, "Negative test failed"
# Remove permissions for state file.
os.chmod(snapshot.vmstate, 0o000)
try:
_, _ = vm_builder.build_from_snapshot(snapshot,
resume=True,
diff_snapshots=True)
except AssertionError as error:
# Check if proper error is returned.
assert ("Cannot perform open on the snapshot backing file:"
" Permission denied" in str(error))
else:
assert False, "Negative test failed"
# Restore permissions for state file.
os.chmod(snapshot.vmstate, 0o744)
os.chmod(snapshot.mem, 0o744)
# Remove permissions for block file.
os.chmod(snapshot.disks[0], 0o000)
try:
_, _ = vm_builder.build_from_snapshot(snapshot,
resume=True,
diff_snapshots=True)
except AssertionError as error:
# Check if proper error is returned.
assert "Block(BackingFile(Os { code: 13, kind: PermissionDenied" in str(
error)
else:
assert False, "Negative test failed"
def _test_snapshot_create_latency(context):
logger = context.custom['logger']
vm_builder = context.custom['builder']
snapshot_type = context.custom['snapshot_type']
enable_diff_snapshots = snapshot_type == SnapshotType.DIFF
# Create a rw copy artifact.
rw_disk = context.disk.copy()
# Get ssh key from read-only artifact.
ssh_key = context.disk.ssh_key()
logger.info("Fetching firecracker/jailer versions from {}."
.format(DEFAULT_TEST_IMAGES_S3_BUCKET))
artifacts = ArtifactCollection(_test_images_s3_bucket())
firecracker_versions = artifacts.firecracker_versions(
older_than=get_firecracker_version_from_toml())
assert len(firecracker_versions) > 0
# Test snapshot creation for every supported target version.
for target_version in firecracker_versions:
logger.info("""Measuring snapshot create({}) latency for target
version: {} and microvm: \"{}\", kernel {}, disk {} """
.format(snapshot_type,
target_version,
context.microvm.name(),
context.kernel.name(),
context.disk.name()))
# Measure a burst of snapshot create calls.
for i in range(SAMPLE_COUNT):
# Create a fresh microVM from artifacts.
vm = vm_builder.build(kernel=context.kernel,
disks=[rw_disk],
ssh_key=ssh_key,
config=context.microvm,
enable_diff_snapshots=enable_diff_snapshots,
use_ramdisk=True)
# Configure metrics system.
metrics_fifo_path = os.path.join(vm.path, 'metrics_fifo')
metrics_fifo = log_tools.Fifo(metrics_fifo_path)
response = vm.metrics.put(
metrics_path=vm.create_jailed_resource(metrics_fifo.path)
)
assert vm.api_session.is_status_no_content(response.status_code)
vm.start()
# Check if the needed CPU cores are available. We have the API
# thread, VMM thread and then one thread for each configured vCPU.
assert CpuMap.len() >= 2 + vm.vcpus_count
# Pin uVM threads to physical cores.
current_cpu_id = 0
assert vm.pin_vmm(current_cpu_id), \
"Failed to pin firecracker thread."
current_cpu_id += 1
assert vm.pin_api(current_cpu_id), \
"Failed to pin fc_api thread."
for idx_vcpu in range(vm.vcpus_count):
current_cpu_id += 1
assert vm.pin_vcpu(idx_vcpu, current_cpu_id + idx_vcpu), \
f"Failed to pin fc_vcpu {idx_vcpu} thread."
# Create a snapshot builder from a microVM.
snapshot_builder = SnapshotBuilder(vm)
snapshot_builder.create(disks=[rw_disk],
ssh_key=ssh_key,
snapshot_type=snapshot_type,
target_version=target_version,
use_ramdisk=True)
metrics = vm.flush_metrics(metrics_fifo)
vm_name = context.microvm.name()
if snapshot_type == SnapshotType.FULL:
value = metrics['latencies_us']['full_create_snapshot']
baseline = CREATE_LATENCY_BASELINES[PLATFORM][vm_name]['FULL']
else:
value = metrics['latencies_us']['diff_create_snapshot']
baseline = CREATE_LATENCY_BASELINES[PLATFORM][vm_name]['DIFF']
value = value / USEC_IN_MSEC
assert baseline > value, "CreateSnapshot latency degraded."
logger.info("Latency {}/3: {} ms".format(i + 1, value))
vm.kill()
def _test_seq_snapshots(context):
logger = context.custom["logger"]
seq_len = context.custom["seq_len"]
vm_builder = context.custom["builder"]
snapshot_type = context.custom["snapshot_type"]
diff_snapshots = snapshot_type == SnapshotType.DIFF
logger.info('Testing {} with microvm: "{}", kernel {}, disk {} '.format(
snapshot_type,
context.microvm.name(),
context.kernel.name(),
context.disk.name(),
))
# Create a rw copy artifact.
root_disk = context.disk.copy()
# Get ssh key from read-only artifact.
ssh_key = context.disk.ssh_key()
# Create a fresh microvm from artifacts.
vm_instance = vm_builder.build(
kernel=context.kernel,
disks=[root_disk],
ssh_key=ssh_key,
config=context.microvm,
diff_snapshots=diff_snapshots,
)
basevm = vm_instance.vm
basevm.vsock.put(vsock_id="vsock0",
guest_cid=3,
uds_path="/{}".format(VSOCK_UDS_PATH))
basevm.start()
ssh_connection = net_tools.SSHConnection(basevm.ssh_config)
# Verify if guest can run commands.
exit_code, _, _ = ssh_connection.execute_command("sync")
assert exit_code == 0
test_fc_session_root_path = context.custom["test_fc_session_root_path"]
vsock_helper = context.custom["bin_vsock_path"]
vm_blob_path = "/tmp/vsock/test.blob"
# Generate a random data file for vsock.
blob_path, blob_hash = make_blob(test_fc_session_root_path)
# Copy the data file and a vsock helper to the guest.
_copy_vsock_data_to_guest(ssh_connection, blob_path, vm_blob_path,
vsock_helper)
logger.info("Create {} #0.".format(snapshot_type))
# Create a snapshot builder from a microvm.
snapshot_builder = SnapshotBuilder(basevm)
# Create base snapshot.
snapshot = snapshot_builder.create([root_disk.local_path()], ssh_key,
snapshot_type)
base_snapshot = snapshot
basevm.kill()
for i in range(seq_len):
logger.info("Load snapshot #{}, mem {}".format(i, snapshot.mem))
microvm, _ = vm_builder.build_from_snapshot(
snapshot, resume=True, diff_snapshots=diff_snapshots)
# Attempt to connect to resumed microvm.
ssh_connection = net_tools.SSHConnection(microvm.ssh_config)
# Test vsock guest-initiated connections.
path = os.path.join(
microvm.path, make_host_port_path(VSOCK_UDS_PATH,
ECHO_SERVER_PORT))
check_guest_connections(microvm, path, vm_blob_path, blob_hash)
# Test vsock host-initiated connections.
path = os.path.join(microvm.jailer.chroot_path(), VSOCK_UDS_PATH)
check_host_connections(microvm, path, blob_path, blob_hash)
# Start a new instance of fio on each iteration.
_guest_run_fio_iteration(ssh_connection, i)
logger.info("Create snapshot #{}.".format(i + 1))
# Create a snapshot builder from the currently running microvm.
snapshot_builder = SnapshotBuilder(microvm)
snapshot = snapshot_builder.create([root_disk.local_path()], ssh_key,
snapshot_type)
# If we are testing incremental snapshots we must merge the base with
# current layer.
if snapshot_type == SnapshotType.DIFF:
logger.info("Base: {}, Layer: {}".format(base_snapshot.mem,
snapshot.mem))
snapshot.rebase_snapshot(base_snapshot)
# Update the base for next iteration.
base_snapshot = snapshot
microvm.kill()
def test_older_snapshot_resume_latency(bin_cloner_path):
"""Test scenario: Older snapshot load performance measurement."""
logger = logging.getLogger("old_snapshot_load")
artifacts = ArtifactCollection(_test_images_s3_bucket())
# Fetch all firecracker binaries.
# With each binary create a snapshot and try to restore in current
# version.
firecracker_artifacts = artifacts.firecrackers()
for firecracker in firecracker_artifacts:
firecracker.download()
jailer = firecracker.jailer()
jailer.download()
fc_version = firecracker.base_name()[1:]
logger.info("Firecracker version: %s", fc_version)
logger.info("Source Firecracker: %s", firecracker.local_path())
logger.info("Source Jailer: %s", jailer.local_path())
for i in range(SAMPLE_COUNT):
# Create a fresh microvm with the binary artifacts.
vm_instance = VMMicro.spawn(bin_cloner_path, True,
firecracker.local_path(),
jailer.local_path())
# Attempt to connect to the fresh microvm.
ssh_connection = net_tools.SSHConnection(vm_instance.vm.ssh_config)
exit_code, _, _ = ssh_connection.execute_command("sync")
assert exit_code == 0
# Create a snapshot builder from a microvm.
snapshot_builder = SnapshotBuilder(vm_instance.vm)
# The snapshot builder expects disks as paths, not artifacts.
disks = []
for disk in vm_instance.disks:
disks.append(disk.local_path())
snapshot_builder = SnapshotBuilder(vm_instance.vm)
snapshot = snapshot_builder.create(disks, vm_instance.ssh_key,
SnapshotType.FULL)
vm_instance.vm.kill()
builder = MicrovmBuilder(bin_cloner_path)
microvm, metrics_fifo = builder.build_from_snapshot(
snapshot, True, False)
# Attempt to connect to resumed microvm.
ssh_connection = net_tools.SSHConnection(microvm.ssh_config)
# Check if guest still runs commands.
exit_code, _, _ = ssh_connection.execute_command("dmesg")
assert exit_code == 0
value = 0
# Parse all metric data points in search of load_snapshot time.
metrics = microvm.get_all_metrics(metrics_fifo)
for data_point in metrics:
metrics = json.loads(data_point)
cur_value = metrics['latencies_us']['load_snapshot']
if cur_value > 0:
value = cur_value / USEC_IN_MSEC
break
baseline = LOAD_LATENCY_BASELINES[fc_version]
logger.info("Latency %s/%s: %s ms", i + 1, SAMPLE_COUNT, value)
assert baseline > value, "LoadSnapshot latency degraded."
microvm.kill()
def _test_seq_snapshots(context):
logger = context.custom['logger']
seq_len = context.custom['seq_len']
vm_builder = context.custom['builder']
snapshot_type = context.custom['snapshot_type']
enable_diff_snapshots = snapshot_type == SnapshotType.DIFF
logger.info("Testing {} with microvm: \"{}\", kernel {}, disk {} ".format(
snapshot_type, context.microvm.name(), context.kernel.name(),
context.disk.name()))
# Create a rw copy artifact.
rw_disk = context.disk.copy()
# Get ssh key from read-only artifact.
ssh_key = context.disk.ssh_key()
# Create a fresh microvm from aftifacts.
basevm = vm_builder.build(kernel=context.kernel,
disks=[rw_disk],
ssh_key=ssh_key,
config=context.microvm,
enable_diff_snapshots=enable_diff_snapshots)
network_config = net_tools.UniqueIPv4Generator.instance()
_, host_ip, guest_ip = basevm.ssh_network_config(network_config,
'1',
tapname="tap0")
logger.debug("Host IP: {}, Guest IP: {}".format(host_ip, guest_ip))
# We will need netmask_len in build_from_snapshot() call later.
netmask_len = network_config.get_netmask_len()
basevm.start()
ssh_connection = net_tools.SSHConnection(basevm.ssh_config)
# Verify if guest can run commands.
exit_code, _, _ = ssh_connection.execute_command("sync")
assert exit_code == 0
logger.info("Create {} #0.".format(snapshot_type))
# Create a snapshot builder from a microvm.
snapshot_builder = SnapshotBuilder(basevm)
# Create base snapshot.
snapshot = snapshot_builder.create([rw_disk], ssh_key, snapshot_type)
base_snapshot = snapshot
basevm.kill()
for i in range(seq_len):
logger.info("Load snapshot #{}, mem {}".format(i, snapshot.mem))
microvm, _ = vm_builder.build_from_snapshot(snapshot, host_ip,
guest_ip, netmask_len,
True,
enable_diff_snapshots)
# Attempt to connect to resumed microvm.
ssh_connection = net_tools.SSHConnection(microvm.ssh_config)
# Start a new instance of fio on each iteration.
fio = """fio --filename=/dev/vda --direct=1 --rw=randread --bs=4k
--ioengine=libaio --iodepth=16 --runtime=10 --numjobs=4 --time_based
--group_reporting --name=iops-test-job --eta-newline=1 --readonly"""
ssh_cmd = "screen -L -Logfile /tmp/fio{} -dmS test{} {}"
ssh_cmd = ssh_cmd.format(i, i, fio)
exit_code, _, _ = ssh_connection.execute_command(ssh_cmd)
logger.info("Create snapshot #{}.".format(i + 1))
# Create a snapshot builder from the currently running microvm.
snapshot_builder = SnapshotBuilder(microvm)
snapshot = snapshot_builder.create([rw_disk], ssh_key, snapshot_type)
# If we are testing incremental snapshots we must merge the base with
# current layer.
if snapshot_type == SnapshotType.DIFF:
logger.info("Base: {}, Layer: {}".format(base_snapshot.mem,
snapshot.mem))
snapshot.rebase_snapshot(base_snapshot)
# Update the base for next iteration.
base_snapshot = snapshot
microvm.kill()
def _test_seq_snapshots(context):
logger = context.custom['logger']
seq_len = context.custom['seq_len']
vm_builder = context.custom['builder']
snapshot_type = context.custom['snapshot_type']
bin_vsock_path = context.custom['bin_vsock_path']
test_session_root_path = context.custom['test_session_root_path']
enable_diff_snapshots = snapshot_type == SnapshotType.DIFF
logger.info("Testing {} with microvm: \"{}\", kernel {}, disk {} ".format(
snapshot_type, context.microvm.name(), context.kernel.name(),
context.disk.name()))
# Create a rw copy artifact.
root_disk = context.disk.copy()
# Get ssh key from read-only artifact.
ssh_key = context.disk.ssh_key()
# Create a fresh microvm from aftifacts.
basevm = vm_builder.build(kernel=context.kernel,
disks=[root_disk],
ssh_key=ssh_key,
config=context.microvm,
enable_diff_snapshots=enable_diff_snapshots)
# Configure vsock device.
basevm.vsock.put(vsock_id="vsock0",
guest_cid=3,
uds_path="/{}".format(VSOCK_UDS_PATH))
# Generate a random data file for vsock.
blob_path, blob_hash = make_blob(test_session_root_path)
basevm.start()
ssh_connection = net_tools.SSHConnection(basevm.ssh_config)
# Verify if guest can run commands.
exit_code, _, _ = ssh_connection.execute_command("sync")
assert exit_code == 0
# Copy the data file and a vsock helper to the guest.
cmd = "mkdir -p /tmp/vsock && mount -t tmpfs tmpfs /tmp/vsock"
ecode, _, _ = ssh_connection.execute_command(cmd)
assert ecode == 0, "Failed to set up tmpfs drive on the guest."
vsock_helper = bin_vsock_path
ssh_connection.scp_file(vsock_helper, '/bin/vsock_helper')
vm_blob_path = "/tmp/vsock/test.blob"
ssh_connection.scp_file(blob_path, vm_blob_path)
logger.info("Create {} #0.".format(snapshot_type))
# Create a snapshot builder from a microvm.
snapshot_builder = SnapshotBuilder(basevm)
# Create base snapshot.
snapshot = snapshot_builder.create([root_disk.local_path()], ssh_key,
snapshot_type)
base_snapshot = snapshot
basevm.kill()
for i in range(seq_len):
logger.info("Load snapshot #{}, mem {}".format(i, snapshot.mem))
microvm, _ = vm_builder.build_from_snapshot(snapshot, True,
enable_diff_snapshots)
# Attempt to connect to resumed microvm.
ssh_connection = net_tools.SSHConnection(microvm.ssh_config)
# Test vsock guest-initiated connections.
path = os.path.join(microvm.path,
"{}_{}".format(VSOCK_UDS_PATH, ECHO_SERVER_PORT))
check_guest_connections(microvm, path, vm_blob_path, blob_hash)
# Test vsock host-initiated connections.
path = os.path.join(microvm.jailer.chroot_path(), VSOCK_UDS_PATH)
check_host_connections(microvm, path, blob_path, blob_hash)
# Start a new instance of fio on each iteration.
_guest_run_fio_iteration(ssh_connection, i)
logger.info("Create snapshot #{}.".format(i + 1))
# Create a snapshot builder from the currently running microvm.
snapshot_builder = SnapshotBuilder(microvm)
snapshot = snapshot_builder.create([root_disk.local_path()], ssh_key,
snapshot_type)
# If we are testing incremental snapshots we must merge the base with
# current layer.
if snapshot_type == SnapshotType.DIFF:
logger.info("Base: {}, Layer: {}".format(base_snapshot.mem,
snapshot.mem))
snapshot.rebase_snapshot(base_snapshot)
# Update the base for next iteration.
base_snapshot = snapshot
microvm.kill()
def test_mmds_snapshot(bin_cloner_path):
"""
Exercise tokens' behavior with snapshots.
Ensures that valid tokens created on a base microVM
are not accepted on the clone VM.
@type: functional
"""
vm_builder = MicrovmBuilder(bin_cloner_path)
vm_instance = vm_builder.build_vm_nano(diff_snapshots=True)
test_microvm = vm_instance.vm
root_disk = vm_instance.disks[0]
ssh_key = vm_instance.ssh_key
data_store = {
'latest': {
'meta-data': {
'ami-id': 'ami-12345678'
}
}
}
_populate_data_store(test_microvm, data_store)
test_microvm.start()
snapshot_builder = SnapshotBuilder(test_microvm)
disks = [root_disk.local_path()]
ssh_connection = net_tools.SSHConnection(test_microvm.ssh_config)
cmd = 'ip route add 169.254.169.254 dev eth0'
_, stdout, stderr = ssh_connection.execute_command(cmd)
_assert_out(stdout, stderr, '')
# Generate token.
token = _generate_mmds_session_token(
ssh_connection,
ipv4_address="169.254.169.254",
token_ttl=60
)
pre = 'curl -m 2 -s'
pre += ' -X GET'
pre += ' -H "X-metadata-token: {}"'.format(token)
pre += ' http://169.254.169.254/'
cmd = pre + 'latest/meta-data/'
_, stdout, stderr = ssh_connection.execute_command(cmd)
_assert_out(stdout, stderr, "ami-id")
# Setting MMDS version to V2 when V2 is already in use should
# have no effect on tokens generated so far.
_set_mmds_version(test_microvm, version='V2')
_, stdout, stderr = ssh_connection.execute_command(cmd)
_assert_out(stdout, stderr, "ami-id")
# Create diff snapshot.
snapshot = snapshot_builder.create(disks,
ssh_key,
SnapshotType.DIFF)
# Resume microVM and ensure session token is still valid.
response = test_microvm.vm.patch(state='Resumed')
assert test_microvm.api_session.is_status_no_content(response.status_code)
_, stdout, stderr = ssh_connection.execute_command(
pre + 'latest/meta-data/'
)
_assert_out(stdout, stderr, "ami-id")
# Kill base microVM.
test_microvm.kill()
# Load microVM clone from snapshot.
test_microvm, _ = vm_builder.build_from_snapshot(snapshot,
resume=True,
diff_snapshots=True)
_populate_data_store(test_microvm, data_store)
ssh_connection = net_tools.SSHConnection(test_microvm.ssh_config)
# Ensure that token created on the baseVM is not valid inside the clone.
cmd = 'curl -m 2 -s'
cmd += ' -X GET'
cmd += ' -H "X-metadata-token: {}"'.format(token)
cmd += ' http://169.254.169.254/latest/meta-data/'
_, stdout, stderr = ssh_connection.execute_command(cmd)
_assert_out(stdout, stderr, "MMDS token not valid.")
# Generate new session token.
token = _generate_mmds_session_token(
ssh_connection,
ipv4_address="169.254.169.254",
token_ttl=60
)
# Ensure the newly created token is valid.
cmd = 'curl -m 2 -s'
cmd += ' -X GET'
cmd += ' -H "X-metadata-token: {}"'.format(token)
cmd += ' http://169.254.169.254/latest/meta-data/'
_, stdout, stderr = ssh_connection.execute_command(cmd)
_assert_out(stdout, stderr, "ami-id")