def test_older_snapshot_resume_latency(bin_cloner_path, results_file_dumper):
"""
Test scenario: Older snapshot load performance measurement.
@type: performance
"""
logger = logging.getLogger("old_snapshot_load")
artifacts = ArtifactCollection(_test_images_s3_bucket())
# Fetch all firecracker binaries.
# With each binary create a snapshot and try to restore in current
# version.
firecracker_artifacts = ArtifactSet(artifacts.firecrackers(
max_version=get_firecracker_version_from_toml()))
assert len(firecracker_artifacts) > 0
microvm_artifacts = ArtifactSet(artifacts.microvms(keyword="2vcpu_512mb"))
test_context = TestContext()
test_context.custom = {
'builder': MicrovmBuilder(bin_cloner_path),
'logger': logger,
'snapshot_type': SnapshotType.FULL,
'name': 'old_snapshot_resume_latency',
'results_file_dumper': results_file_dumper
}
# Create the test matrix.
test_matrix = TestMatrix(context=test_context,
artifact_sets=[
firecracker_artifacts,
microvm_artifacts,
])
test_matrix.run_test(_test_older_snapshot_resume_latency)
def test_mmds_older_snapshot(bin_cloner_path):
"""
Test MMDS behavior restoring older snapshots in the current version.
Ensures that the MMDS version is persisted or initialised with the default
if the FC version does not support this feature.
@type: functional
"""
vm_builder = MicrovmBuilder(bin_cloner_path)
# Validate restoring a past snapshot in the current version.
artifacts = ArtifactCollection(_test_images_s3_bucket())
# Fetch all firecracker binaries.
firecracker_artifacts = artifacts.firecrackers(
max_version=get_firecracker_version_from_toml())
for firecracker in firecracker_artifacts:
firecracker.download()
jailer = firecracker.jailer()
jailer.download()
net_iface = NetIfaceConfig()
vm_instance = vm_builder.build_vm_nano(
net_ifaces=[net_iface],
fc_binary=firecracker.local_path(),
jailer_binary=jailer.local_path()
)
fc_version = firecracker.base_name()[1:]
# If the version is smaller or equal to 1.0.0, we expect that
# MMDS will be initialised with V1 by default.
# Otherwise, we may configure V2.
if compare_versions(fc_version, "1.0.0") <= 0:
mmds_version = "V1"
else:
mmds_version = "V2"
# Check if we need to configure MMDS the old way, by
# setting `allow_mmds_requests`.
# If we do (for v0.25), reissue the network PUT api call.
if compare_versions(fc_version, "1.0.0") < 0:
basevm = vm_instance.vm
guest_mac = net_tools.mac_from_ip(net_iface.guest_ip)
response = basevm.network.put(
iface_id=net_iface.dev_name,
host_dev_name=net_iface.tap_name,
guest_mac=guest_mac,
allow_mmds_requests=True
)
assert basevm.api_session.is_status_no_content(
response.status_code)
_validate_mmds_snapshot(
vm_instance,
vm_builder,
mmds_version,
target_fc_version=fc_version
)
def test_restore_old_version(bin_cloner_path):
"""
Restore current snapshot with previous versions of Firecracker.
@type: functional
"""
# Microvm: 2vCPU 256MB RAM, balloon, 4 disks and 4 net devices.
logger = logging.getLogger("old_snapshot_version_many_devices")
builder = MicrovmBuilder(bin_cloner_path)
artifacts = ArtifactCollection(_test_images_s3_bucket())
# Fetch all firecracker binaries.
# Create a snapshot with current build and restore with each FC binary
# artifact.
firecracker_artifacts = artifacts.firecrackers(
# current snapshot (i.e a machine snapshotted with current build)
# is incompatible with any past release due to notification suppression.
min_version="1.2.0",
max_version=get_firecracker_version_from_toml(),
)
for firecracker in firecracker_artifacts:
firecracker.download()
jailer = firecracker.jailer()
jailer.download()
logger.info("Creating snapshot with local build")
# Old version from artifact.
target_version = firecracker.base_name()[1:]
# Create a snapshot with current FC version targeting the old version.
snapshot = create_snapshot_helper(
builder,
logger,
target_version=target_version,
drives=scratch_drives,
ifaces=net_ifaces,
balloon=True,
diff_snapshots=True,
)
logger.info("Restoring snapshot with Firecracker: %s",
firecracker.local_path())
logger.info("Using Jailer: %s", jailer.local_path())
# Resume microvm using FC/Jailer binary artifacts.
vm, _ = builder.build_from_snapshot(
snapshot,
resume=True,
diff_snapshots=False,
fc_binary=firecracker.local_path(),
jailer_binary=jailer.local_path(),
)
validate_all_devices(logger, vm, net_ifaces, scratch_drives, True)
logger.debug("========== Firecracker restore snapshot log ==========")
logger.debug(vm.log_data)
def test_restore_old_snapshot(bin_cloner_path):
"""
Restore from snapshots obtained with previous versions of Firecracker.
@type: functional
"""
# Microvm: 2vCPU 256MB RAM, balloon, 4 disks and 4 net devices.
logger = logging.getLogger("old_snapshot_many_devices")
builder = MicrovmBuilder(bin_cloner_path)
artifacts = ArtifactCollection(_test_images_s3_bucket())
# Fetch all firecracker binaries.
# With each binary create a snapshot and try to restore in current
# version.
firecracker_artifacts = artifacts.firecrackers(
max_version=get_firecracker_version_from_toml())
for firecracker in firecracker_artifacts:
firecracker.download()
jailer = firecracker.jailer()
jailer.download()
logger.info("Creating snapshot with Firecracker: %s",
firecracker.local_path())
logger.info("Using Jailer: %s", jailer.local_path())
target_version = firecracker.base_name()[1:]
# v0.23 does not support creating diff snapshots.
# v0.23 does not support balloon.
diff_snapshots = "0.23" not in target_version
# Create a snapshot.
snapshot = create_snapshot_helper(
builder,
logger,
drives=scratch_drives,
ifaces=net_ifaces,
fc_binary=firecracker.local_path(),
jailer_binary=jailer.local_path(),
diff_snapshots=diff_snapshots,
balloon=diff_snapshots,
)
# Resume microvm using current build of FC/Jailer.
microvm, _ = builder.build_from_snapshot(snapshot,
resume=True,
diff_snapshots=False)
validate_all_devices(logger, microvm, net_ifaces, scratch_drives,
diff_snapshots)
logger.debug("========== Firecracker restore snapshot log ==========")
logger.debug(microvm.log_data)
def test_mmds_snapshot(bin_cloner_path, version):
"""
Test MMDS behavior by restoring a snapshot on current and past FC versions.
Ensures that the version is persisted or initialised with the default if
the firecracker version does not support it.
@type: functional
"""
vm_builder = MicrovmBuilder(bin_cloner_path)
vm_instance = vm_builder.build_vm_nano(
net_ifaces=[NetIfaceConfig()]
)
# Validate current version.
_validate_mmds_snapshot(
vm_instance, vm_builder, version)
# Validate restoring in past versions.
artifacts = ArtifactCollection(_test_images_s3_bucket())
# Fetch all firecracker binaries.
# Create a snapshot with current build and restore with each FC binary
# artifact.
firecracker_artifacts = artifacts.firecrackers(
# v1.1.0 breaks snapshot compatibility with older versions.
min_version="1.1.0",
max_version=get_firecracker_version_from_toml())
for firecracker in firecracker_artifacts:
vm_instance = vm_builder.build_vm_nano(
net_ifaces=[NetIfaceConfig()]
)
firecracker.download()
jailer = firecracker.jailer()
jailer.download()
target_version = firecracker.base_name()[1:]
# If the version is smaller or equal to 1.0.0, we expect that
# MMDS will be initialised with V1 by default.
if compare_versions(target_version, "1.0.0") <= 0:
mmds_version = "V1"
else:
mmds_version = version
_validate_mmds_snapshot(
vm_instance,
vm_builder,
mmds_version,
target_fc_version=target_version,
fc_path=firecracker.local_path(),
jailer_path=jailer.local_path()
)
def test_restore_old_version_all_devices(bin_cloner_path):
"""Test scenario: restore snapshot in previous versions of Firecracker."""
# Microvm: 2vCPU 256MB RAM, baloon, 4 disks and 4 netdevices.
logger = logging.getLogger("snapshot_many_devices")
artifacts = ArtifactCollection(_test_images_s3_bucket())
# Fetch all firecracker binaries.
# Create a snapshot with current build and restore with each FC binary
# artifact.
firecracker_artifacts = artifacts.firecrackers(
older_than=get_firecracker_version_from_toml())
for firecracker in firecracker_artifacts:
firecracker.download()
jailer = firecracker.jailer()
jailer.download()
logger.info("Creating snapshot with local build")
# Old version from artifact.
target_version = firecracker.base_name()[1:]
# v0.23 does not have a ballon device.
balloon = "0.23" not in target_version
# Create a snapshot with current FC version targeting the old version.
snapshot = create_snapshot_helper(bin_cloner_path,
logger,
target_version=target_version,
drives=scratch_drives,
ifaces=net_ifaces,
balloon=balloon,
diff_snapshots=True)
logger.info("Restoring snapshot with Firecracker: %s",
firecracker.local_path())
logger.info("Using Jailer: %s", jailer.local_path())
# Resume microvm using FC/Jailer binary artifacts.
builder = MicrovmBuilder(bin_cloner_path, firecracker.local_path(),
jailer.local_path())
microvm, _ = builder.build_from_snapshot(snapshot,
resume=True,
enable_diff_snapshots=False)
validate_all_devices(logger, microvm, net_ifaces, scratch_drives,
balloon)
logger.debug("========== Firecracker restore snapshot log ==========")
logger.debug(microvm.log_data)
def _test_snapshot_create_latency(context):
logger = context.custom['logger']
vm_builder = context.custom['builder']
snapshot_type = context.custom['snapshot_type']
enable_diff_snapshots = snapshot_type == SnapshotType.DIFF
# Create a rw copy artifact.
rw_disk = context.disk.copy()
# Get ssh key from read-only artifact.
ssh_key = context.disk.ssh_key()
logger.info("Fetching firecracker/jailer versions from {}."
.format(DEFAULT_TEST_IMAGES_S3_BUCKET))
artifacts = ArtifactCollection(_test_images_s3_bucket())
firecracker_versions = artifacts.firecracker_versions(
older_than=get_firecracker_version_from_toml())
assert len(firecracker_versions) > 0
# Test snapshot creation for every supported target version.
for target_version in firecracker_versions:
logger.info("""Measuring snapshot create({}) latency for target
version: {} and microvm: \"{}\", kernel {}, disk {} """
.format(snapshot_type,
target_version,
context.microvm.name(),
context.kernel.name(),
context.disk.name()))
# Measure a burst of snapshot create calls.
for i in range(SAMPLE_COUNT):
# Create a fresh microVM from artifacts.
vm = vm_builder.build(kernel=context.kernel,
disks=[rw_disk],
ssh_key=ssh_key,
config=context.microvm,
enable_diff_snapshots=enable_diff_snapshots,
use_ramdisk=True)
# Configure metrics system.
metrics_fifo_path = os.path.join(vm.path, 'metrics_fifo')
metrics_fifo = log_tools.Fifo(metrics_fifo_path)
response = vm.metrics.put(
metrics_path=vm.create_jailed_resource(metrics_fifo.path)
)
assert vm.api_session.is_status_no_content(response.status_code)
vm.start()
# Check if the needed CPU cores are available. We have the API
# thread, VMM thread and then one thread for each configured vCPU.
assert CpuMap.len() >= 2 + vm.vcpus_count
# Pin uVM threads to physical cores.
current_cpu_id = 0
assert vm.pin_vmm(current_cpu_id), \
"Failed to pin firecracker thread."
current_cpu_id += 1
assert vm.pin_api(current_cpu_id), \
"Failed to pin fc_api thread."
for idx_vcpu in range(vm.vcpus_count):
current_cpu_id += 1
assert vm.pin_vcpu(idx_vcpu, current_cpu_id + idx_vcpu), \
f"Failed to pin fc_vcpu {idx_vcpu} thread."
# Create a snapshot builder from a microVM.
snapshot_builder = SnapshotBuilder(vm)
snapshot_builder.create(disks=[rw_disk],
ssh_key=ssh_key,
snapshot_type=snapshot_type,
target_version=target_version,
use_ramdisk=True)
metrics = vm.flush_metrics(metrics_fifo)
vm_name = context.microvm.name()
if snapshot_type == SnapshotType.FULL:
value = metrics['latencies_us']['full_create_snapshot']
baseline = CREATE_LATENCY_BASELINES[PLATFORM][vm_name]['FULL']
else:
value = metrics['latencies_us']['diff_create_snapshot']
baseline = CREATE_LATENCY_BASELINES[PLATFORM][vm_name]['DIFF']
value = value / USEC_IN_MSEC
assert baseline > value, "CreateSnapshot latency degraded."
logger.info("Latency {}/3: {} ms".format(i + 1, value))
vm.kill()
def test_older_snapshot_resume_latency(bin_cloner_path):
"""Test scenario: Older snapshot load performance measurement."""
logger = logging.getLogger("old_snapshot_load")
artifacts = ArtifactCollection(_test_images_s3_bucket())
# Fetch all firecracker binaries.
# With each binary create a snapshot and try to restore in current
# version.
firecracker_artifacts = artifacts.firecrackers(
older_than=get_firecracker_version_from_toml())
assert len(firecracker_artifacts) > 0
for firecracker in firecracker_artifacts:
firecracker.download()
jailer = firecracker.jailer()
jailer.download()
fc_version = firecracker.base_name()[1:]
logger.info("Firecracker version: %s", fc_version)
logger.info("Source Firecracker: %s", firecracker.local_path())
logger.info("Source Jailer: %s", jailer.local_path())
for i in range(SAMPLE_COUNT):
# Create a fresh microvm with the binary artifacts.
vm_instance = VMMicro.spawn(bin_cloner_path, True,
firecracker.local_path(),
jailer.local_path())
# Attempt to connect to the fresh microvm.
ssh_connection = net_tools.SSHConnection(vm_instance.vm.ssh_config)
exit_code, _, _ = ssh_connection.execute_command("sync")
assert exit_code == 0
# The snapshot builder expects disks as paths, not artifacts.
disks = []
for disk in vm_instance.disks:
disks.append(disk.local_path())
# Create a snapshot builder from a microvm.
snapshot_builder = SnapshotBuilder(vm_instance.vm)
snapshot = snapshot_builder.create(disks,
vm_instance.ssh_key,
SnapshotType.FULL)
vm_instance.vm.kill()
builder = MicrovmBuilder(bin_cloner_path)
microvm, metrics_fifo = builder.build_from_snapshot(snapshot,
True,
False)
# Attempt to connect to resumed microvm.
ssh_connection = net_tools.SSHConnection(microvm.ssh_config)
# Check if guest still runs commands.
exit_code, _, _ = ssh_connection.execute_command("dmesg")
assert exit_code == 0
value = 0
# Parse all metric data points in search of load_snapshot time.
metrics = microvm.get_all_metrics(metrics_fifo)
for data_point in metrics:
metrics = json.loads(data_point)
cur_value = metrics['latencies_us']['load_snapshot']
if cur_value > 0:
value = cur_value / USEC_IN_MSEC
break
baseline = LOAD_LATENCY_BASELINES[PLATFORM]['2vcpu_512mb.json']
logger.info("Latency %s/%s: %s ms", i + 1, SAMPLE_COUNT, value)
assert baseline > value, "LoadSnapshot latency degraded."
microvm.kill()
def test_describe_snapshot_all_versions(bin_cloner_path):
"""
Test `--describe-snapshot` correctness for all snapshot versions.
@type: functional
"""
logger = logging.getLogger("describe_snapshot")
builder = MicrovmBuilder(bin_cloner_path)
artifacts = ArtifactCollection(_test_images_s3_bucket())
# Fetch all firecracker binaries.
# For each binary create a snapshot and verify the data version
# of the snapshot state file.
firecracker_artifacts = artifacts.firecrackers(
max_version=get_firecracker_version_from_toml())
for firecracker in firecracker_artifacts:
firecracker.download()
jailer = firecracker.jailer()
jailer.download()
target_version = firecracker.base_name()[1:]
# Skip for aarch64, since the snapshotting feature
# was introduced in v0.24.0.
if platform.machine() == "aarch64" and "v0.23" in target_version:
continue
logger.info("Creating snapshot with Firecracker: %s",
firecracker.local_path())
logger.info("Using Jailer: %s", jailer.local_path())
logger.info("Using target version: %s", target_version)
# v0.23 does not support creating diff snapshots.
diff_snapshots = "0.23" not in target_version
vm_instance = builder.build_vm_nano(fc_binary=firecracker.local_path(),
jailer_binary=jailer.local_path(),
diff_snapshots=diff_snapshots)
vm = vm_instance.vm
vm.start()
# Create a snapshot builder from a microvm.
snapshot_builder = SnapshotBuilder(vm)
disks = [vm_instance.disks[0].local_path()]
# Version 0.24 and greater have Diff support.
snap_type = SnapshotType.DIFF if diff_snapshots else SnapshotType.FULL
snapshot = snapshot_builder.create(disks,
vm_instance.ssh_key,
target_version=target_version,
snapshot_type=snap_type)
logger.debug("========== Firecracker create snapshot log ==========")
logger.debug(vm.log_data)
vm.kill()
# Fetch Firecracker binary for the latest version
fc_binary, _ = get_firecracker_binaries()
# Verify the output of `--describe-snapshot` command line parameter
cmd = [fc_binary] + ["--describe-snapshot", snapshot.vmstate]
code, stdout, stderr = run_cmd(cmd)
assert code == 0
assert stderr == ''
assert target_version in stdout
def _test_snapshot_create_latency(context):
logger = context.custom['logger']
vm_builder = context.custom['builder']
snapshot_type = context.custom['snapshot_type']
file_dumper = context.custom['results_file_dumper']
diff_snapshots = snapshot_type == SnapshotType.DIFF
# Create a rw copy artifact.
rw_disk = context.disk.copy()
# Get ssh key from read-only artifact.
ssh_key = context.disk.ssh_key()
logger.info("Fetching firecracker/jailer versions from {}.".format(
DEFAULT_TEST_IMAGES_S3_BUCKET))
artifacts = ArtifactCollection(_test_images_s3_bucket())
firecracker_versions = artifacts.firecracker_versions(
# v1.0.0 breaks snapshot compatibility with older versions.
min_version="1.0.0",
max_version=get_firecracker_version_from_toml())
assert len(firecracker_versions) > 0
# Test snapshot creation for every supported target version.
for target_version in firecracker_versions:
logger.info("""Measuring snapshot create({}) latency for target
version: {} and microvm: \"{}\", kernel {}, disk {} """.format(
snapshot_type, target_version, context.microvm.name(),
context.kernel.name(), context.disk.name()))
# Create a fresh microVM from artifacts.
vm_instance = vm_builder.build(kernel=context.kernel,
disks=[rw_disk],
ssh_key=ssh_key,
config=context.microvm,
diff_snapshots=diff_snapshots,
use_ramdisk=True)
vm = vm_instance.vm
# Configure metrics system.
metrics_fifo_path = os.path.join(vm.path, 'metrics_fifo')
metrics_fifo = log_tools.Fifo(metrics_fifo_path)
response = vm.metrics.put(
metrics_path=vm.create_jailed_resource(metrics_fifo.path))
assert vm.api_session.is_status_no_content(response.status_code)
vm.start()
# Check if the needed CPU cores are available. We have the API
# thread, VMM thread and then one thread for each configured vCPU.
assert CpuMap.len() >= 2 + vm.vcpus_count
# Pin uVM threads to physical cores.
current_cpu_id = 0
assert vm.pin_vmm(current_cpu_id), \
"Failed to pin firecracker thread."
current_cpu_id += 1
assert vm.pin_api(current_cpu_id), \
"Failed to pin fc_api thread."
for idx_vcpu in range(vm.vcpus_count):
current_cpu_id += 1
assert vm.pin_vcpu(idx_vcpu, current_cpu_id + idx_vcpu), \
f"Failed to pin fc_vcpu {idx_vcpu} thread."
st_core = core.Core(
name="snapshot_create_full_latency" if snapshot_type
== SnapshotType.FULL else "snapshot_create_diff_latency",
iterations=SAMPLE_COUNT)
prod = producer.LambdaProducer(func=snapshot_create_producer,
func_kwargs={
"logger": logger,
"vm": vm,
"disks": [rw_disk],
"ssh_key": ssh_key,
"target_version": target_version,
"metrics_fifo": metrics_fifo,
"snapshot_type": snapshot_type
})
cons = consumer.LambdaConsumer(
func=lambda cons, result: cons.consume_stat(
st_name="max", ms_name="latency", value=result),
func_kwargs={})
eager_map(
cons.set_measurement_def,
snapshot_create_measurements(context.microvm.name(),
snapshot_type))
st_core.add_pipe(producer=prod,
consumer=cons,
tag=context.microvm.name())
# Gather results and verify pass criteria.
try:
result = st_core.run_exercise()
except core.CoreException as err:
handle_failure(file_dumper, err)
dump_test_result(file_dumper, result)
