def test_patch_drive_snapshot(bin_cloner_path):
"""
Test that a patched drive is correctly used by guests loaded from snapshot.
@type: functional
"""
logger = logging.getLogger("snapshot_sequence")
vm_builder = MicrovmBuilder(bin_cloner_path)
snapshot_type = SnapshotType.FULL
diff_snapshots = False
# Use a predefined vm instance.
vm_instance = vm_builder.build_vm_nano()
basevm = vm_instance.vm
root_disk = vm_instance.disks[0]
ssh_key = vm_instance.ssh_key
# Add a scratch 128MB RW non-root block device.
scratchdisk1 = drive_tools.FilesystemFile(tempfile.mktemp(), size=128)
basevm.add_drive("scratch", scratchdisk1.path)
basevm.start()
ssh_connection = net_tools.SSHConnection(basevm.ssh_config)
# Verify if guest can run commands.
exit_code, _, _ = ssh_connection.execute_command("sync")
assert exit_code == 0
# Update drive to have another backing file, double in size.
new_file_size_mb = 2 * int(scratchdisk1.size() / (1024 * 1024))
logger.info("Patch drive, new file: size %sMB.", new_file_size_mb)
scratchdisk1 = drive_tools.FilesystemFile(tempfile.mktemp(),
new_file_size_mb)
basevm.patch_drive("scratch", scratchdisk1)
logger.info("Create %s #0.", snapshot_type)
# Create a snapshot builder from a microvm.
snapshot_builder = SnapshotBuilder(basevm)
disks = [root_disk.local_path(), scratchdisk1.path]
# Create base snapshot.
snapshot = snapshot_builder.create(disks, ssh_key, snapshot_type)
basevm.kill()
# Load snapshot in a new Firecracker microVM.
logger.info("Load snapshot, mem %s", snapshot.mem)
microvm, _ = vm_builder.build_from_snapshot(snapshot,
resume=True,
diff_snapshots=diff_snapshots)
# Attempt to connect to resumed microvm.
ssh_connection = net_tools.SSHConnection(microvm.ssh_config)
# Verify the new microVM has the right scratch drive.
guest_drive_size = _get_guest_drive_size(ssh_connection)
assert guest_drive_size == str(scratchdisk1.size())
microvm.kill()
def snapshot_resume_producer(logger, vm_builder, snapshot, snapshot_type,
use_ramdisk):
"""Produce results for snapshot resume tests."""
microvm, metrics_fifo = vm_builder.build_from_snapshot(
snapshot,
True,
snapshot_type == SnapshotType.DIFF,
use_ramdisk=use_ramdisk)
# Attempt to connect to resumed microvm.
ssh_connection = net_tools.SSHConnection(microvm.ssh_config)
# Verify if guest can run commands.
exit_code, _, _ = ssh_connection.execute_command("ls")
assert exit_code == 0
value = 0
# Parse all metric data points in search of load_snapshot time.
metrics = microvm.get_all_metrics(metrics_fifo)
for data_point in metrics:
metrics = json.loads(data_point)
cur_value = metrics['latencies_us']['load_snapshot'] / USEC_IN_MSEC
if cur_value > 0:
value = cur_value
break
logger.info("Latency {} ms".format(value))
return value
def test_attach_maximum_devices(test_microvm_with_api, network_config):
"""
Test attaching maximum number of devices to the microVM.
@type: functional
"""
test_microvm = test_microvm_with_api
test_microvm.spawn()
# Set up a basic microVM.
test_microvm.basic_config()
# Add (`MAX_DEVICES_ATTACHED` - 1) devices because the rootfs
# has already been configured in the `basic_config()`function.
guest_ips = []
for i in range(MAX_DEVICES_ATTACHED - 1):
# Create tap before configuring interface.
_tap, _host_ip, guest_ip = test_microvm.ssh_network_config(
network_config, str(i))
guest_ips.append(guest_ip)
test_microvm.start()
# Test that network devices attached are operational.
for i in range(MAX_DEVICES_ATTACHED - 1):
test_microvm.ssh_config['hostname'] = guest_ips[i]
ssh_connection = net_tools.SSHConnection(test_microvm.ssh_config)
# Verify if guest can run commands.
exit_code, _, _ = ssh_connection.execute_command("sync")
assert exit_code == 0
def test_deflate_on_oom_true(test_microvm_with_ssh_and_balloon,
network_config):
"""Verify that setting the `deflate_on_oom` to True works correctly."""
test_microvm = test_microvm_with_ssh_and_balloon
test_microvm.spawn()
test_microvm.basic_config()
_tap, _, _ = test_microvm.ssh_network_config(network_config, '1')
test_microvm.ssh_config['ssh_key_path'] = os.path.join(
test_microvm.fsfiles, 'debian.rootfs.id_rsa')
# Add a deflated memory balloon.
response = test_microvm.balloon.put(amount_mb=0,
deflate_on_oom=True,
must_tell_host=False,
stats_polling_interval_s=0)
assert test_microvm.api_session.is_status_no_content(response.status_code)
# Start the microvm.
test_microvm.start()
# Get an ssh connection to the microvm.
ssh_connection = net_tools.SSHConnection(test_microvm.ssh_config)
# Inflate the balloon
response = test_microvm.balloon.patch(amount_mb=172)
assert test_microvm.api_session.is_status_no_content(response.status_code)
time.sleep(5)
# Check that using memory doesn't lead to an out of memory error.
# Note that due to `test_deflate_on_oom_false`, we know that
# if `deflate_on_oom` were set to False, then such an error
# would have happened.
make_guest_dirty_memory(ssh_connection)
def test_handled_signals(test_microvm_with_ssh, network_config):
"""Test that handled signals don't kill the microVM."""
microvm = test_microvm_with_ssh
microvm.spawn()
# We don't need to monitor the memory for this test.
microvm.memory_events_queue = None
microvm.basic_config(vcpu_count=2)
# Configure a network interface.
_tap, _, _ = microvm.ssh_network_config(network_config, '1')
microvm.start()
firecracker_pid = int(microvm.jailer_clone_pid)
# Open a SSH connection to validate the microVM stays alive.
ssh_connection = net_tools.SSHConnection(microvm.ssh_config)
# Just validate a simple command: `nproc`
cmd = "nproc"
_, stdout, stderr = ssh_connection.execute_command(cmd)
assert stderr.read().decode("utf-8") == ""
assert int(stdout.read().decode("utf-8")) == 2
# We have a handler installed for this signal.
os.kill(firecracker_pid, SIGRTMIN+1)
# Validate the microVM is still up and running.
_, stdout, stderr = ssh_connection.execute_command(cmd)
assert stderr.read().decode("utf-8") == ""
assert int(stdout.read().decode("utf-8")) == 2
def test_with_config_and_metadata_no_api(test_microvm_with_api, vm_config_file,
metadata_file):
"""
Test microvm start when config/mmds and API server thread is disabled.
Ensures the metadata is stored successfully inside the MMDS and
is available to reach from the guest's side.
@type: functional
"""
test_microvm = test_microvm_with_api
_configure_vm_from_json(test_microvm, vm_config_file)
_add_metadata_file(test_microvm, metadata_file)
_configure_network_interface(test_microvm)
test_microvm.jailer.extra_args.update({'no-api': None})
test_microvm.spawn()
ssh_connection = net_tools.SSHConnection(test_microvm.ssh_config)
# Get MMDS version and IPv4 address configured from the file.
version, ipv4_address = _get_optional_fields_from_file(vm_config_file)
cmd = 'ip route add {} dev eth0'.format(ipv4_address)
_, stdout, stderr = ssh_connection.execute_command(cmd)
assert stderr.read() == stdout.read() == ''
# Fetch data from MMDS from the guest's side.
cmd = _build_cmd_to_fetch_metadata(ssh_connection, version, ipv4_address)
_, stdout, _ = ssh_connection.execute_command(cmd)
# Compare response against the expected MMDS contents.
with open(metadata_file, encoding='utf-8') as metadata:
assert json.load(stdout) == json.load(metadata)
def _check_drives(test_microvm, assert_dict, keys_array):
ssh_connection = net_tools.SSHConnection(test_microvm.ssh_config)
_, stdout, stderr = ssh_connection.execute_command('blockdev --report')
assert stderr.read().decode('utf-8') == ''
_process_blockdev_output(stdout.read().decode('utf-8'), assert_dict,
keys_array)
def create_snapshot(bin_cloner_path):
"""Create a snapshot of a microVM."""
vm_builder = MicrovmBuilder(bin_cloner_path)
vm_instance = vm_builder.build_vm_nano()
basevm = vm_instance.vm
root_disk = vm_instance.disks[0]
ssh_key = vm_instance.ssh_key
# Add a memory balloon.
response = basevm.balloon.put(amount_mib=0,
deflate_on_oom=True,
stats_polling_interval_s=0)
assert basevm.api_session.is_status_no_content(response.status_code)
basevm.start()
ssh_connection = net_tools.SSHConnection(basevm.ssh_config)
# Verify if guest can run commands.
exit_code, _, _ = ssh_connection.execute_command("sync")
assert exit_code == 0
# Create a snapshot builder from a microvm.
snapshot_builder = SnapshotBuilder(basevm)
# Create base snapshot.
snapshot = snapshot_builder.create([root_disk.local_path()], ssh_key)
basevm.kill()
return snapshot
def test_deflate_on_oom_false(test_microvm_with_ssh_and_balloon,
network_config):
"""Verify that setting the `deflate_on_oom` to False works correctly."""
test_microvm = test_microvm_with_ssh_and_balloon
test_microvm.spawn()
test_microvm.basic_config()
_tap, _, _ = test_microvm.ssh_network_config(network_config, '1')
test_microvm.ssh_config['ssh_key_path'] = os.path.join(
test_microvm.fsfiles, 'debian.rootfs.id_rsa')
# Add a memory balloon.
response = test_microvm.balloon.put(amount_mib=0,
deflate_on_oom=False,
stats_polling_interval_s=0)
assert test_microvm.api_session.is_status_no_content(response.status_code)
# Start the microvm.
test_microvm.start()
# Get an ssh connection to the microvm.
firecracker_pid = test_microvm.jailer_clone_pid
ssh_connection = net_tools.SSHConnection(test_microvm.ssh_config)
# Inflate the balloon.
response = test_microvm.balloon.patch(amount_mib=172)
assert test_microvm.api_session.is_status_no_content(response.status_code)
# This call will internally wait for rss to become stable.
_ = get_stable_rss_mem_by_pid(firecracker_pid)
# Check that using memory does lead to an out of memory error.
make_guest_dirty_memory(ssh_connection, should_oom=True)
def create_512mb_full_snapshot(bin_cloner_path,
target_version: str = None,
fc_binary=None,
jailer_binary=None):
"""Create a snapshoft from a 2vcpu 512MB microvm."""
vm_instance = C3micro.spawn(bin_cloner_path, True, fc_binary,
jailer_binary)
# Attempt to connect to the fresh microvm.
ssh_connection = net_tools.SSHConnection(vm_instance.vm.ssh_config)
# Run a fio workload and validate succesfull execution.
fio = """fio --filename=/dev/vda --direct=1 --rw=randread --bs=4k \
--ioengine=libaio --iodepth=16 --runtime=2 --numjobs=4 --time_based \
--group_reporting --name=iops-test-job --eta-newline=1 --readonly"""
exit_code, _, _ = ssh_connection.execute_command(fio)
assert exit_code == 0
# Create a snapshot builder from a microvm.
snapshot_builder = SnapshotBuilder(vm_instance.vm)
# The snapshot builder expects disks as paths, not artifacts.
disks = []
for disk in vm_instance.disks:
disks.append(disk.local_path())
snapshot = snapshot_builder.create(disks, vm_instance.ssh_key,
SnapshotType.FULL, target_version)
vm_instance.vm.kill()
return snapshot
def test_restore_in_past_versions(bin_cloner_path):
"""Test scenario: create a snapshot and restore in previous versions."""
logger = logging.getLogger("snapshot_version")
artifacts = ArtifactCollection(_test_images_s3_bucket())
# Fetch all snapshots artifacts.
# "fc_release" is the key that should be used for per release snapshot
# artifacts. Such snapshots are created at release time and target the
# current version. We are going to restore all these snapshots with current
# testing build.
firecracker_artifacts = artifacts.firecrackers()
for firecracker in firecracker_artifacts:
firecracker.download()
jailer = firecracker.jailer()
jailer.download()
# The target version is in the name of the firecracker binary from S3.
# We also strip the "v" as fc expects X.Y.Z version string.
target_version = firecracker.base_name()[1:]
logger.info("Creating snapshot for version: %s", target_version)
# Create a fresh snapshot targeted at the binary artifact version.
snapshot = create_512mb_full_snapshot(bin_cloner_path, target_version)
builder = MicrovmBuilder(bin_cloner_path, firecracker.local_path(),
jailer.local_path())
microvm, _ = builder.build_from_snapshot(snapshot, True, False)
logger.info("Using Firecracker: %s", firecracker.local_path())
logger.info("Using Jailer: %s", jailer.local_path())
# Attempt to connect to resumed microvm.
ssh_connection = net_tools.SSHConnection(microvm.ssh_config)
exit_code, _, _ = ssh_connection.execute_command("sleep 1 && sync")
assert exit_code == 0
def _g2h_send_ping(context):
"""Send ping from guest to host."""
logger = context.custom['logger']
vm_builder = context.custom['builder']
interval_between_req = context.custom['interval']
name = context.custom['name']
file_dumper = context.custom['results_file_dumper']
logger.info("Testing {} with microvm: \"{}\", kernel {}, disk {} ".format(
name, context.microvm.name(), context.kernel.name(),
context.disk.name()))
# Create a rw copy artifact.
rw_disk = context.disk.copy()
# Get ssh key from read-only artifact.
ssh_key = context.disk.ssh_key()
# Create a fresh microvm from aftifacts.
basevm = vm_builder.build(kernel=context.kernel,
disks=[rw_disk],
ssh_key=ssh_key,
config=context.microvm)
basevm.start()
# Check if the needed CPU cores are available. We have the API thread, VMM
# thread and then one thread for each configured vCPU.
assert CpuMap.len() >= 2 + basevm.vcpus_count
# Pin uVM threads to physical cores.
current_cpu_id = 0
assert basevm.pin_vmm(current_cpu_id), \
"Failed to pin firecracker thread."
current_cpu_id += 1
assert basevm.pin_api(current_cpu_id), \
"Failed to pin fc_api thread."
for i in range(basevm.vcpus_count):
current_cpu_id += 1
assert basevm.pin_vcpu(i, current_cpu_id + i), \
f"Failed to pin fc_vcpu {i} thread."
custom = {
"microvm": context.microvm.name(),
"kernel": context.kernel.name(),
"disk": context.disk.name(),
"cpu_model_name": get_cpu_model_name()
}
st_core = core.Core(name="network_latency", iterations=1, custom=custom)
cons = consumer.LambdaConsumer(
func=consume_ping_output,
func_kwargs={"requests": context.custom['requests']})
cmd = PING.format(context.custom['requests'], interval_between_req,
DEFAULT_HOST_IP)
prod = producer.SSHCommand(cmd, net_tools.SSHConnection(basevm.ssh_config))
st_core.add_pipe(producer=prod, consumer=cons, tag="ping")
# Gather results and verify pass criteria.
result = st_core.run_exercise(file_dumper is None)
if file_dumper:
file_dumper.writeln(json.dumps(result))
def test_restore_from_past_versions(bin_cloner_path):
"""Test scenario: restore all previous version snapshots."""
logger = logging.getLogger("snapshot_version")
artifacts = ArtifactCollection(_test_images_s3_bucket())
# Fetch all firecracker binaries.
# With each binary create a snapshot and try to restore in current
# version.
firecracker_artifacts = artifacts.firecrackers()
for firecracker in firecracker_artifacts:
firecracker.download()
jailer = firecracker.jailer()
jailer.download()
logger.info("Source Firecracker: %s", firecracker.local_path())
logger.info("Source Jailer: %s", jailer.local_path())
# Create a fresh snapshot using the binary artifacts.
builder = MicrovmBuilder(bin_cloner_path, firecracker.local_path(),
jailer.local_path())
snapshot = create_512mb_full_snapshot(bin_cloner_path, None,
firecracker.local_path(),
jailer.local_path())
microvm, _ = builder.build_from_snapshot(snapshot, True, False)
ssh_connection = net_tools.SSHConnection(microvm.ssh_config)
exit_code, _, _ = ssh_connection.execute_command("sleep 1 && sync")
assert exit_code == 0
def _test_rss_memory_lower(test_microvm):
"""Check inflating the balloon makes guest use less rss memory."""
# Get the firecracker pid, and open an ssh connection.
firecracker_pid = test_microvm.jailer_clone_pid
ssh_connection = net_tools.SSHConnection(test_microvm.ssh_config)
# Using deflate_on_oom, get the RSS as low as possible
response = test_microvm.balloon.patch(amount_mib=200)
assert test_microvm.api_session.is_status_no_content(response.status_code)
# Get initial rss consumption.
init_rss = get_stable_rss_mem_by_pid(firecracker_pid)
# Get the balloon back to 0.
response = test_microvm.balloon.patch(amount_mib=0)
assert test_microvm.api_session.is_status_no_content(response.status_code)
# This call will internally wait for rss to become stable.
_ = get_stable_rss_mem_by_pid(firecracker_pid)
# Dirty memory, then inflate balloon and get ballooned rss consumption.
make_guest_dirty_memory(ssh_connection)
response = test_microvm.balloon.patch(amount_mib=200)
assert test_microvm.api_session.is_status_no_content(response.status_code)
balloon_rss = get_stable_rss_mem_by_pid(firecracker_pid)
# Check that the ballooning reclaimed the memory.
assert balloon_rss - init_rss <= 15000
def test_handled_signals(test_microvm_with_ssh, network_config):
"""Test that handled signals don't kill the microVM."""
microvm = test_microvm_with_ssh
microvm.spawn()
# We don't need to monitor the memory for this test.
microvm.memory_monitor = None
microvm.basic_config(vcpu_count=2)
# Configure a network interface.
_tap, _, _ = microvm.ssh_network_config(network_config, '1')
microvm.start()
firecracker_pid = int(microvm.jailer_clone_pid)
# Open a SSH connection to validate the microVM stays alive.
ssh_connection = net_tools.SSHConnection(microvm.ssh_config)
# Just validate a simple command: `nproc`
cmd = "nproc"
_, stdout, stderr = ssh_connection.execute_command(cmd)
assert stderr.read() == ""
assert int(stdout.read()) == 2
# We have a handler installed for this signal.
# The 35 is the SIGRTMIN for musl libc.
# We hardcode this value since the SIGRTMIN python reports
# is 34, which is likely the one for glibc.
os.kill(firecracker_pid, 35)
# Validate the microVM is still up and running.
_, stdout, stderr = ssh_connection.execute_command(cmd)
assert stderr.read() == ""
assert int(stdout.read()) == 2
def test_reboot(test_microvm_with_ssh, network_config):
"""Test reboot from guest kernel."""
test_microvm = test_microvm_with_ssh
test_microvm.jailer.daemonize = False
test_microvm.spawn()
# We don't need to monitor the memory for this test because we are
# just rebooting and the process dies before pmap gets the RSS.
test_microvm.memory_monitor = None
# Set up the microVM with 4 vCPUs, 256 MiB of RAM, 0 network ifaces, and
# a root file system with the rw permission. The network interfaces is
# added after we get a unique MAC and IP.
test_microvm.basic_config(vcpu_count=4)
_tap, _, _ = test_microvm.ssh_network_config(network_config, '1')
# Configure metrics system.
metrics_fifo_path = os.path.join(test_microvm.path, 'metrics_fifo')
metrics_fifo = log_tools.Fifo(metrics_fifo_path)
response = test_microvm.metrics.put(
metrics_path=test_microvm.create_jailed_resource(metrics_fifo.path)
)
assert test_microvm.api_session.is_status_no_content(response.status_code)
test_microvm.start()
# Get Firecracker PID so we can count the number of threads.
firecracker_pid = test_microvm.jailer_clone_pid
# Get number of threads in Firecracker
cmd = 'ps -o nlwp {} | tail -1 | awk \'{{print $1}}\''.format(
firecracker_pid
)
_, stdout, _ = utils.run_cmd(cmd)
nr_of_threads = stdout.rstrip()
assert int(nr_of_threads) == 6
# Consume existing metrics
lines = metrics_fifo.sequential_reader(100)
assert len(lines) == 1
# Rebooting Firecracker sends an exit event and should gracefully kill.
# the instance.
ssh_connection = net_tools.SSHConnection(test_microvm.ssh_config)
ssh_connection.execute_command("reboot")
while True:
# Pytest's timeout will kill the test even if the loop doesn't exit.
try:
os.kill(firecracker_pid, 0)
time.sleep(0.01)
except OSError:
break
# Consume existing metrics
lines = metrics_fifo.sequential_reader(100)
assert len(lines) == 1
# Make sure that the FC process was not killed by a seccomp fault
assert json.loads(lines[0])["seccomp"]["num_faults"] == 0
def test_device_ordering(test_microvm_with_ssh, network_config):
"""Verify device ordering.
The root device should correspond to /dev/vda in the guest and
the order of the other devices should match their configuration order.
"""
test_microvm = test_microvm_with_ssh
test_microvm.spawn()
# Add first scratch block device.
fs1 = drive_tools.FilesystemFile(
os.path.join(test_microvm.fsfiles, 'scratch1'),
size=128
)
test_microvm.add_drive(
'scratch1',
fs1.path
)
# Set up the microVM with 1 vCPUs, 256 MiB of RAM, 0 network ifaces,
# a read-write root file system (this is the second block device added).
# The network interface is added after we get a unique MAC and IP.
test_microvm.basic_config()
# Add the third block device.
fs2 = drive_tools.FilesystemFile(
os.path.join(test_microvm.fsfiles, 'scratch2'),
size=512
)
test_microvm.add_drive(
'scratch2',
fs2.path
)
_tap, _, _ = test_microvm_with_ssh.ssh_network_config(network_config, '1')
test_microvm.start()
# Determine the size of the microVM rootfs in bytes.
try:
result = utils.run_cmd(
'du --apparent-size --block-size=1 {}'
.format(test_microvm.rootfs_file),
)
except ChildProcessError:
pytest.skip('Failed to get microVM rootfs size: {}'
.format(result.stderr))
assert len(result.stdout.split()) == 2
rootfs_size = result.stdout.split('\t')[0]
# The devices were added in this order: fs1, rootfs, fs2.
# However, the rootfs is the root device and goes first,
# so we expect to see this order: rootfs, fs1, fs2.
# The devices are identified by their size.
ssh_connection = net_tools.SSHConnection(test_microvm.ssh_config)
_check_block_size(ssh_connection, '/dev/vda', rootfs_size)
_check_block_size(ssh_connection, '/dev/vdb', fs1.size())
_check_block_size(ssh_connection, '/dev/vdc', fs2.size())
def test_cpu_template(test_microvm_with_ssh, network_config, cpu_template):
"""Check that AVX2 & AVX512 instructions are disabled.
This is a rather dummy test for checking that some features are not
exposed by mistake. It is a first step into checking the t2 & c3
templates. In a next iteration we should check **all** cpuid entries, not
just these features. We can achieve this with a template
containing all features on a t2/c3 instance and check that the cpuid in
the guest is an exact match of the template.
"""
common_masked_features = ["avx512", "mpx", "clflushopt", "clwb", "xsavec",
"xgetbv1", "xsaves", "pku", "ospke"]
c3_masked_features = ["avx2"]
test_microvm = test_microvm_with_ssh
test_microvm.spawn()
test_microvm.basic_config(vcpu_count=1)
# Set template as specified in the `cpu_template` parameter.
response = test_microvm.machine_cfg.put(
vcpu_count=1,
mem_size_mib=256,
ht_enabled=False,
cpu_template=cpu_template,
)
assert test_microvm.api_session.is_status_no_content(response.status_code)
_tap, _, _ = test_microvm.ssh_network_config(network_config, '1')
response = test_microvm.actions.put(action_type='InstanceStart')
if get_cpu_vendor() != CpuVendor.INTEL:
# We shouldn't be able to apply Intel templates on AMD hosts
assert test_microvm.api_session.is_status_bad_request(
response.status_code)
return
assert test_microvm.api_session.is_status_no_content(
response.status_code)
ssh_connection = net_tools.SSHConnection(test_microvm.ssh_config)
guest_cmd = "cat /proc/cpuinfo | grep 'flags' | head -1"
_, stdout, stderr = ssh_connection.execute_command(guest_cmd)
assert stderr.read() == ''
cpu_flags_output = stdout.readline().rstrip()
if cpu_template == "C3":
for feature in c3_masked_features:
assert feature not in cpu_flags_output
# Check that all features in `common_masked_features` are properly masked.
for feature in common_masked_features:
assert feature not in cpu_flags_output
# Check if XSAVE PKRU is masked for T3/C2.
expected_cpu_features = {
"XCR0 supported: PKRU state": "false"
}
_check_guest_cmd_output(test_microvm, "cpuid -1", None, '=',
expected_cpu_features)
def test_guest_mmds_hang(test_microvm_with_ssh, network_config):
"""Test the MMDS json response."""
test_microvm = test_microvm_with_ssh
test_microvm.spawn()
response = test_microvm.mmds.get()
assert test_microvm.api_session.is_status_ok(response.status_code)
assert response.json() == {}
data_store = {
'latest': {
'meta-data': {
'ami-id': 'ami-12345678'
}
}
}
response = test_microvm.mmds.put(json=data_store)
assert test_microvm.api_session.is_status_no_content(response.status_code)
response = test_microvm.mmds.get()
assert test_microvm.api_session.is_status_ok(response.status_code)
assert response.json() == data_store
test_microvm.basic_config(vcpu_count=1)
_tap = test_microvm.ssh_network_config(
network_config,
'1',
allow_mmds_requests=True
)
test_microvm.start()
ssh_connection = net_tools.SSHConnection(test_microvm.ssh_config)
cmd = 'ip route add 169.254.169.254 dev eth0'
_, stdout, stderr = ssh_connection.execute_command(cmd)
_assert_out(stdout, stderr, '')
# Test for a GET request with a content length longer than
# the actual length of the body.
cmd = 'curl -m 2 -s'
cmd += ' -X GET'
cmd += ' -H "Content-Length: 100"'
cmd += ' -H "Accept: application/json"'
cmd += ' -d "some body"'
cmd += ' http://169.254.169.254/'
_, stdout, _ = ssh_connection.execute_command(cmd)
assert 'Invalid request' in stdout.read()
# Do the same for a PUT request.
cmd = 'curl -m 2 -s'
cmd += ' -X PUT'
cmd += ' -H "Content-Length: 100"'
cmd += ' -H "Accept: application/json"'
cmd += ' -d "some body"'
cmd += ' http://169.254.169.254/'
_, stdout, _ = ssh_connection.execute_command(cmd)
assert 'Invalid request' in stdout.read()
def test_drive_io_engine(test_microvm_with_api, network_config):
"""
Test io_engine configuration.
Test that the io_engine can be configured via the API on kernels that
support the given type and that FC returns an error otherwise.
@type: functional
"""
test_microvm = test_microvm_with_api
test_microvm.spawn()
test_microvm.basic_config(add_root_device=False)
test_microvm.ssh_network_config(network_config, '1')
supports_io_uring = is_io_uring_supported()
response = test_microvm.drive.put(
drive_id='rootfs',
path_on_host=test_microvm.create_jailed_resource(
test_microvm.rootfs_file),
is_root_device=True,
is_read_only=False,
# Set the opposite of the default backend type.
io_engine="Sync" if supports_io_uring else "Async"
)
if not supports_io_uring:
# The Async engine is not supported for older kernels.
assert test_microvm.api_session.is_status_bad_request(
response.status_code)
test_microvm.check_log_message(
"Received Error. Status code: 400 Bad Request. Message: Unable"
" to create the block device FileEngine(UnsupportedEngine(Async))")
# Now configure the default engine type and check that it works.
response = test_microvm.drive.put_with_default_io_engine(
drive_id='rootfs',
path_on_host=test_microvm.create_jailed_resource(
test_microvm.rootfs_file),
is_root_device=True,
is_read_only=False,
)
assert test_microvm.api_session.is_status_no_content(
response.status_code)
test_microvm.start()
ssh_conn = net_tools.SSHConnection(test_microvm.ssh_config)
# Execute a simple command to check that the guest booted successfully.
rc, _, stderr = ssh_conn.execute_command("sync")
assert rc == 0
assert stderr.read() == ''
assert test_microvm.full_cfg.get().json(
)['drives'][0]['io_engine'] == "Sync"
def _test_pause_resume(context):
logger = context.custom['logger']
vm_builder = context.custom['builder']
logger.info("Testing microvm: \"{}\" with kernel {} and disk {} ".format(
context.microvm.name(), context.kernel.name(), context.disk.name()))
microvm = vm_builder.build(context.kernel, [context.disk], context.microvm)
tap = microvm.ssh_network_config(context.custom['network_config'], '1')
# Pausing the microVM before being started is not allowed.
response = microvm.vm.patch(state='Paused')
assert microvm.api_session.is_status_bad_request(response.status_code)
# Resuming the microVM before being started is also not allowed.
response = microvm.vm.patch(state='Resumed')
assert microvm.api_session.is_status_bad_request(response.status_code)
microvm.start()
ssh_connection = net_tools.SSHConnection(microvm.ssh_config)
# Verify guest is active.
exit_code, _, _ = ssh_connection.execute_command("ls")
assert exit_code == 0
# Pausing the microVM after it's been started is successful.
response = microvm.vm.patch(state='Paused')
assert microvm.api_session.is_status_no_content(response.status_code)
# Verify guest is no longer active.
exit_code, _, _ = ssh_connection.execute_command("ls")
assert exit_code != 0
# Pausing the microVM when it is already `Paused` is allowed
# (microVM remains in `Paused` state).
response = microvm.vm.patch(state='Paused')
assert microvm.api_session.is_status_no_content(response.status_code)
# Resuming the microVM is successful.
response = microvm.vm.patch(state='Resumed')
assert microvm.api_session.is_status_no_content(response.status_code)
# Verify guest is active again.
exit_code, _, _ = ssh_connection.execute_command("ls")
assert exit_code == 0
# Resuming the microVM when it is already `Resumed` is allowed
# (microVM remains in the running state).
response = microvm.vm.patch(state='Resumed')
assert microvm.api_session.is_status_no_content(response.status_code)
# Verify guest is still active.
exit_code, _, _ = ssh_connection.execute_command("ls")
assert exit_code == 0
microvm.kill()
del tap
def test_pause_resume(test_microvm_with_ssh, network_config):
"""Test pausing and resuming the vCPUs."""
test_microvm = test_microvm_with_ssh
test_microvm.spawn()
# Set up the microVM with 2 vCPUs, 256 MiB of RAM and a root file
# system with the rw permission.
test_microvm.basic_config()
# Create tap before configuring interface.
_tap1, _, _ = test_microvm.ssh_network_config(network_config, '1')
# Pausing the microVM before being started is not allowed.
response = test_microvm.vm.patch(state='Paused')
assert test_microvm.api_session.is_status_bad_request(response.status_code)
# Resuming the microVM before being started is also not allowed.
response = test_microvm.vm.patch(state='Resumed')
assert test_microvm.api_session.is_status_bad_request(response.status_code)
# Start microVM.
test_microvm.start()
ssh_connection = net_tools.SSHConnection(test_microvm.ssh_config)
# Verify guest is active.
exit_code, _, _ = ssh_connection.execute_command("ls")
assert exit_code == 0
# Pausing the microVM after it's been started is successful.
response = test_microvm.vm.patch(state='Paused')
assert test_microvm.api_session.is_status_no_content(response.status_code)
# Verify guest is no longer active.
exit_code, _, _ = ssh_connection.execute_command("ls")
assert exit_code != 0
# Pausing the microVM when it is already `Paused` is allowed
# (microVM remains in `Paused` state).
response = test_microvm.vm.patch(state='Paused')
assert test_microvm.api_session.is_status_no_content(response.status_code)
# Resuming the microVM is successful.
response = test_microvm.vm.patch(state='Resumed')
assert test_microvm.api_session.is_status_no_content(response.status_code)
# Verify guest is active again.
exit_code, _, _ = ssh_connection.execute_command("ls")
assert exit_code == 0
# Resuming the microVM when it is already `Resumed` is allowed
# (microVM remains in the running state).
response = test_microvm.vm.patch(state='Resumed')
assert test_microvm.api_session.is_status_no_content(response.status_code)
# Verify guest is still active.
exit_code, _, _ = ssh_connection.execute_command("ls")
assert exit_code == 0
def check_masked_features(test_microvm, cpu_template):
"""Check that AVX2 & AVX512 instructions are disabled."""
common_masked_features_lscpu = ["dtes64", "monitor", "ds_cpl", "tm2",
"cnxt-id", "sdbg", "xtpr", "pdcm",
"osxsave",
"psn", "ds", "acpi", "tm", "ss", "pbe",
"fpdp", "rdt_m", "rdt_a", "mpx", "avx512f",
"intel_pt",
"avx512_vpopcntdq",
"3dnowprefetch", "pdpe1gb"]
common_masked_features_cpuid = {"SGX": "false", "HLE": "false",
"RTM": "false", "RDSEED": "false",
"ADX": "false", "AVX512IFMA": "false",
"CLFLUSHOPT": "false", "CLWB": "false",
"AVX512PF": "false", "AVX512ER": "false",
"AVX512CD": "false", "SHA": "false",
"AVX512BW": "false", "AVX512VL": "false",
"AVX512VBMI": "false", "PKU": "false",
"OSPKE": "false", "RDPID": "false",
"SGX_LC": "false",
"AVX512_4VNNIW": "false",
"AVX512_4FMAPS": "false",
"XSAVEC": "false", "XGETBV": "false",
"XSAVES": "false"}
# These are all discoverable by cpuid -1.
c3_masked_features = {"FMA": "false", "MOVBE": "false", "BMI": "false",
"AVX2": "false", "BMI2": "false", "INVPCID": "false"}
# Check that all common features discoverable with lscpu
# are properly masked.
ssh_connection = net_tools.SSHConnection(test_microvm.ssh_config)
guest_cmd = "cat /proc/cpuinfo | grep 'flags' | head -1"
_, stdout, stderr = ssh_connection.execute_command(guest_cmd)
assert stderr.read() == ''
cpu_flags_output = stdout.readline().rstrip().split(' ')
for feature in common_masked_features_lscpu:
assert feature not in cpu_flags_output, feature
# Check that all common features discoverable with cpuid
# are properly masked.
utils.check_guest_cpuid_output(test_microvm, "cpuid -1", None, '=',
common_masked_features_cpuid)
if cpu_template == "C3":
utils.check_guest_cpuid_output(test_microvm, "cpuid -1", None, '=',
c3_masked_features)
# Check if XSAVE PKRU is masked for T3/C2.
expected_cpu_features = {
"XCR0 supported: PKRU state": "false"
}
utils.check_guest_cpuid_output(test_microvm, "cpuid -1", None, '=',
expected_cpu_features)
def _run_iperf_on_guest(test_microvm, iperf_cmd, hostname):
"""Run a client related iperf command through an SSH connection."""
test_microvm.ssh_config['hostname'] = hostname
ssh_connection = net_tools.SSHConnection(test_microvm.ssh_config)
_, stdout, stderr = ssh_connection.execute_command(iperf_cmd)
assert stderr.read() == ''
out = stdout.read()
return out
def test_brand_string(test_microvm_with_api, network_config):
"""
Ensure good formatting for the guest brand string.
* For Intel CPUs, the guest brand string should be:
Intel(R) Xeon(R) Processor @ {host frequency}
where {host frequency} is the frequency reported by the host CPUID
(e.g. 4.01GHz)
* For AMD CPUs, the guest brand string should be:
AMD EPYC
* For other CPUs, the guest brand string should be:
""
@type: functional
"""
cif = open("/proc/cpuinfo", "r", encoding="utf-8")
host_brand_string = None
while True:
line = cif.readline()
if line == "":
break
mo = re.search("^model name\\s+:\\s+(.+)$", line)
if mo:
host_brand_string = mo.group(1)
cif.close()
assert host_brand_string is not None
test_microvm = test_microvm_with_api
test_microvm.spawn()
test_microvm.basic_config(vcpu_count=1)
_tap, _, _ = test_microvm.ssh_network_config(network_config, "1")
test_microvm.start()
ssh_connection = net_tools.SSHConnection(test_microvm.ssh_config)
guest_cmd = "cat /proc/cpuinfo | grep 'model name' | head -1"
_, stdout, stderr = ssh_connection.execute_command(guest_cmd)
assert stderr.read() == ""
line = stdout.readline().rstrip()
mo = re.search("^model name\\s+:\\s+(.+)$", line)
assert mo
guest_brand_string = mo.group(1)
assert guest_brand_string
cpu_vendor = utils.get_cpu_vendor()
expected_guest_brand_string = ""
if cpu_vendor == utils.CpuVendor.AMD:
expected_guest_brand_string += "AMD EPYC"
elif cpu_vendor == utils.CpuVendor.INTEL:
expected_guest_brand_string = "Intel(R) Xeon(R) Processor"
mo = re.search("[.0-9]+[MG]Hz", host_brand_string)
if mo:
expected_guest_brand_string += " @ " + mo.group(0)
assert guest_brand_string == expected_guest_brand_string
def create_snapshots(context):
"""Snapshot microVM built from vm configuration file."""
vm = setup_vm(context)
# Get ssh key from read-only artifact.
ssh_key = context.disk.ssh_key()
ssh_key.download(vm.path)
vm.ssh_config['ssh_key_path'] = ssh_key.local_path()
os.chmod(vm.ssh_config['ssh_key_path'], 0o400)
cpu_template = context.custom['cpu_template']
fn = partial(add_cpu_template, cpu_template)
_configure_vm_from_json(vm, VM_CONFIG_FILE, json_xform=fn)
configure_network_interfaces(vm)
vm.spawn()
# Ensure the microVM has started.
response = vm.machine_cfg.get()
assert vm.api_session.is_status_ok(response.status_code)
assert vm.state == "Running"
# Populate MMDS.
data_store = {
'latest': {
'meta-data': {
'ami-id': 'ami-12345678',
'reservation-id': 'r-fea54097',
'local-hostname': 'ip-10-251-50-12.ec2.internal',
'public-hostname': 'ec2-203-0-113-25.compute-1.amazonaws.com'
}
}
}
populate_mmds(vm, data_store)
# Iterate and validate connectivity on all ifaces after boot.
for iface in net_ifaces:
vm.ssh_config['hostname'] = iface.guest_ip
ssh_connection = net_tools.SSHConnection(vm.ssh_config)
exit_code, _, _ = ssh_connection.execute_command("sync")
assert exit_code == 0
# Validate MMDS.
validate_mmds(ssh_connection, data_store)
# Create a snapshot builder from a microVM.
snapshot_builder = SnapshotBuilder(vm)
# Snapshot the microVM.
snapshot = snapshot_builder.create([vm.rootfs_file],
ssh_key,
SnapshotType.DIFF,
net_ifaces=net_ifaces)
copy_snapshot_artifacts(snapshot, vm.rootfs_file, context.kernel.name(),
ssh_key, cpu_template)
vm.kill()
def test_rescan_dev(test_microvm_with_api, network_config):
"""
Verify that rescan works with a device-backed virtio device.
@type: functional
"""
test_microvm = test_microvm_with_api
test_microvm.spawn()
session = test_microvm.api_session
# Set up the microVM with 1 vCPUs, 256 MiB of RAM, 0 network ifaces and
# a root file system with the rw permission. The network interface is
# added after we get a unique MAC and IP.
test_microvm.basic_config()
_tap, _, _ = test_microvm_with_api.ssh_network_config(network_config, '1')
# Add a scratch block device.
fs1 = drive_tools.FilesystemFile(os.path.join(test_microvm.fsfiles, 'fs1'))
test_microvm.add_drive(
'scratch',
fs1.path
)
test_microvm.start()
ssh_connection = net_tools.SSHConnection(test_microvm.ssh_config)
_check_block_size(ssh_connection, '/dev/vdb', fs1.size())
fs2 = drive_tools.FilesystemFile(
os.path.join(test_microvm.fsfiles, 'fs2'),
size=512
)
losetup = ['losetup', '--find', '--show', fs2.path]
loopback_device = None
result = None
try:
result = utils.run_cmd(losetup)
loopback_device = result.stdout.rstrip()
except ChildProcessError:
pytest.skip('failed to create a lookback device: ' +
f'stdout={result.stdout}, stderr={result.stderr}')
try:
response = test_microvm.drive.patch(
drive_id='scratch',
path_on_host=test_microvm.create_jailed_resource(loopback_device),
)
assert session.is_status_no_content(response.status_code)
_check_block_size(ssh_connection, '/dev/vdb', fs2.size())
finally:
if loopback_device:
utils.run_cmd(['losetup', '--detach', loopback_device])
def _test_memory_scrub(context):
vm_builder = context.custom['builder']
# Create a rw copy artifact.
root_disk = context.disk.copy()
# Get ssh key from read-only artifact.
ssh_key = context.disk.ssh_key()
# Create a fresh microvm from aftifacts.
microvm = vm_builder.build(
kernel=context.kernel,
disks=[root_disk],
ssh_key=ssh_key,
config=context.microvm
)
copy_util_to_rootfs(root_disk.local_path(), 'fillmem')
copy_util_to_rootfs(root_disk.local_path(), 'readmem')
# Add a memory balloon with stats enabled.
response = microvm.balloon.put(
amount_mib=0,
deflate_on_oom=True,
stats_polling_interval_s=1
)
assert microvm.api_session.is_status_no_content(response.status_code)
microvm.start()
ssh_connection = net_tools.SSHConnection(microvm.ssh_config)
# Dirty 60MB of pages.
make_guest_dirty_memory(ssh_connection, amount=(60 * MB_TO_PAGES))
# Now inflate the balloon with 60MB of pages.
response = microvm.balloon.patch(amount_mib=60)
assert microvm.api_session.is_status_no_content(response.status_code)
# Get the firecracker pid, and open an ssh connection.
firecracker_pid = microvm.jailer_clone_pid
# Wait for the inflate to complete.
_ = get_stable_rss_mem_by_pid(firecracker_pid)
# Deflate the balloon completely.
response = microvm.balloon.patch(amount_mib=0)
assert microvm.api_session.is_status_no_content(response.status_code)
# Wait for the deflate to complete.
_ = get_stable_rss_mem_by_pid(firecracker_pid)
exit_code, _, _ = ssh_connection.execute_command(
"/sbin/readmem {} {}".format(60, 1)
)
assert exit_code == 0
microvm.kill()
def test_brand_string(test_microvm_with_ssh, network_config):
"""Ensure good formatting for the guest band string.
* For Intel CPUs, the guest brand string should be:
Intel(R) Xeon(R) Processor @ {host frequency}
where {host frequency} is the frequency reported by the host CPUID
(e.g. 4.01GHz)
* For non-Intel CPUs, the guest brand string should be:
Intel(R) Xeon(R) Processor
"""
cif = open('/proc/cpuinfo', 'r')
host_brand_string = None
host_vendor_id = None
while True:
line = cif.readline()
if line == '':
break
mo = re.search("^vendor_id\\s+:\\s+(.+)$", line)
if mo:
host_vendor_id = mo.group(1)
mo = re.search("^model name\\s+:\\s+(.+)$", line)
if mo:
host_brand_string = mo.group(1)
cif.close()
assert host_vendor_id is not None
assert host_brand_string is not None
test_microvm = test_microvm_with_ssh
test_microvm.spawn()
test_microvm.basic_config(vcpu_count=1)
_tap, _, _ = test_microvm.ssh_network_config(network_config, '1')
test_microvm.start()
ssh_connection = net_tools.SSHConnection(test_microvm.ssh_config)
guest_cmd = "cat /proc/cpuinfo | grep 'model name' | head -1"
_, stdout, stderr = ssh_connection.execute_command(guest_cmd)
assert stderr.read().decode("utf-8") == ''
line = stdout.readline().decode('utf-8').rstrip()
mo = re.search("^model name\\s+:\\s+(.+)$", line)
assert mo
guest_brand_string = mo.group(1)
assert guest_brand_string
expected_guest_brand_string = ""
if host_vendor_id == "GenuineIntel":
expected_guest_brand_string = "Intel(R) Xeon(R) Processor"
mo = re.search("[.0-9]+[MG]Hz", host_brand_string)
if mo:
expected_guest_brand_string += " @ " + mo.group(0)
elif host_vendor_id == "AuthenticAMD":
expected_guest_brand_string += "AMD EPYC"
assert guest_brand_string == expected_guest_brand_string
def test_serial_block(test_microvm_with_api, network_config):
"""
Test that writing to stdout never blocks the vCPU thread.
@type: functional
"""
test_microvm = test_microvm_with_api
test_microvm.jailer.daemonize = False
test_microvm.spawn()
# Set up the microVM with 1 vCPU so we make sure the vCPU thread
# responsible for the SSH connection will also run the serial.
test_microvm.basic_config(
vcpu_count=1,
mem_size_mib=512,
boot_args="console=ttyS0 reboot=k panic=1 pci=off",
)
_tap, _, _ = test_microvm.ssh_network_config(network_config, "1")
# Configure the metrics.
metrics_fifo_path = os.path.join(test_microvm.path, "metrics_fifo")
metrics_fifo = log_tools.Fifo(metrics_fifo_path)
response = test_microvm.metrics.put(
metrics_path=test_microvm.create_jailed_resource(metrics_fifo.path))
assert test_microvm.api_session.is_status_no_content(response.status_code)
test_microvm.start()
ssh_connection = net_tools.SSHConnection(test_microvm.ssh_config)
# Get an initial reading of missed writes to the serial.
fc_metrics = test_microvm.flush_metrics(metrics_fifo)
init_count = fc_metrics["uart"]["missed_write_count"]
screen_pid = test_microvm.screen_pid
# Stop `screen` process which captures stdout so we stop consuming stdout.
subprocess.check_call("kill -s STOP {}".format(screen_pid), shell=True)
# Generate a random text file.
exit_code, _, _ = ssh_connection.execute_command(
"base64 /dev/urandom | head -c 100000 > file.txt")
# Dump output to terminal
exit_code, _, _ = ssh_connection.execute_command(
"cat file.txt > /dev/ttyS0")
assert exit_code == 0
# Check that the vCPU isn't blocked.
exit_code, _, _ = ssh_connection.execute_command("cd /")
assert exit_code == 0
# Check the metrics to see if the serial missed bytes.
fc_metrics = test_microvm.flush_metrics(metrics_fifo)
last_count = fc_metrics["uart"]["missed_write_count"]
# Should be significantly more than before the `cat` command.
assert last_count - init_count > 10000
