def test_mmds_snapshot(bin_cloner_path, version):
"""
Test MMDS behavior by restoring a snapshot on current and past FC versions.
Ensures that the version is persisted or initialised with the default if
the firecracker version does not support it.
@type: functional
"""
vm_builder = MicrovmBuilder(bin_cloner_path)
vm_instance = vm_builder.build_vm_nano(
net_ifaces=[NetIfaceConfig()]
)
# Validate current version.
_validate_mmds_snapshot(
vm_instance, vm_builder, version)
# Validate restoring in past versions.
artifacts = ArtifactCollection(_test_images_s3_bucket())
# Fetch all firecracker binaries.
# Create a snapshot with current build and restore with each FC binary
# artifact.
firecracker_artifacts = artifacts.firecrackers(
# v1.1.0 breaks snapshot compatibility with older versions.
min_version="1.1.0",
max_version=get_firecracker_version_from_toml())
for firecracker in firecracker_artifacts:
vm_instance = vm_builder.build_vm_nano(
net_ifaces=[NetIfaceConfig()]
)
firecracker.download()
jailer = firecracker.jailer()
jailer.download()
target_version = firecracker.base_name()[1:]
# If the version is smaller or equal to 1.0.0, we expect that
# MMDS will be initialised with V1 by default.
if compare_versions(target_version, "1.0.0") <= 0:
mmds_version = "V1"
else:
mmds_version = version
_validate_mmds_snapshot(
vm_instance,
vm_builder,
mmds_version,
target_fc_version=target_version,
fc_path=firecracker.local_path(),
jailer_path=jailer.local_path()
)
def create(self,
disks,
ssh_key: Artifact,
snapshot_type: SnapshotType = SnapshotType.FULL,
target_version: str = None,
mem_file_name: str = "vm.mem",
snapshot_name: str = "vm.vmstate",
net_ifaces=None):
"""Create a Snapshot object from a microvm and artifacts."""
# Disable API timeout as the APIs for snapshot related procedures
# take longer.
self._microvm.api_session.untime()
snapshot_dir = self.create_snapshot_dir()
self._microvm.pause_to_snapshot(
mem_file_path="/snapshot/" + mem_file_name,
snapshot_path="/snapshot/" + snapshot_name,
diff=snapshot_type == SnapshotType.DIFF,
version=target_version)
# Create a copy of the ssh_key artifact.
ssh_key_copy = ssh_key.copy()
mem_path = os.path.join(snapshot_dir, mem_file_name)
vmstate_path = os.path.join(snapshot_dir, snapshot_name)
return Snapshot(
mem=mem_path,
vmstate=vmstate_path,
# TODO: To support more disks we need to figure out a
# simple and flexible way to store snapshot artifacts
# in S3. This should be done in a PR where we add tests
# that resume from S3 snapshot artifacts.
disks=disks,
net_ifaces=net_ifaces or [NetIfaceConfig()],
ssh_key=ssh_key_copy.local_path())
def test_mmds_older_snapshot(bin_cloner_path):
"""
Test MMDS behavior restoring older snapshots in the current version.
Ensures that the MMDS version is persisted or initialised with the default
if the FC version does not support this feature.
@type: functional
"""
vm_builder = MicrovmBuilder(bin_cloner_path)
# Validate restoring a past snapshot in the current version.
artifacts = ArtifactCollection(_test_images_s3_bucket())
# Fetch all firecracker binaries.
firecracker_artifacts = artifacts.firecrackers(
max_version=get_firecracker_version_from_toml())
for firecracker in firecracker_artifacts:
firecracker.download()
jailer = firecracker.jailer()
jailer.download()
net_iface = NetIfaceConfig()
vm_instance = vm_builder.build_vm_nano(
net_ifaces=[net_iface],
fc_binary=firecracker.local_path(),
jailer_binary=jailer.local_path()
)
fc_version = firecracker.base_name()[1:]
# If the version is smaller or equal to 1.0.0, we expect that
# MMDS will be initialised with V1 by default.
# Otherwise, we may configure V2.
if compare_versions(fc_version, "1.0.0") <= 0:
mmds_version = "V1"
else:
mmds_version = "V2"
# Check if we need to configure MMDS the old way, by
# setting `allow_mmds_requests`.
# If we do (for v0.25), reissue the network PUT api call.
if compare_versions(fc_version, "1.0.0") < 0:
basevm = vm_instance.vm
guest_mac = net_tools.mac_from_ip(net_iface.guest_ip)
response = basevm.network.put(
iface_id=net_iface.dev_name,
host_dev_name=net_iface.tap_name,
guest_mac=guest_mac,
allow_mmds_requests=True
)
assert basevm.api_session.is_status_no_content(
response.status_code)
_validate_mmds_snapshot(
vm_instance,
vm_builder,
mmds_version,
target_fc_version=fc_version
)
def test_create_with_newer_virtio_features(bin_cloner_path):
"""
Attempt to create a snapshot with newer virtio features.
@type: functional
"""
builder = MicrovmBuilder(bin_cloner_path)
test_microvm = builder.build_vm_nano().vm
test_microvm.start()
# Init a ssh connection in order to wait for the VM to boot. This way
# we can be sure that the block device was activated.
iface = NetIfaceConfig()
test_microvm.ssh_config['hostname'] = iface.guest_ip
_ssh_connection = net_tools.SSHConnection(test_microvm.ssh_config)
# Create directory and files for saving snapshot state and memory.
snapshot_builder = SnapshotBuilder(test_microvm)
_snapshot_dir = snapshot_builder.create_snapshot_dir()
# Pause microVM for snapshot.
response = test_microvm.vm.patch(state='Paused')
assert test_microvm.api_session.is_status_no_content(response.status_code)
# We try to create a snapshot to a target version < 0.26.0.
# This should fail because Fc versions < 0.26.0 don't support
# virtio notification suppression.
target_fc_versions = ["0.24.0", "0.25.0"]
if platform.machine() == "x86_64":
target_fc_versions.insert(0, "0.23.0")
for target_fc_version in target_fc_versions:
response = test_microvm.snapshot.create(
mem_file_path="/snapshot/vm.mem",
snapshot_path="/snapshot/vm.vmstate",
version=target_fc_version
)
assert test_microvm.api_session.is_status_bad_request(
response.status_code
)
assert "The virtio devices use a features that is incompatible " \
"with older versions of Firecracker: notification suppression" \
in response.text
# It should work when we target a version >= 0.26.0
response = test_microvm.snapshot.create(
mem_file_path="/snapshot/vm.mem",
snapshot_path="/snapshot/vm.vmstate",
version="0.26.0"
)
assert test_microvm.api_session.is_status_no_content(
response.status_code
)
def create(self,
disks,
ssh_key: Artifact,
snapshot_type: SnapshotType = SnapshotType.FULL,
target_version: str = None,
mem_file_name: str = "vm.mem",
snapshot_name: str = "vm.vmstate",
net_ifaces=None,
use_ramdisk=False):
"""Create a Snapshot object from a microvm and artifacts."""
if use_ramdisk:
snaps_dir = self._microvm.jailer.chroot_ramfs_path()
mem_full_path = os.path.join(snaps_dir, mem_file_name)
vmstate_full_path = os.path.join(snaps_dir, snapshot_name)
memsize = self._microvm.machine_cfg.configuration['mem_size_mib']
# Pre-allocate ram for memfile to eliminate allocation variability.
utils.run_cmd('dd if=/dev/zero of={} bs=1M count={}'.format(
mem_full_path, memsize
))
cmd = 'chown {}:{} {}'.format(
self._microvm.jailer.uid,
self._microvm.jailer.gid,
mem_full_path
)
utils.run_cmd(cmd)
else:
snaps_dir = self.create_snapshot_dir()
mem_full_path = os.path.join(snaps_dir, mem_file_name)
vmstate_full_path = os.path.join(snaps_dir, snapshot_name)
snaps_dir_name = os.path.basename(snaps_dir)
self._microvm.pause_to_snapshot(
mem_file_path=os.path.join('/', snaps_dir_name, mem_file_name),
snapshot_path=os.path.join('/', snaps_dir_name, snapshot_name),
diff=snapshot_type == SnapshotType.DIFF,
version=target_version)
# Create a copy of the ssh_key artifact.
ssh_key_copy = ssh_key.copy()
return Snapshot(mem=mem_full_path,
vmstate=vmstate_full_path,
# TODO: To support more disks we need to figure out a
# simple and flexible way to store snapshot artifacts
# in S3. This should be done in a PR where we add tests
# that resume from S3 snapshot artifacts.
disks=disks,
net_ifaces=net_ifaces or [NetIfaceConfig()],
ssh_key=ssh_key_copy.local_path())
def _configure_network_interface(test_microvm):
"""
Create tap interface before spawning the microVM.
The network namespace and tap interface have to be created
beforehand when starting the microVM from a config file.
"""
# Create network namespace.
utils.run_cmd(f"ip netns add {test_microvm.jailer.netns}")
# Create tap device and SSH config.
net_iface = NetIfaceConfig()
_tap = test_microvm.create_tap_and_ssh_config(net_iface.host_ip,
net_iface.guest_ip,
net_iface.netmask,
net_iface.tap_name)
DEBUG = False
TEST_ID = "snapshot_restore_performance"
BASE_VCPU_COUNT = 1
BASE_MEM_SIZE_MIB = 128
BASE_NET_COUNT = 1
BASE_BLOCK_COUNT = 1
USEC_IN_MSEC = 1000
# Measurements tags.
RESTORE_LATENCY = "restore_latency"
CONFIG = json.load(open(defs.CFG_LOCATION / "snap_restore_test_config.json"))
# Define 4 net device configurations.
net_ifaces = [
NetIfaceConfig(),
NetIfaceConfig(host_ip="192.168.1.1",
guest_ip="192.168.1.2",
tap_name="tap1",
dev_name="eth1"),
NetIfaceConfig(host_ip="192.168.2.1",
guest_ip="192.168.2.2",
tap_name="tap2",
dev_name="eth2"),
NetIfaceConfig(host_ip="192.168.3.1",
guest_ip="192.168.3.2",
tap_name="tap3",
dev_name="eth3")
]
# We are using this as a global variable in order to only
def build(self,
kernel: Artifact,
disks: [DiskArtifact],
ssh_key: Artifact,
config: Artifact,
net_ifaces=None,
enable_diff_snapshots=False,
cpu_template=None,
use_ramdisk=False):
"""Build a fresh microvm."""
vm = init_microvm(self.root_path, self.bin_cloner_path,
self._fc_binary, self._jailer_binary)
# Start firecracker.
vm.spawn(use_ramdisk=use_ramdisk)
# Link the microvm to kernel, rootfs, ssh_key artifacts.
vm.kernel_file = kernel.local_path()
vm.rootfs_file = disks[0].local_path()
# copy rootfs to ramdisk if needed
jailed_rootfs_path = vm.copy_to_jail_ramfs(vm.rootfs_file) if \
use_ramdisk else vm.create_jailed_resource(vm.rootfs_file)
# Download ssh key into the microvm root.
ssh_key.download(self.root_path)
vm.ssh_config['ssh_key_path'] = ssh_key.local_path()
os.chmod(vm.ssh_config['ssh_key_path'], 0o400)
# Provide a default network configuration.
if net_ifaces is None or len(net_ifaces) == 0:
ifaces = [NetIfaceConfig()]
else:
ifaces = net_ifaces
# Configure network interfaces using artifacts.
for iface in ifaces:
vm.create_tap_and_ssh_config(host_ip=iface.host_ip,
guest_ip=iface.guest_ip,
netmask_len=iface.netmask,
tapname=iface.tap_name)
guest_mac = net_tools.mac_from_ip(iface.guest_ip)
response = vm.network.put(
iface_id=iface.dev_name,
host_dev_name=iface.tap_name,
guest_mac=guest_mac,
allow_mmds_requests=True,
)
assert vm.api_session.is_status_no_content(response.status_code)
with open(config.local_path()) as microvm_config_file:
microvm_config = json.load(microvm_config_file)
response = vm.basic_config(
add_root_device=False,
boot_args='console=ttyS0 reboot=k panic=1'
)
# Add the root file system with rw permissions.
response = vm.drive.put(
drive_id='rootfs',
path_on_host=jailed_rootfs_path,
is_root_device=True,
is_read_only=False
)
assert vm.api_session \
.is_status_no_content(response.status_code), \
response.text
# Apply the microvm artifact configuration and template.
response = vm.machine_cfg.put(
vcpu_count=int(microvm_config['vcpu_count']),
mem_size_mib=int(microvm_config['mem_size_mib']),
ht_enabled=bool(microvm_config['ht_enabled']),
track_dirty_pages=enable_diff_snapshots,
cpu_template=cpu_template,
)
assert vm.api_session.is_status_no_content(response.status_code)
vm.vcpus_count = int(microvm_config['vcpu_count'])
# Reset root path so next microvm is built some place else.
self.init_root_path()
return vm
def test_mmds_snapshot(bin_cloner_path):
"""
Exercise MMDS behavior with snapshots.
Ensures that MMDS V2 behavior is not affected by taking a snapshot
and that MMDS V2 is not available after snapshot load.
@type: functional
"""
vm_builder = MicrovmBuilder(bin_cloner_path)
net_iface = NetIfaceConfig()
vm_instance = vm_builder.build_vm_nano(
net_ifaces=[net_iface],
diff_snapshots=True
)
test_microvm = vm_instance.vm
root_disk = vm_instance.disks[0]
ssh_key = vm_instance.ssh_key
ipv4_address = '169.254.169.250'
# Configure MMDS version with custom IPv4 address.
_configure_mmds(
test_microvm,
version='V2',
iface_id=DEFAULT_DEV_NAME,
ipv4_address=ipv4_address
)
data_store = {
'latest': {
'meta-data': {
'ami-id': 'ami-12345678'
}
}
}
_populate_data_store(test_microvm, data_store)
test_microvm.start()
snapshot_builder = SnapshotBuilder(test_microvm)
disks = [root_disk.local_path()]
ssh_connection = net_tools.SSHConnection(test_microvm.ssh_config)
cmd = 'ip route add {} dev eth0'.format(ipv4_address)
_, stdout, stderr = ssh_connection.execute_command(cmd)
_assert_out(stdout, stderr, '')
# Generate token.
token = generate_mmds_session_token(
ssh_connection,
ipv4_address=ipv4_address,
token_ttl=60
)
pre = 'curl -m 2 -s'
pre += ' -X GET'
pre += ' -H "X-metadata-token: {}"'.format(token)
pre += ' http://{}/'.format(ipv4_address)
# Fetch metadata.
cmd = pre + 'latest/meta-data/'
_, stdout, stderr = ssh_connection.execute_command(cmd)
_assert_out(stdout, stderr, "ami-id")
# Create diff snapshot.
snapshot = snapshot_builder.create(disks,
ssh_key,
SnapshotType.DIFF)
# Resume microVM and ensure session token is still valid on the base.
response = test_microvm.vm.patch(state='Resumed')
assert test_microvm.api_session.is_status_no_content(response.status_code)
_, stdout, stderr = ssh_connection.execute_command(
pre + 'latest/meta-data/'
)
_assert_out(stdout, stderr, "ami-id")
# Kill base microVM.
test_microvm.kill()
# Load microVM clone from snapshot.
test_microvm, _ = vm_builder.build_from_snapshot(snapshot,
resume=True,
diff_snapshots=True)
_populate_data_store(test_microvm, data_store)
ssh_connection = net_tools.SSHConnection(test_microvm.ssh_config)
# Mmds V2 is not available with snapshots.
# Test that `PUT` requests are not allowed.
cmd = 'curl -m 2 -s'
cmd += ' -X PUT'
cmd += ' -H "X-metadata-token-ttl-seconds: 1"'
cmd += ' http://{}/latest/api/token'.format(ipv4_address)
_, stdout, stderr = ssh_connection.execute_command(cmd)
expected = "Not allowed HTTP method."
_assert_out(stdout, stderr, expected)
# Fetch metadata using V1 requests and ensure IPv4 configuration
# is persistent between snapshots.
cmd = 'curl -s http://{}/latest/meta-data/ami-id/'.format(ipv4_address)
_, stdout, stderr = ssh_connection.execute_command(cmd)
_assert_out(stdout, stderr, 'ami-12345678')
