def _kill_cgroup_tasks(self, controller):
"""Simulate wait on pid.
Read the tasks file and stay there until /proc/{pid}
disappears. The retry function that calls this code makes
sure we do not timeout.
"""
# pylint: disable=subprocess-run-check
tasks_file = '/sys/fs/cgroup/{}/{}/{}/tasks'.format(
controller, FC_BINARY_NAME, self.jailer_id)
# If tests do not call start on machines, the cgroups will not be
# created.
if not os.path.exists(tasks_file):
return True
cmd = 'cat {}'.format(tasks_file)
result = utils.run_cmd(cmd)
tasks_split = result.stdout.splitlines()
for task in tasks_split:
if os.path.exists("/proc/{}".format(task)):
raise TimeoutError
return True
def test_config_with_default_limit(test_microvm_with_api, vm_config_file):
"""
Test for request payload limit.
@type: functional
"""
test_microvm = test_microvm_with_api
_configure_vm_from_json(test_microvm, vm_config_file)
test_microvm.spawn()
response = test_microvm.machine_cfg.get()
assert test_microvm.api_session.is_status_ok(response.status_code)
assert test_microvm.state == "Running"
data_store = {"latest": {"meta-data": {}}}
data_store["latest"]["meta-data"]["ami-id"] = "abc"
response = test_microvm.mmds.put(json=data_store)
assert test_microvm.api_session.is_status_no_content(response.status_code)
cmd_err = "curl --unix-socket {} -i".format(test_microvm.api_socket)
cmd_err += ' -X PUT "http://localhost/mmds/config"'
cmd_err += ' -H "Content-Length: 51201"'
cmd_err += ' -H "Accept: application/json"'
cmd_err += ' -d "some body"'
response_err = "HTTP/1.1 400 \r\n"
response_err += "Server: Firecracker API\r\n"
response_err += "Connection: keep-alive\r\n"
response_err += "Content-Type: application/json\r\n"
response_err += "Content-Length: 149\r\n\r\n"
response_err += '{ "error": "Request payload with size 51201 is larger '
response_err += "than the limit of 51200 allowed by server.\n"
response_err += 'All previous unanswered requests will be dropped." }'
_, stdout, _stderr = utils.run_cmd(cmd_err)
assert stdout.encode("utf-8") == response_err.encode("utf-8")
def run_seccompiler(bpf_path, json_path=defs.SECCOMP_JSON_DIR, basic=False):
"""
Run seccompiler.
:param bpf_path: path to the seccompiler output file
:param json_path: optional path to json file
"""
cargo_target = '{}-unknown-linux-musl'.format(platform.machine())
# If no custom json filter, use the default one for the current target.
if json_path == defs.SECCOMP_JSON_DIR:
json_path = json_path / "{}.json".format(cargo_target)
cmd = 'cargo run -p seccompiler --target-dir {} --target {} --\
--input-file {} --target-arch {} --output-file {}'.format(
defs.SECCOMPILER_TARGET_DIR, cargo_target, json_path,
platform.machine(), bpf_path)
if basic:
cmd += ' --basic'
rc, _, _ = utils.run_cmd(cmd)
assert rc == 0
def kill(self):
"""All clean up associated with this microVM should go here."""
# pylint: disable=subprocess-run-check
if self.logging_thread is not None:
self.logging_thread.stop()
if self.expect_kill_by_signal is False and \
"Shutting down VM after intercepting signal" in self.log_data:
# Too late to assert at this point, pytest will still report the
# test as passed. BUT we can dump full logs for debugging,
# as well as an intentional eye-sore in the test report.
LOG.error(self.log_data)
if self._jailer.daemonize:
if self.jailer_clone_pid:
utils.run_cmd(
'kill -9 {}'.format(self.jailer_clone_pid),
ignore_return_code=True)
else:
# Killing screen will send SIGHUP to underlying Firecracker.
# Needed to avoid false positives in case kill() is called again.
self.expect_kill_by_signal = True
utils.run_cmd(
'kill -9 {} || true'.format(self.screen_pid))
# Check if Firecracker was launched by the jailer in a new pid ns.
fc_pid_in_new_ns = self.pid_in_new_ns
if fc_pid_in_new_ns:
# We need to explicitly kill the Firecracker pid, since it's
# different from the jailer pid that was previously killed.
utils.run_cmd(f'kill -9 {fc_pid_in_new_ns}',
ignore_return_code=True)
if self._memory_monitor and self._memory_monitor.is_alive():
self._memory_monitor.signal_stop()
self._memory_monitor.join(timeout=1)
self._memory_monitor.check_samples()
if self._cpu_load_monitor:
self._cpu_load_monitor.signal_stop()
self._cpu_load_monitor.join()
self._cpu_load_monitor.check_samples()
def __init__(self, name, netns, ip=None):
"""Set up the name and network namespace for this tap interface.
It also creates a new tap device, and brings it up. The tap will
stay on the host as long as the object obtained by instantiating this
class will be in scope. Once it goes out of scope, its destructor will
get called and the tap interface will get removed.
The function also moves the interface to the specified
namespace.
"""
utils.run_cmd('ip tuntap add mode tap name ' + name)
utils.run_cmd('ip link set {} netns {}'.format(name, netns))
if ip:
utils.run_cmd('ip netns exec {} ifconfig {} {} up'.format(
netns,
name,
ip
))
self._name = name
self._netns = netns
def test_device_ordering(test_microvm_with_api, network_config):
"""
Verify device ordering.
The root device should correspond to /dev/vda in the guest and
the order of the other devices should match their configuration order.
@type: functional
"""
test_microvm = test_microvm_with_api
test_microvm.spawn()
# Add first scratch block device.
fs1 = drive_tools.FilesystemFile(
os.path.join(test_microvm.fsfiles, 'scratch1'),
size=128
)
test_microvm.add_drive(
'scratch1',
fs1.path
)
# Set up the microVM with 1 vCPUs, 256 MiB of RAM, 0 network ifaces,
# a read-write root file system (this is the second block device added).
# The network interface is added after we get a unique MAC and IP.
test_microvm.basic_config()
# Add the third block device.
fs2 = drive_tools.FilesystemFile(
os.path.join(test_microvm.fsfiles, 'scratch2'),
size=512
)
test_microvm.add_drive(
'scratch2',
fs2.path
)
_tap, _, _ = test_microvm_with_api.ssh_network_config(network_config, '1')
test_microvm.start()
# Determine the size of the microVM rootfs in bytes.
try:
result = utils.run_cmd(
'du --apparent-size --block-size=1 {}'
.format(test_microvm.rootfs_file),
)
except ChildProcessError:
pytest.skip('Failed to get microVM rootfs size: {}'
.format(result.stderr))
assert len(result.stdout.split()) == 2
rootfs_size = result.stdout.split('\t')[0]
# The devices were added in this order: fs1, rootfs, fs2.
# However, the rootfs is the root device and goes first,
# so we expect to see this order: rootfs, fs1, fs2.
# The devices are identified by their size.
ssh_connection = net_tools.SSHConnection(test_microvm.ssh_config)
_check_block_size(ssh_connection, '/dev/vda', rootfs_size)
_check_block_size(ssh_connection, '/dev/vdb', fs1.size())
_check_block_size(ssh_connection, '/dev/vdc', fs2.size())
def spawn(self, create_logger=True, log_file='log_fifo', log_level='Info'):
"""Start a microVM as a daemon or in a screen session."""
# pylint: disable=subprocess-run-check
self._jailer.setup()
self._api_socket = self._jailer.api_socket_path()
self._api_session = Session()
self.actions = Actions(self._api_socket, self._api_session)
self.balloon = Balloon(self._api_socket, self._api_session)
self.boot = BootSource(self._api_socket, self._api_session)
self.desc_inst = DescribeInstance(self._api_socket, self._api_session)
self.drive = Drive(self._api_socket, self._api_session)
self.logger = Logger(self._api_socket, self._api_session)
self.machine_cfg = MachineConfigure(
self._api_socket,
self._api_session
)
self.metrics = Metrics(self._api_socket, self._api_session)
self.mmds = MMDS(self._api_socket, self._api_session)
self.network = Network(self._api_socket, self._api_session)
self.vm = Vm(self._api_socket, self._api_session)
self.vsock = Vsock(self._api_socket, self._api_session)
self.init_snapshot_api()
if create_logger:
log_fifo_path = os.path.join(self.path, log_file)
log_fifo = log_tools.Fifo(log_fifo_path)
self.create_jailed_resource(log_fifo.path, create_jail=True)
# The default value for `level`, when configuring the
# logger via cmd line, is `Warning`. We set the level
# to `Info` to also have the boot time printed in fifo.
self.jailer.extra_args.update({'log-path': log_file,
'level': log_level})
self.start_console_logger(log_fifo)
jailer_param_list = self._jailer.construct_param_list()
# When the daemonize flag is on, we want to clone-exec into the
# jailer rather than executing it via spawning a shell. Going
# forward, we'll probably switch to this method for running
# Firecracker in general, because it represents the way it's meant
# to be run by customers (together with CLONE_NEWPID flag).
#
# We have to use an external tool for CLONE_NEWPID, because
# 1) Python doesn't provide a os.clone() interface, and
# 2) Python's ctypes libc interface appears to be broken, causing
# our clone / exec to deadlock at some point.
if self._jailer.daemonize:
self.daemonize_jailer(jailer_param_list)
else:
# Delete old screen log if any.
try:
os.unlink(self.SCREEN_LOGFILE)
except OSError:
pass
# Log screen output to SCREEN_LOGFILE
# This file will collect any output from 'screen'ed Firecracker.
start_cmd = 'screen -L -Logfile {logfile} '\
'-dmS {session} {binary} {params}'.format(
logfile=self.SCREEN_LOGFILE,
session=self._session_name,
binary=self._jailer_binary_path,
params=' '.join(jailer_param_list))
utils.run_cmd(start_cmd)
# Build a regex object to match (number).session_name
regex_object = re.compile(
r'([0-9]+)\.{}'.format(self._session_name))
# Run 'screen -ls' in a retry_call loop, 30 times with a one
# second delay between calls.
# If the output of 'screen -ls' matches the regex object, it will
# return the PID. Otherwise a RuntimeError will be raised.
screen_pid = retry_call(
utils.search_output_from_cmd,
fkwargs={
"cmd": 'screen -ls',
"find_regex": regex_object
},
exceptions=RuntimeError,
tries=30,
delay=1).group(1)
self.jailer_clone_pid = int(open('/proc/{0}/task/{0}/children'
.format(screen_pid)
).read().strip())
# Configure screen to flush stdout to file.
flush_cmd = 'screen -S {session} -X colon "logfile flush 0^M"'
utils.run_cmd(flush_cmd.format(session=self._session_name))
# Wait for the jailer to create resources needed, and Firecracker to
# create its API socket.
# We expect the jailer to start within 80 ms. However, we wait for
# 1 sec since we are rechecking the existence of the socket 5 times
# and leave 0.2 delay between them.
if 'no-api' not in self._jailer.extra_args:
self._wait_create()
if create_logger:
self.check_log_message("Running Firecracker")
def test_reboot(test_microvm_with_api, network_config):
"""
Test reboot from guest.
@type: functional
"""
vm = test_microvm_with_api
vm.jailer.daemonize = False
vm.spawn()
# We don't need to monitor the memory for this test because we are
# just rebooting and the process dies before pmap gets the RSS.
vm.memory_monitor = None
# Set up the microVM with 4 vCPUs, 256 MiB of RAM, 0 network ifaces, and
# a root file system with the rw permission. The network interfaces is
# added after we get a unique MAC and IP.
vm.basic_config(vcpu_count=4)
_tap, _, _ = vm.ssh_network_config(network_config, '1')
# Configure metrics system.
metrics_fifo_path = os.path.join(vm.path, 'metrics_fifo')
metrics_fifo = log_tools.Fifo(metrics_fifo_path)
response = vm.metrics.put(
metrics_path=vm.create_jailed_resource(metrics_fifo.path)
)
assert vm.api_session.is_status_no_content(response.status_code)
vm.start()
# Get Firecracker PID so we can count the number of threads.
firecracker_pid = vm.jailer_clone_pid
# Get number of threads in Firecracker
cmd = 'ps -o nlwp {} | tail -1 | awk \'{{print $1}}\''.format(
firecracker_pid
)
_, stdout, _ = utils.run_cmd(cmd)
nr_of_threads = stdout.rstrip()
assert int(nr_of_threads) == 6
# Consume existing metrics
lines = metrics_fifo.sequential_reader(100)
assert len(lines) == 1
# Rebooting Firecracker sends an exit event and should gracefully kill.
# the instance.
ssh_connection = net_tools.SSHConnection(vm.ssh_config)
ssh_connection.execute_command("reboot")
while True:
# Pytest's timeout will kill the test even if the loop doesn't exit.
try:
os.kill(firecracker_pid, 0)
time.sleep(0.01)
except OSError:
break
# Consume existing metrics
lines = metrics_fifo.sequential_reader(100)
assert len(lines) == 1
if platform.machine() != "x86_64":
vm.check_log_message("Received KVM_SYSTEM_EVENT: type: 2, event: 0")
vm.check_log_message("Vmm is stopping.")
# Make sure that the FC process was not killed by a seccomp fault
assert json.loads(lines[0])["seccomp"]["num_faults"] == 0
def test_rust_clippy(target):
"""Fails if clippy generates any error, warnings are ignored."""
utils.run_cmd(
'cargo clippy --target {} --all --profile test'
' -- -D warnings'.format(target))
def test_describe_snapshot_all_versions(bin_cloner_path):
"""
Test `--describe-snapshot` correctness for all snapshot versions.
@type: functional
"""
logger = logging.getLogger("describe_snapshot")
builder = MicrovmBuilder(bin_cloner_path)
artifacts = ArtifactCollection(_test_images_s3_bucket())
# Fetch all firecracker binaries.
# For each binary create a snapshot and verify the data version
# of the snapshot state file.
firecracker_artifacts = artifacts.firecrackers(
max_version=get_firecracker_version_from_toml())
for firecracker in firecracker_artifacts:
firecracker.download()
jailer = firecracker.jailer()
jailer.download()
target_version = firecracker.base_name()[1:]
# Skip for aarch64, since the snapshotting feature
# was introduced in v0.24.0.
if platform.machine() == "aarch64" and "v0.23" in target_version:
continue
logger.info("Creating snapshot with Firecracker: %s",
firecracker.local_path())
logger.info("Using Jailer: %s", jailer.local_path())
logger.info("Using target version: %s", target_version)
# v0.23 does not support creating diff snapshots.
diff_snapshots = "0.23" not in target_version
vm_instance = builder.build_vm_nano(fc_binary=firecracker.local_path(),
jailer_binary=jailer.local_path(),
diff_snapshots=diff_snapshots)
vm = vm_instance.vm
vm.start()
# Create a snapshot builder from a microvm.
snapshot_builder = SnapshotBuilder(vm)
disks = [vm_instance.disks[0].local_path()]
# Version 0.24 and greater have Diff support.
snap_type = SnapshotType.DIFF if diff_snapshots else SnapshotType.FULL
snapshot = snapshot_builder.create(disks,
vm_instance.ssh_key,
target_version=target_version,
snapshot_type=snap_type)
logger.debug("========== Firecracker create snapshot log ==========")
logger.debug(vm.log_data)
vm.kill()
# Fetch Firecracker binary for the latest version
fc_binary, _ = get_firecracker_binaries()
# Verify the output of `--describe-snapshot` command line parameter
cmd = [fc_binary] + ["--describe-snapshot", snapshot.vmstate]
code, stdout, stderr = run_cmd(cmd)
assert code == 0
assert stderr == ''
assert target_version in stdout
def test_rust_clippy():
"""Fails if clippy generates any error, warnings are ignored."""
utils.run_cmd('cargo clippy --all --profile test -- -D warnings')
def _gcc_compile(src_file, output_file):
"""Build a source file with gcc."""
compile_cmd = 'gcc {} -o {} -static -O3'.format(src_file, output_file)
utils.run_cmd(compile_cmd)
def produce(self) -> Any:
"""Return output of the executed command."""
result = utils.run_cmd(self._cmd)
return result.stdout
def get_instance_type():
"""Get the instance type through IMDS."""
imds_cmd = "curl http://169.254.169.254/latest/meta-data/instance-type"
_, stdout, _ = run_cmd(imds_cmd)
return stdout
def resize(self, new_size):
"""Resize the filesystem."""
utils.run_cmd("truncate --size " + str(new_size) + "M " + self.path)
utils.run_cmd("resize2fs " + self.path)
def get(self):
"""Get the version of the current microvm, from the cmdline."""
_, stdout, _ = run_cmd("{} --version".format(self._fc_binary_path))
return re.match(r"^Firecracker v([0-9]+\.[0-9]+(\.[0-9]+)?)",
stdout.partition("\n")[0]).group(1)
def produce_iperf_output(basevm, guest_cmd_builder, current_avail_cpu, runtime,
omit, load_factor, modes):
"""Produce iperf raw output from server-client connection."""
# Check if we have enough CPUs to pin the servers on the host.
# The available CPUs are the total minus vcpus, vmm and API threads.
assert load_factor * basevm.vcpus_count < CpuMap.len() - \
basevm.vcpus_count - 2
host_uds_path = os.path.join(basevm.path, VSOCK_UDS_PATH)
# Start the servers.
for server_idx in range(load_factor * basevm.vcpus_count):
assigned_cpu = CpuMap(current_avail_cpu)
iperf_server = \
CmdBuilder(f"taskset --cpu-list {assigned_cpu}") \
.with_arg(IPERF3) \
.with_arg("-sD") \
.with_arg("--vsock") \
.with_arg("-B", host_uds_path) \
.with_arg("-p", f"{BASE_PORT + server_idx}") \
.with_arg("-1") \
.build()
run_cmd(iperf_server)
current_avail_cpu += 1
# Wait for iperf3 servers to start.
time.sleep(SERVER_STARTUP_TIME)
# Start `vcpus` iperf3 clients. We can not use iperf3 parallel streams
# due to non deterministic results and lack of scaling.
def spawn_iperf_client(conn, client_idx, mode):
# Add the port where the iperf3 client is going to send/receive.
cmd = guest_cmd_builder.with_arg("-p", BASE_PORT +
client_idx).with_arg(mode).build()
# Bind the UDS in the jailer's root.
basevm.create_jailed_resource(
os.path.join(
basevm.path,
_make_host_port_path(VSOCK_UDS_PATH, BASE_PORT + client_idx)))
pinned_cmd = f"taskset --cpu-list {client_idx % basevm.vcpus_count}" \
f" {cmd}"
rc, stdout, _ = conn.execute_command(pinned_cmd)
assert rc == 0
return stdout.read()
with concurrent.futures.ThreadPoolExecutor() as executor:
futures = list()
cpu_load_future = executor.submit(get_cpu_percent,
basevm.jailer_clone_pid,
runtime - SERVER_STARTUP_TIME, omit)
modes_len = len(modes)
ssh_connection = net_tools.SSHConnection(basevm.ssh_config)
for client_idx in range(load_factor * basevm.vcpus_count):
futures.append(
executor.submit(
spawn_iperf_client,
ssh_connection,
client_idx,
# Distribute the modes evenly.
modes[client_idx % modes_len]))
cpu_load = cpu_load_future.result()
for future in futures[:-1]:
res = json.loads(future.result())
res[IPERF3_END_RESULTS_TAG][
IPERF3_CPU_UTILIZATION_PERCENT_OUT_TAG] = None
yield res
# Attach the real CPU utilization vmm/vcpus to
# the last iperf3 server-client pair measurements.
res = json.loads(futures[-1].result())
# We expect a single emulation thread tagged with `firecracker` name.
tag = "firecracker"
assert tag in cpu_load and len(cpu_load[tag]) == 1
thread_id = list(cpu_load[tag])[0]
data = cpu_load[tag][thread_id]
vmm_util = sum(data) / len(data)
cpu_util_perc = res[IPERF3_END_RESULTS_TAG][
IPERF3_CPU_UTILIZATION_PERCENT_OUT_TAG] = dict()
cpu_util_perc[CPU_UTILIZATION_VMM_TAG] = vmm_util
vcpus_util = 0
for vcpu in range(basevm.vcpus_count):
# We expect a single fc_vcpu thread tagged with
# f`fc_vcpu {vcpu}`.
tag = f"fc_vcpu {vcpu}"
assert tag in cpu_load and len(cpu_load[tag]) == 1
thread_id = list(cpu_load[tag])[0]
data = cpu_load[tag][thread_id]
vcpus_util += (sum(data) / len(data))
cpu_util_perc[CPU_UTILIZATION_VCPUS_TOTAL_TAG] = vcpus_util
yield res
def resize(self, new_size):
"""Resize the filesystem."""
utils.run_cmd('truncate --size ' + str(new_size) + 'M ' + self.path)
utils.run_cmd('resize2fs ' + self.path)
def get_cpu_model_name():
"""Return the CPU model name."""
_, stdout, _ = run_cmd("cat /proc/cpuinfo | grep 'model name' | uniq")
info = stdout.strip().split(sep=":")
assert len(info) == 2
return info[1].strip()
def get_rss_from_pmap():
_, output, _ = run_cmd("pmap -X {}".format(pid))
return int(output.split('\n')[-2].split()[1], 10)
def _gcc_compile(src_file, output_file, extra_flags="-static -O3"):
"""Build a source file with gcc."""
compile_cmd = 'gcc {} -o {} {}'.format(src_file, output_file, extra_flags)
utils.run_cmd(compile_cmd)
def _run_local_iperf(iperf_cmd):
"""Execute a client related iperf command locally."""
process = utils.run_cmd(iperf_cmd)
return process.stdout
def test_coverage(test_session_root_path, test_session_tmp_path):
"""Test line coverage with kcov.
The result is extracted from the $KCOV_COVERAGE_FILE file created by kcov
after a coverage run.
"""
proc_model = [item for item in COVERAGE_DICT if item in PROC_MODEL]
assert len(proc_model) == 1, "Could not get processor model!"
coverage_target_pct = COVERAGE_DICT[proc_model[0]]
exclude_pattern = (
'${CARGO_HOME:-$HOME/.cargo/},'
'build/,'
'tests/,'
'usr/lib/gcc,'
'lib/x86_64-linux-gnu/,'
# The following files/directories are auto-generated
'bootparam.rs,'
'elf.rs,'
'mpspec.rs,'
'msr_index.rs,'
'_gen'
)
exclude_region = '\'mod tests {\''
cmd = (
'CARGO_TARGET_DIR={} cargo kcov --all '
'--output {} -- '
'--exclude-pattern={} '
'--exclude-region={} --verify'
).format(
os.path.join(test_session_root_path, CARGO_KCOV_REL_PATH),
test_session_tmp_path,
exclude_pattern,
exclude_region
)
# By default, `cargo kcov` passes `--exclude-pattern=$CARGO_HOME --verify`
# to kcov. To pass others arguments, we need to include the defaults.
utils.run_cmd(cmd)
coverage_file = os.path.join(test_session_tmp_path, KCOV_COVERAGE_FILE)
with open(coverage_file) as cov_output:
contents = cov_output.read()
covered_lines = int(re.findall(KCOV_COVERED_LINES_REGEX, contents)[0])
total_lines = int(re.findall(KCOV_TOTAL_LINES_REGEX, contents)[0])
coverage = covered_lines / total_lines * 100
print("Number of executable lines: {}".format(total_lines))
print("Number of covered lines: {}".format(covered_lines))
print("Thus, coverage is: {:.2f}%".format(coverage))
coverage_low_msg = (
'Current code coverage ({:.2f}%) is below the target ({}%).'
.format(coverage, coverage_target_pct)
)
min_coverage = coverage_target_pct - COVERAGE_MAX_DELTA
assert coverage >= min_coverage, coverage_low_msg
# Get the name of the variable that needs updating.
namespace = globals()
cov_target_name = [name for name in namespace if namespace[name]
is COVERAGE_DICT][0]
coverage_high_msg = (
'Current code coverage ({:.2f}%) is above the target ({}%).\n'
'Please update the value of {}.'
.format(coverage, coverage_target_pct, cov_target_name)
)
assert coverage - coverage_target_pct <= COVERAGE_MAX_DELTA,\
coverage_high_msg
def test_coverage(test_fc_session_root_path, test_session_tmp_path):
"""Test line coverage for rust tests is within bounds.
The result is extracted from the $KCOV_COVERAGE_FILE file created by kcov
after a coverage run.
@type: build
"""
proc_model = [item for item in COVERAGE_DICT if item in PROC_MODEL]
assert len(proc_model) == 1, "Could not get processor model!"
coverage_target_pct = COVERAGE_DICT[proc_model[0]]
exclude_pattern = (
'${CARGO_HOME:-$HOME/.cargo/},'
'build/,'
'tests/,'
'usr/lib/gcc,'
'lib/x86_64-linux-gnu/,'
'test_utils.rs,'
# The following files/directories are auto-generated
'bootparam.rs,'
'elf.rs,'
'mpspec.rs,'
'msr_index.rs,'
'bindings.rs,'
'_gen')
exclude_region = '\'mod tests {\''
target = "{}-unknown-linux-musl".format(platform.machine())
cmd = ('RUSTFLAGS="{}" CARGO_TARGET_DIR={} cargo kcov --all '
'--target {} --output {} -- '
'--exclude-pattern={} '
'--exclude-region={} --verify').format(
host.get_rustflags(),
os.path.join(test_fc_session_root_path, CARGO_KCOV_REL_PATH),
target, test_session_tmp_path, exclude_pattern, exclude_region)
# We remove the seccompiler custom build directory, created by the
# vmm-level `build.rs`.
# If we don't delete it before and after running the kcov command, we will
# run into linker errors.
shutil.rmtree(SECCOMPILER_BUILD_DIR, ignore_errors=True)
# By default, `cargo kcov` passes `--exclude-pattern=$CARGO_HOME --verify`
# to kcov. To pass others arguments, we need to include the defaults.
utils.run_cmd(cmd)
shutil.rmtree(SECCOMPILER_BUILD_DIR)
coverage_file = os.path.join(test_session_tmp_path, KCOV_COVERAGE_FILE)
with open(coverage_file, encoding='utf-8') as cov_output:
contents = cov_output.read()
covered_lines = int(re.findall(KCOV_COVERED_LINES_REGEX, contents)[0])
total_lines = int(re.findall(KCOV_TOTAL_LINES_REGEX, contents)[0])
coverage = covered_lines / total_lines * 100
print("Number of executable lines: {}".format(total_lines))
print("Number of covered lines: {}".format(covered_lines))
print("Thus, coverage is: {:.2f}%".format(coverage))
coverage_low_msg = (
'Current code coverage ({:.2f}%) is below the target ({}%).'.format(
coverage, coverage_target_pct))
min_coverage = coverage_target_pct - COVERAGE_MAX_DELTA
assert coverage >= min_coverage, coverage_low_msg
# Get the name of the variable that needs updating.
namespace = globals()
cov_target_name = [
name for name in namespace if namespace[name] is COVERAGE_DICT
][0]
coverage_high_msg = (
'Current code coverage ({:.2f}%) is above the target ({}%).\n'
'Please update the value of {}.'.format(coverage, coverage_target_pct,
cov_target_name))
assert coverage - coverage_target_pct <= COVERAGE_MAX_DELTA,\
coverage_high_msg
return f"{coverage}%", \
f"{coverage_target_pct}% +/- {COVERAGE_MAX_DELTA * 100}%"
def produce_iperf_output(basevm,
guest_cmd_builder,
current_avail_cpu,
runtime,
omit,
load_factor,
modes):
"""Produce iperf raw output from server-client connection."""
# Check if we have enough CPUs to pin the servers on the host.
# The available CPUs are the total minus vcpus, vmm and API threads.
assert load_factor * basevm.vcpus_count < CpuMap.len() - \
basevm.vcpus_count - 2
# Start the servers.
for server_idx in range(load_factor*basevm.vcpus_count):
assigned_cpu = CpuMap(current_avail_cpu)
iperf_server = \
CmdBuilder(f"taskset --cpu-list {assigned_cpu}") \
.with_arg(basevm.jailer.netns_cmd_prefix()) \
.with_arg(IPERF3) \
.with_arg("-sD") \
.with_arg("-p", f"{BASE_PORT + server_idx}") \
.with_arg("-1") \
.build()
run_cmd(iperf_server)
current_avail_cpu += 1
# Wait for iperf3 server to start.
time.sleep(2)
# Start `vcpus` iperf3 clients. We can not use iperf3 parallel streams
# due to non deterministic results and lack of scaling.
def spawn_iperf_client(conn, client_idx, mode):
# Add the port where the iperf3 client is going to send/receive.
cmd = guest_cmd_builder \
.with_arg("-p", f"{BASE_PORT + client_idx}") \
.with_arg(mode) \
.build()
pinned_cmd = f"taskset --cpu-list {client_idx % basevm.vcpus_count}" \
f" {cmd}"
_, stdout, _ = conn.execute_command(pinned_cmd)
return stdout.read()
# Remove inaccurate readings from the workloads end.
cpu_load_runtime = runtime - 2
with concurrent.futures.ThreadPoolExecutor() as executor:
futures = list()
cpu_load_future = executor.submit(get_cpu_percent,
basevm.jailer_clone_pid,
cpu_load_runtime,
omit)
modes_len = len(modes)
ssh_connection = net_tools.SSHConnection(basevm.ssh_config)
for client_idx in range(load_factor*basevm.vcpus_count):
futures.append(executor.submit(spawn_iperf_client,
ssh_connection,
client_idx,
# Distribute the modes evenly.
modes[client_idx % modes_len]))
cpu_load = cpu_load_future.result()
for future in futures[:-1]:
res = json.loads(future.result())
res[IPERF3_END_RESULTS_TAG][
IPERF3_CPU_UTILIZATION_PERCENT_OUT_TAG] = None
yield res
# Attach the real CPU utilization vmm/vcpus to
# the last iperf3 server-client pair measurements.
res = json.loads(futures[-1].result())
# We expect a single emulation thread tagged with `firecracker` name.
tag = "firecracker"
assert tag in cpu_load and len(cpu_load[tag]) == 1
for thread_id in cpu_load[tag]:
data = cpu_load[tag][thread_id]
data_len = len(data)
assert data_len == cpu_load_runtime
vmm_util = sum(data)/data_len
cpu_util_perc = res[IPERF3_END_RESULTS_TAG][
IPERF3_CPU_UTILIZATION_PERCENT_OUT_TAG] = dict()
cpu_util_perc[CPU_UTILIZATION_VMM] = vmm_util
if DEBUG:
res[IPERF3_END_RESULTS_TAG][
DEBUG_CPU_UTILIZATION_VMM_SAMPLES_TAG] \
= data
vcpus_util = 0
for vcpu in range(basevm.vcpus_count):
# We expect a single fc_vcpu thread tagged with
# f`fc_vcpu {vcpu}`.
tag = f"fc_vcpu {vcpu}"
assert tag in cpu_load and len(cpu_load[tag]) == 1
for thread_id in cpu_load[tag]:
data = cpu_load[tag][thread_id]
data_len = len(data)
assert data_len == cpu_load_runtime
if DEBUG:
res[IPERF3_END_RESULTS_TAG][
f"cpu_utilization_fc_vcpu_{vcpu}_samples"] = data
vcpus_util += sum(data)/data_len
cpu_util_perc[CPU_UTILIZATION_VCPUS_TOTAL] = vcpus_util
yield res
def set_tx_queue_len(self, tx_queue_len):
"""Set the length of the tap's TX queue."""
utils.run_cmd("ip netns exec {} ip link set {} txqueuelen {}".format(
self.netns, self.name, tx_queue_len))
def test_advanced_seccomp(bin_seccomp_paths):
"""
Test seccompiler-bin with `demo_jailer`.
Test that the demo jailer (with advanced seccomp) allows the harmless demo
binary, denies the malicious demo binary and that an empty allowlist
denies everything.
@type: security
"""
# pylint: disable=redefined-outer-name
# pylint: disable=subprocess-run-check
# The fixture pattern causes a pylint false positive for that rule.
demo_jailer = bin_seccomp_paths["demo_jailer"]
demo_harmless = bin_seccomp_paths["demo_harmless"]
demo_malicious = bin_seccomp_paths["demo_malicious"]
assert os.path.exists(demo_jailer)
assert os.path.exists(demo_harmless)
assert os.path.exists(demo_malicious)
json_filter = """{{
"main": {{
"default_action": "trap",
"filter_action": "allow",
"filter": [
{},
{{
"syscall": "write",
"args": [
{{
"index": 0,
"type": "dword",
"op": "eq",
"val": 1,
"comment": "stdout fd"
}},
{{
"index": 2,
"type": "qword",
"op": "eq",
"val": 14,
"comment": "nr of bytes"
}}
]
}}
]
}}
}}""".format(_get_basic_syscall_list())
# Run seccompiler-bin.
bpf_path = _run_seccompiler_bin(json_filter)
# Run the mini jailer for harmless binary.
outcome = utils.run_cmd([demo_jailer, demo_harmless, bpf_path],
no_shell=True,
ignore_return_code=True)
# The demo harmless binary should have terminated gracefully.
assert outcome.returncode == 0
# Run the mini jailer for malicious binary.
outcome = utils.run_cmd([demo_jailer, demo_malicious, bpf_path],
no_shell=True,
ignore_return_code=True)
# The demo malicious binary should have received `SIGSYS`.
assert outcome.returncode == -31
os.unlink(bpf_path)
# Run seccompiler-bin with `--basic` flag.
bpf_path = _run_seccompiler_bin(json_filter, basic=True)
# Run the mini jailer for malicious binary.
outcome = utils.run_cmd([demo_jailer, demo_malicious, bpf_path],
no_shell=True,
ignore_return_code=True)
# The malicious binary also terminates gracefully, since the --basic option
# disables all argument checks.
assert outcome.returncode == 0
os.unlink(bpf_path)
# Run the mini jailer with an empty allowlist. It should trap on any
# syscall.
json_filter = """{
"main": {
"default_action": "trap",
"filter_action": "allow",
"filter": []
}
}"""
# Run seccompiler-bin.
bpf_path = _run_seccompiler_bin(json_filter)
outcome = utils.run_cmd([demo_jailer, demo_harmless, bpf_path],
no_shell=True,
ignore_return_code=True)
# The demo binary should have received `SIGSYS`.
assert outcome.returncode == -31
os.unlink(bpf_path)
def merge_memory_bitmaps(base, layer, block_size=4096):
"""Merge a sparse layer on top of base."""
dd_command = 'dd bs={} if={} of={} conv=sparse,notrunc'
dd_command = dd_command.format(block_size, layer, base)
_ = run_cmd(dd_command)
def serial_input(self, input_string):
"""Send a string to the Firecracker serial console via screen."""
input_cmd = 'screen -S {session} -p 0 -X stuff "{input_string}^M"'
utils.run_cmd(input_cmd.format(session=self._session_name,
input_string=input_string))
def run_fio(env_id, basevm, ssh_conn, mode, bs):
"""Run a fio test in the specified mode with block size bs."""
logs_path = f"{basevm.jailer.chroot_base_with_id()}/{env_id}/{mode}{bs}"
# Compute the fio command. Pin it to the first guest CPU.
cmd = CmdBuilder(FIO) \
.with_arg(f"--name={mode}-{bs}") \
.with_arg(f"--rw={mode}") \
.with_arg(f"--bs={bs}") \
.with_arg("--filename=/dev/vdb") \
.with_arg("--time_base=1") \
.with_arg(f"--size={CONFIG['block_device_size']}M") \
.with_arg("--direct=1") \
.with_arg("--ioengine=libaio") \
.with_arg("--iodepth=32") \
.with_arg(f"--ramp_time={CONFIG['omit']}") \
.with_arg(f"--numjobs={CONFIG['load_factor'] * basevm.vcpus_count}") \
.with_arg("--randrepeat=0") \
.with_arg(f"--runtime={CONFIG['time']}") \
.with_arg(f"--write_iops_log={mode}{bs}") \
.with_arg(f"--write_bw_log={mode}{bs}") \
.with_arg("--log_avg_msec=1000") \
.with_arg("--output-format=json+") \
.build()
rc, _, stderr = ssh_conn.execute_command(
"echo 'none' > /sys/block/vdb/queue/scheduler")
assert rc == 0, stderr.read()
assert stderr.read() == ""
# First, flush all guest cached data to host, then drop guest FS caches.
rc, _, stderr = ssh_conn.execute_command("sync")
assert rc == 0, stderr.read()
assert stderr.read() == ""
rc, _, stderr = ssh_conn.execute_command(
"echo 3 > /proc/sys/vm/drop_caches")
assert rc == 0, stderr.read()
assert stderr.read() == ""
# Then, flush all host cached data to hardware, also drop host FS caches.
run_cmd("sync")
run_cmd("echo 3 > /proc/sys/vm/drop_caches")
# Start the CPU load monitor.
with concurrent.futures.ThreadPoolExecutor() as executor:
cpu_load_future = executor.submit(get_cpu_percent,
basevm.jailer_clone_pid,
CONFIG["time"],
omit=CONFIG["omit"])
# Print the fio command in the log and run it
rc, _, stderr = ssh_conn.execute_command(cmd)
assert rc == 0, stderr.read()
assert stderr.read() == ""
if os.path.isdir(logs_path):
shutil.rmtree(logs_path)
os.makedirs(logs_path)
ssh_conn.scp_get_file("*.log", logs_path)
rc, _, stderr = ssh_conn.execute_command("rm *.log")
assert rc == 0, stderr.read()
result = dict()
cpu_load = cpu_load_future.result()
tag = "firecracker"
assert tag in cpu_load and len(cpu_load[tag]) == 1
data = list(cpu_load[tag].values())[0]
data_len = len(data)
assert data_len == CONFIG["time"]
result[CPU_UTILIZATION_VMM] = sum(data) / data_len
if DEBUG:
result[CPU_UTILIZATION_VMM_SAMPLES_TAG] = data
vcpus_util = 0
for vcpu in range(basevm.vcpus_count):
# We expect a single fc_vcpu thread tagged with
# f`fc_vcpu {vcpu}`.
tag = f"fc_vcpu {vcpu}"
assert tag in cpu_load and len(cpu_load[tag]) == 1
data = list(cpu_load[tag].values())[0]
data_len = len(data)
assert data_len == CONFIG["time"]
if DEBUG:
samples_tag = f"cpu_utilization_fc_vcpu_{vcpu}_samples"
result[samples_tag] = data
vcpus_util += sum(data) / data_len
result[CPU_UTILIZATION_VCPUS_TOTAL] = vcpus_util
return result
