def _g2h_send_ping(context):
"""Send ping from guest to host."""
logger = context.custom['logger']
vm_builder = context.custom['builder']
interval_between_req = context.custom['interval']
name = context.custom['name']
file_dumper = context.custom['results_file_dumper']
logger.info("Testing {} with microvm: \"{}\", kernel {}, disk {} ".format(
name, context.microvm.name(), context.kernel.name(),
context.disk.name()))
# Create a rw copy artifact.
rw_disk = context.disk.copy()
# Get ssh key from read-only artifact.
ssh_key = context.disk.ssh_key()
# Create a fresh microvm from aftifacts.
basevm = vm_builder.build(kernel=context.kernel,
disks=[rw_disk],
ssh_key=ssh_key,
config=context.microvm)
basevm.start()
# Check if the needed CPU cores are available. We have the API thread, VMM
# thread and then one thread for each configured vCPU.
assert CpuMap.len() >= 2 + basevm.vcpus_count
# Pin uVM threads to physical cores.
current_cpu_id = 0
assert basevm.pin_vmm(current_cpu_id), \
"Failed to pin firecracker thread."
current_cpu_id += 1
assert basevm.pin_api(current_cpu_id), \
"Failed to pin fc_api thread."
for i in range(basevm.vcpus_count):
current_cpu_id += 1
assert basevm.pin_vcpu(i, current_cpu_id + i), \
f"Failed to pin fc_vcpu {i} thread."
custom = {
"microvm": context.microvm.name(),
"kernel": context.kernel.name(),
"disk": context.disk.name(),
"cpu_model_name": get_cpu_model_name()
}
st_core = core.Core(name="network_latency", iterations=1, custom=custom)
cons = consumer.LambdaConsumer(
func=consume_ping_output,
func_kwargs={"requests": context.custom['requests']})
cmd = PING.format(context.custom['requests'], interval_between_req,
DEFAULT_HOST_IP)
prod = producer.SSHCommand(cmd, net_tools.SSHConnection(basevm.ssh_config))
st_core.add_pipe(producer=prod, consumer=cons, tag="ping")
# Gather results and verify pass criteria.
result = st_core.run_exercise(file_dumper is None)
if file_dumper:
file_dumper.writeln(json.dumps(result))
def iperf_workload(context):
"""Iperf between guest and host in both directions for TCP workload."""
vm_builder = context.custom['builder']
logger = context.custom["logger"]
file_dumper = context.custom['results_file_dumper']
# Create a rw copy artifact.
rw_disk = context.disk.copy()
# Get ssh key from read-only artifact.
ssh_key = context.disk.ssh_key()
# Create a fresh microvm from artifacts.
basevm = vm_builder.build(kernel=context.kernel,
disks=[rw_disk],
ssh_key=ssh_key,
config=context.microvm)
basevm.start()
custom = {
"microvm": context.microvm.name(),
"kernel": context.kernel.name(),
"disk": context.disk.name(),
"cpu_model_name": get_cpu_model_name()
}
st_core = core.Core(name="network_tcp_throughput",
iterations=1,
custom=custom)
# Check if the needed CPU cores are available. We have the API thread, VMM
# thread and then one thread for each configured vCPU.
assert CpuMap.len() >= 2 + basevm.vcpus_count
# Pin uVM threads to physical cores.
current_avail_cpu = 0
assert basevm.pin_vmm(current_avail_cpu), \
"Failed to pin firecracker thread."
current_avail_cpu += 1
assert basevm.pin_api(current_avail_cpu), \
"Failed to pin fc_api thread."
for i in range(basevm.vcpus_count):
current_avail_cpu += 1
assert basevm.pin_vcpu(i, current_avail_cpu), \
f"Failed to pin fc_vcpu {i} thread."
logger.info("Testing with microvm: \"{}\", kernel {}, disk {}"
.format(context.microvm.name(),
context.kernel.name(),
context.disk.name()))
for cons, prod, tag in \
pipes(basevm,
DEFAULT_HOST_IP,
current_avail_cpu + 1,
f"{context.kernel.name()}/{context.disk.name()}"):
st_core.add_pipe(prod, cons, tag)
# Start running the commands on guest, gather results and verify pass
# criteria.
results = st_core.run_exercise(check_criteria=file_dumper is None)
if file_dumper:
file_dumper.writeln(json.dumps(results))
def iperf_workload(context):
"""Run a statistic exercise."""
vm_builder = context.custom['builder']
logger = context.custom["logger"]
file_dumper = context.custom['results_file_dumper']
# Create a rw copy artifact.
rw_disk = context.disk.copy()
# Get ssh key from read-only artifact.
ssh_key = context.disk.ssh_key()
# Create a fresh microvm from artifacts.
basevm = vm_builder.build(kernel=context.kernel,
disks=[rw_disk],
ssh_key=ssh_key,
config=context.microvm)
# Create a vsock device
basevm.vsock.put(vsock_id="vsock0",
guest_cid=3,
uds_path="/" + VSOCK_UDS_PATH)
basevm.start()
st_core = core.Core(name="vsock_throughput",
iterations=1,
custom={'cpu_model_name': get_cpu_model_name()})
# Check if the needed CPU cores are available. We have the API thread, VMM
# thread and then one thread for each configured vCPU.
assert CpuMap.len() >= 2 + basevm.vcpus_count
# Pin uVM threads to physical cores.
current_avail_cpu = 0
assert basevm.pin_vmm(current_avail_cpu), \
"Failed to pin firecracker thread."
current_avail_cpu += 1
assert basevm.pin_api(current_avail_cpu), \
"Failed to pin fc_api thread."
for i in range(basevm.vcpus_count):
current_avail_cpu += 1
assert basevm.pin_vcpu(i, current_avail_cpu), \
f"Failed to pin fc_vcpu {i} thread."
logger.info("Testing with microvm: \"{}\", kernel {}, disk {}".format(
context.microvm.name(), context.kernel.name(), context.disk.name()))
for cons, prod, tag in \
pipes(basevm,
current_avail_cpu + 1,
f"{context.kernel.name()}/{context.disk.name()}"):
st_core.add_pipe(prod, cons, tag)
# Start running the commands on guest, gather results and verify pass
# criteria.
results = st_core.run_exercise(file_dumper is None)
if file_dumper:
file_dumper.writeln(json.dumps(results))
basevm.kill()
def produce_iperf_output(basevm,
guest_cmd_builder,
current_avail_cpu,
runtime,
omit,
load_factor,
modes):
"""Produce iperf raw output from server-client connection."""
# Check if we have enough CPUs to pin the servers on the host.
# The available CPUs are the total minus vcpus, vmm and API threads.
assert load_factor * basevm.vcpus_count < CpuMap.len() - \
basevm.vcpus_count - 2
# Start the servers.
for server_idx in range(load_factor*basevm.vcpus_count):
assigned_cpu = CpuMap(current_avail_cpu)
iperf_server = \
CmdBuilder(f"taskset --cpu-list {assigned_cpu}") \
.with_arg(basevm.jailer.netns_cmd_prefix()) \
.with_arg(IPERF3) \
.with_arg("-sD") \
.with_arg("-p", f"{BASE_PORT + server_idx}") \
.with_arg("-1") \
.build()
run_cmd(iperf_server)
current_avail_cpu += 1
# Wait for iperf3 server to start.
time.sleep(2)
# Start `vcpus` iperf3 clients. We can not use iperf3 parallel streams
# due to non deterministic results and lack of scaling.
def spawn_iperf_client(conn, client_idx, mode):
# Add the port where the iperf3 client is going to send/receive.
cmd = guest_cmd_builder \
.with_arg("-p", f"{BASE_PORT + client_idx}") \
.with_arg(mode) \
.build()
pinned_cmd = f"taskset --cpu-list {client_idx % basevm.vcpus_count}" \
f" {cmd}"
_, stdout, _ = conn.execute_command(pinned_cmd)
return stdout.read()
# Remove inaccurate readings from the workloads end.
cpu_load_runtime = runtime - 2
with concurrent.futures.ThreadPoolExecutor() as executor:
futures = list()
cpu_load_future = executor.submit(get_cpu_percent,
basevm.jailer_clone_pid,
cpu_load_runtime,
omit)
modes_len = len(modes)
ssh_connection = net_tools.SSHConnection(basevm.ssh_config)
for client_idx in range(load_factor*basevm.vcpus_count):
futures.append(executor.submit(spawn_iperf_client,
ssh_connection,
client_idx,
# Distribute the modes evenly.
modes[client_idx % modes_len]))
cpu_load = cpu_load_future.result()
for future in futures[:-1]:
res = json.loads(future.result())
res[IPERF3_END_RESULTS_TAG][
IPERF3_CPU_UTILIZATION_PERCENT_OUT_TAG] = None
yield res
# Attach the real CPU utilization vmm/vcpus to
# the last iperf3 server-client pair measurements.
res = json.loads(futures[-1].result())
# We expect a single emulation thread tagged with `firecracker` name.
tag = "firecracker"
assert tag in cpu_load and len(cpu_load[tag]) == 1
for thread_id in cpu_load[tag]:
data = cpu_load[tag][thread_id]
data_len = len(data)
assert data_len == cpu_load_runtime
vmm_util = sum(data)/data_len
cpu_util_perc = res[IPERF3_END_RESULTS_TAG][
IPERF3_CPU_UTILIZATION_PERCENT_OUT_TAG] = dict()
cpu_util_perc[CPU_UTILIZATION_VMM] = vmm_util
if DEBUG:
res[IPERF3_END_RESULTS_TAG][
DEBUG_CPU_UTILIZATION_VMM_SAMPLES_TAG] \
= data
vcpus_util = 0
for vcpu in range(basevm.vcpus_count):
# We expect a single fc_vcpu thread tagged with
# f`fc_vcpu {vcpu}`.
tag = f"fc_vcpu {vcpu}"
assert tag in cpu_load and len(cpu_load[tag]) == 1
for thread_id in cpu_load[tag]:
data = cpu_load[tag][thread_id]
data_len = len(data)
assert data_len == cpu_load_runtime
if DEBUG:
res[IPERF3_END_RESULTS_TAG][
f"cpu_utilization_fc_vcpu_{vcpu}_samples"] = data
vcpus_util += sum(data)/data_len
cpu_util_perc[CPU_UTILIZATION_VCPUS_TOTAL] = vcpus_util
yield res
def _test_snapshot_create_latency(context):
logger = context.custom['logger']
vm_builder = context.custom['builder']
snapshot_type = context.custom['snapshot_type']
enable_diff_snapshots = snapshot_type == SnapshotType.DIFF
# Create a rw copy artifact.
rw_disk = context.disk.copy()
# Get ssh key from read-only artifact.
ssh_key = context.disk.ssh_key()
logger.info("Fetching firecracker/jailer versions from {}."
.format(DEFAULT_TEST_IMAGES_S3_BUCKET))
artifacts = ArtifactCollection(_test_images_s3_bucket())
firecracker_versions = artifacts.firecracker_versions(
older_than=get_firecracker_version_from_toml())
assert len(firecracker_versions) > 0
# Test snapshot creation for every supported target version.
for target_version in firecracker_versions:
logger.info("""Measuring snapshot create({}) latency for target
version: {} and microvm: \"{}\", kernel {}, disk {} """
.format(snapshot_type,
target_version,
context.microvm.name(),
context.kernel.name(),
context.disk.name()))
# Measure a burst of snapshot create calls.
for i in range(SAMPLE_COUNT):
# Create a fresh microVM from artifacts.
vm = vm_builder.build(kernel=context.kernel,
disks=[rw_disk],
ssh_key=ssh_key,
config=context.microvm,
enable_diff_snapshots=enable_diff_snapshots,
use_ramdisk=True)
# Configure metrics system.
metrics_fifo_path = os.path.join(vm.path, 'metrics_fifo')
metrics_fifo = log_tools.Fifo(metrics_fifo_path)
response = vm.metrics.put(
metrics_path=vm.create_jailed_resource(metrics_fifo.path)
)
assert vm.api_session.is_status_no_content(response.status_code)
vm.start()
# Check if the needed CPU cores are available. We have the API
# thread, VMM thread and then one thread for each configured vCPU.
assert CpuMap.len() >= 2 + vm.vcpus_count
# Pin uVM threads to physical cores.
current_cpu_id = 0
assert vm.pin_vmm(current_cpu_id), \
"Failed to pin firecracker thread."
current_cpu_id += 1
assert vm.pin_api(current_cpu_id), \
"Failed to pin fc_api thread."
for idx_vcpu in range(vm.vcpus_count):
current_cpu_id += 1
assert vm.pin_vcpu(idx_vcpu, current_cpu_id + idx_vcpu), \
f"Failed to pin fc_vcpu {idx_vcpu} thread."
# Create a snapshot builder from a microVM.
snapshot_builder = SnapshotBuilder(vm)
snapshot_builder.create(disks=[rw_disk],
ssh_key=ssh_key,
snapshot_type=snapshot_type,
target_version=target_version,
use_ramdisk=True)
metrics = vm.flush_metrics(metrics_fifo)
vm_name = context.microvm.name()
if snapshot_type == SnapshotType.FULL:
value = metrics['latencies_us']['full_create_snapshot']
baseline = CREATE_LATENCY_BASELINES[PLATFORM][vm_name]['FULL']
else:
value = metrics['latencies_us']['diff_create_snapshot']
baseline = CREATE_LATENCY_BASELINES[PLATFORM][vm_name]['DIFF']
value = value / USEC_IN_MSEC
assert baseline > value, "CreateSnapshot latency degraded."
logger.info("Latency {}/3: {} ms".format(i + 1, value))
vm.kill()
def produce_iperf_output(basevm, guest_cmd_builder, current_avail_cpu, runtime,
omit, load_factor, modes):
"""Produce iperf raw output from server-client connection."""
# Check if we have enough CPUs to pin the servers on the host.
# The available CPUs are the total minus vcpus, vmm and API threads.
assert load_factor * basevm.vcpus_count < CpuMap.len() - \
basevm.vcpus_count - 2
host_uds_path = os.path.join(basevm.path, VSOCK_UDS_PATH)
# Start the servers.
for server_idx in range(load_factor * basevm.vcpus_count):
assigned_cpu = CpuMap(current_avail_cpu)
iperf_server = \
CmdBuilder(f"taskset --cpu-list {assigned_cpu}") \
.with_arg(IPERF3) \
.with_arg("-sD") \
.with_arg("--vsock") \
.with_arg("-B", host_uds_path) \
.with_arg("-p", f"{BASE_PORT + server_idx}") \
.with_arg("-1") \
.build()
run_cmd(iperf_server)
current_avail_cpu += 1
# Wait for iperf3 servers to start.
time.sleep(SERVER_STARTUP_TIME)
# Start `vcpus` iperf3 clients. We can not use iperf3 parallel streams
# due to non deterministic results and lack of scaling.
def spawn_iperf_client(conn, client_idx, mode):
# Add the port where the iperf3 client is going to send/receive.
cmd = guest_cmd_builder.with_arg("-p", BASE_PORT +
client_idx).with_arg(mode).build()
# Bind the UDS in the jailer's root.
basevm.create_jailed_resource(
os.path.join(
basevm.path,
_make_host_port_path(VSOCK_UDS_PATH, BASE_PORT + client_idx)))
pinned_cmd = f"taskset --cpu-list {client_idx % basevm.vcpus_count}" \
f" {cmd}"
rc, stdout, _ = conn.execute_command(pinned_cmd)
assert rc == 0
return stdout.read()
with concurrent.futures.ThreadPoolExecutor() as executor:
futures = list()
cpu_load_future = executor.submit(get_cpu_percent,
basevm.jailer_clone_pid,
runtime - SERVER_STARTUP_TIME, omit)
modes_len = len(modes)
ssh_connection = net_tools.SSHConnection(basevm.ssh_config)
for client_idx in range(load_factor * basevm.vcpus_count):
futures.append(
executor.submit(
spawn_iperf_client,
ssh_connection,
client_idx,
# Distribute the modes evenly.
modes[client_idx % modes_len]))
cpu_load = cpu_load_future.result()
for future in futures[:-1]:
res = json.loads(future.result())
res[IPERF3_END_RESULTS_TAG][
IPERF3_CPU_UTILIZATION_PERCENT_OUT_TAG] = None
yield res
# Attach the real CPU utilization vmm/vcpus to
# the last iperf3 server-client pair measurements.
res = json.loads(futures[-1].result())
# We expect a single emulation thread tagged with `firecracker` name.
tag = "firecracker"
assert tag in cpu_load and len(cpu_load[tag]) == 1
thread_id = list(cpu_load[tag])[0]
data = cpu_load[tag][thread_id]
vmm_util = sum(data) / len(data)
cpu_util_perc = res[IPERF3_END_RESULTS_TAG][
IPERF3_CPU_UTILIZATION_PERCENT_OUT_TAG] = dict()
cpu_util_perc[CPU_UTILIZATION_VMM_TAG] = vmm_util
vcpus_util = 0
for vcpu in range(basevm.vcpus_count):
# We expect a single fc_vcpu thread tagged with
# f`fc_vcpu {vcpu}`.
tag = f"fc_vcpu {vcpu}"
assert tag in cpu_load and len(cpu_load[tag]) == 1
thread_id = list(cpu_load[tag])[0]
data = cpu_load[tag][thread_id]
vcpus_util += (sum(data) / len(data))
cpu_util_perc[CPU_UTILIZATION_VCPUS_TOTAL_TAG] = vcpus_util
yield res
def _test_snapshot_create_latency(context):
logger = context.custom['logger']
vm_builder = context.custom['builder']
snapshot_type = context.custom['snapshot_type']
file_dumper = context.custom['results_file_dumper']
diff_snapshots = snapshot_type == SnapshotType.DIFF
# Create a rw copy artifact.
rw_disk = context.disk.copy()
# Get ssh key from read-only artifact.
ssh_key = context.disk.ssh_key()
logger.info("Fetching firecracker/jailer versions from {}.".format(
DEFAULT_TEST_IMAGES_S3_BUCKET))
artifacts = ArtifactCollection(_test_images_s3_bucket())
firecracker_versions = artifacts.firecracker_versions(
# v1.0.0 breaks snapshot compatibility with older versions.
min_version="1.0.0",
max_version=get_firecracker_version_from_toml())
assert len(firecracker_versions) > 0
# Test snapshot creation for every supported target version.
for target_version in firecracker_versions:
logger.info("""Measuring snapshot create({}) latency for target
version: {} and microvm: \"{}\", kernel {}, disk {} """.format(
snapshot_type, target_version, context.microvm.name(),
context.kernel.name(), context.disk.name()))
# Create a fresh microVM from artifacts.
vm_instance = vm_builder.build(kernel=context.kernel,
disks=[rw_disk],
ssh_key=ssh_key,
config=context.microvm,
diff_snapshots=diff_snapshots,
use_ramdisk=True)
vm = vm_instance.vm
# Configure metrics system.
metrics_fifo_path = os.path.join(vm.path, 'metrics_fifo')
metrics_fifo = log_tools.Fifo(metrics_fifo_path)
response = vm.metrics.put(
metrics_path=vm.create_jailed_resource(metrics_fifo.path))
assert vm.api_session.is_status_no_content(response.status_code)
vm.start()
# Check if the needed CPU cores are available. We have the API
# thread, VMM thread and then one thread for each configured vCPU.
assert CpuMap.len() >= 2 + vm.vcpus_count
# Pin uVM threads to physical cores.
current_cpu_id = 0
assert vm.pin_vmm(current_cpu_id), \
"Failed to pin firecracker thread."
current_cpu_id += 1
assert vm.pin_api(current_cpu_id), \
"Failed to pin fc_api thread."
for idx_vcpu in range(vm.vcpus_count):
current_cpu_id += 1
assert vm.pin_vcpu(idx_vcpu, current_cpu_id + idx_vcpu), \
f"Failed to pin fc_vcpu {idx_vcpu} thread."
st_core = core.Core(
name="snapshot_create_full_latency" if snapshot_type
== SnapshotType.FULL else "snapshot_create_diff_latency",
iterations=SAMPLE_COUNT)
prod = producer.LambdaProducer(func=snapshot_create_producer,
func_kwargs={
"logger": logger,
"vm": vm,
"disks": [rw_disk],
"ssh_key": ssh_key,
"target_version": target_version,
"metrics_fifo": metrics_fifo,
"snapshot_type": snapshot_type
})
cons = consumer.LambdaConsumer(
func=lambda cons, result: cons.consume_stat(
st_name="max", ms_name="latency", value=result),
func_kwargs={})
eager_map(
cons.set_measurement_def,
snapshot_create_measurements(context.microvm.name(),
snapshot_type))
st_core.add_pipe(producer=prod,
consumer=cons,
tag=context.microvm.name())
# Gather results and verify pass criteria.
try:
result = st_core.run_exercise()
except core.CoreException as err:
handle_failure(file_dumper, err)
dump_test_result(file_dumper, result)
def iperf_workload(context):
"""Iperf between guest and host in both directions for TCP workload."""
vm_builder = context.custom["builder"]
logger = context.custom["logger"]
file_dumper = context.custom["results_file_dumper"]
# Create a rw copy artifact.
rw_disk = context.disk.copy()
# Get ssh key from read-only artifact.
ssh_key = context.disk.ssh_key()
# Create a fresh microvm from artifacts.
vm_instance = vm_builder.build(
kernel=context.kernel, disks=[rw_disk], ssh_key=ssh_key, config=context.microvm
)
basevm = vm_instance.vm
basevm.start()
custom = {
"microvm": context.microvm.name(),
"kernel": context.kernel.name(),
"disk": context.disk.name(),
"cpu_model_name": get_cpu_model_name(),
}
st_core = core.Core(name=TEST_ID, iterations=1, custom=custom)
# Check if the needed CPU cores are available. We have the API thread, VMM
# thread and then one thread for each configured vCPU.
assert CpuMap.len() >= 2 + basevm.vcpus_count
# Pin uVM threads to physical cores.
current_avail_cpu = 0
assert basevm.pin_vmm(current_avail_cpu), "Failed to pin firecracker thread."
current_avail_cpu += 1
assert basevm.pin_api(current_avail_cpu), "Failed to pin fc_api thread."
for i in range(basevm.vcpus_count):
current_avail_cpu += 1
assert basevm.pin_vcpu(
i, current_avail_cpu
), f"Failed to pin fc_vcpu {i} thread."
logger.info(
'Testing with microvm: "{}", kernel {}, disk {}'.format(
context.microvm.name(), context.kernel.name(), context.disk.name()
)
)
for cons, prod, tag in pipes(
basevm,
DEFAULT_HOST_IP,
current_avail_cpu + 1,
f"{context.kernel.name()}/{context.disk.name()}/" f"{context.microvm.name()}",
):
st_core.add_pipe(prod, cons, tag)
# Start running the commands on guest, gather results and verify pass
# criteria.
try:
result = st_core.run_exercise()
except core.CoreException as err:
handle_failure(file_dumper, err)
file_dumper.dump(result)