def produce_iperf_output(basevm, guest_cmd_builder, current_avail_cpu, runtime,
omit, load_factor, modes):
"""Produce iperf raw output from server-client connection."""
# Check if we have enough CPUs to pin the servers on the host.
# The available CPUs are the total minus vcpus, vmm and API threads.
assert load_factor * basevm.vcpus_count < CpuMap.len() - \
basevm.vcpus_count - 2
host_uds_path = os.path.join(basevm.path, VSOCK_UDS_PATH)
# Start the servers.
for server_idx in range(load_factor * basevm.vcpus_count):
assigned_cpu = CpuMap(current_avail_cpu)
iperf_server = \
CmdBuilder(f"taskset --cpu-list {assigned_cpu}") \
.with_arg(IPERF3) \
.with_arg("-sD") \
.with_arg("--vsock") \
.with_arg("-B", host_uds_path) \
.with_arg("-p", f"{BASE_PORT + server_idx}") \
.with_arg("-1") \
.build()
run_cmd(iperf_server)
current_avail_cpu += 1
# Wait for iperf3 servers to start.
time.sleep(SERVER_STARTUP_TIME)
# Start `vcpus` iperf3 clients. We can not use iperf3 parallel streams
# due to non deterministic results and lack of scaling.
def spawn_iperf_client(conn, client_idx, mode):
# Add the port where the iperf3 client is going to send/receive.
cmd = guest_cmd_builder.with_arg("-p", BASE_PORT +
client_idx).with_arg(mode).build()
# Bind the UDS in the jailer's root.
basevm.create_jailed_resource(
os.path.join(
basevm.path,
make_host_port_path(VSOCK_UDS_PATH, BASE_PORT + client_idx)))
pinned_cmd = f"taskset --cpu-list {client_idx % basevm.vcpus_count}" \
f" {cmd}"
rc, stdout, stderr = conn.execute_command(pinned_cmd)
assert rc == 0, stderr.read()
return stdout.read()
with concurrent.futures.ThreadPoolExecutor() as executor:
futures = []
cpu_load_future = executor.submit(get_cpu_percent,
basevm.jailer_clone_pid,
runtime - SERVER_STARTUP_TIME, omit)
modes_len = len(modes)
ssh_connection = net_tools.SSHConnection(basevm.ssh_config)
for client_idx in range(load_factor * basevm.vcpus_count):
futures.append(
executor.submit(
spawn_iperf_client,
ssh_connection,
client_idx,
# Distribute the modes evenly.
modes[client_idx % modes_len]))
cpu_load = cpu_load_future.result()
for future in futures[:-1]:
res = json.loads(future.result())
res[IPERF3_END_RESULTS_TAG][
IPERF3_CPU_UTILIZATION_PERCENT_OUT_TAG] = None
yield res
# Attach the real CPU utilization vmm/vcpus to
# the last iperf3 server-client pair measurements.
res = json.loads(futures[-1].result())
# We expect a single emulation thread tagged with `firecracker` name.
tag = "firecracker"
assert tag in cpu_load and len(cpu_load[tag]) == 1
thread_id = list(cpu_load[tag])[0]
data = cpu_load[tag][thread_id]
vmm_util = sum(data) / len(data)
cpu_util_perc = res[IPERF3_END_RESULTS_TAG][
IPERF3_CPU_UTILIZATION_PERCENT_OUT_TAG] = {}
cpu_util_perc[CPU_UTILIZATION_VMM] = vmm_util
vcpus_util = 0
for vcpu in range(basevm.vcpus_count):
# We expect a single fc_vcpu thread tagged with
# f`fc_vcpu {vcpu}`.
tag = f"fc_vcpu {vcpu}"
assert tag in cpu_load and len(cpu_load[tag]) == 1
thread_id = list(cpu_load[tag])[0]
data = cpu_load[tag][thread_id]
vcpus_util += (sum(data) / len(data))
cpu_util_perc[CPU_UTILIZATION_VCPUS_TOTAL] = vcpus_util
yield res
def run_fio(env_id, basevm, ssh_conn, mode, bs):
"""Run a fio test in the specified mode with block size bs."""
logs_path = f"{basevm.jailer.chroot_base_with_id()}/{env_id}/{mode}{bs}"
# Compute the fio command. Pin it to the first guest CPU.
cmd = CmdBuilder(FIO) \
.with_arg(f"--name={mode}-{bs}") \
.with_arg(f"--rw={mode}") \
.with_arg(f"--bs={bs}") \
.with_arg("--filename=/dev/vdb") \
.with_arg("--time_base=1") \
.with_arg(f"--size={CONFIG['block_device_size']}M") \
.with_arg("--direct=1") \
.with_arg("--ioengine=libaio") \
.with_arg("--iodepth=32") \
.with_arg(f"--ramp_time={CONFIG['omit']}") \
.with_arg(f"--numjobs={CONFIG['load_factor'] * basevm.vcpus_count}") \
.with_arg("--randrepeat=0") \
.with_arg(f"--runtime={CONFIG['time']}") \
.with_arg(f"--write_iops_log={mode}{bs}") \
.with_arg(f"--write_bw_log={mode}{bs}") \
.with_arg("--log_avg_msec=1000") \
.with_arg("--output-format=json+") \
.build()
rc, _, stderr = ssh_conn.execute_command(
"echo 'none' > /sys/block/vdb/queue/scheduler")
assert rc == 0, stderr.read()
assert stderr.read() == ""
run_cmd("echo 3 > /proc/sys/vm/drop_caches")
rc, _, stderr = ssh_conn.execute_command(
"echo 3 > /proc/sys/vm/drop_caches")
assert rc == 0, stderr.read()
assert stderr.read() == ""
# Start the CPU load monitor.
with concurrent.futures.ThreadPoolExecutor() as executor:
cpu_load_future = executor.submit(get_cpu_percent,
basevm.jailer_clone_pid,
CONFIG["time"],
omit=CONFIG["omit"])
# Print the fio command in the log and run it
rc, _, stderr = ssh_conn.execute_command(cmd)
assert rc == 0, stderr.read()
assert stderr.read() == ""
if os.path.isdir(logs_path):
shutil.rmtree(logs_path)
os.makedirs(logs_path)
ssh_conn.scp_get_file("*.log", logs_path)
rc, _, stderr = ssh_conn.execute_command("rm *.log")
assert rc == 0, stderr.read()
result = dict()
cpu_load = cpu_load_future.result()
tag = "firecracker"
assert tag in cpu_load and len(cpu_load[tag]) == 1
data = list(cpu_load[tag].values())[0]
data_len = len(data)
assert data_len == CONFIG["time"]
result[CPU_UTILIZATION_VMM] = sum(data) / data_len
if DEBUG:
result[CPU_UTILIZATION_VMM_SAMPLES_TAG] = data
vcpus_util = 0
for vcpu in range(basevm.vcpus_count):
# We expect a single fc_vcpu thread tagged with
# f`fc_vcpu {vcpu}`.
tag = f"fc_vcpu {vcpu}"
assert tag in cpu_load and len(cpu_load[tag]) == 1
data = list(cpu_load[tag].values())[0]
data_len = len(data)
assert data_len == CONFIG["time"]
if DEBUG:
samples_tag = f"cpu_utilization_fc_vcpu_{vcpu}_samples"
result[samples_tag] = data
vcpus_util += sum(data) / data_len
result[CPU_UTILIZATION_VCPUS_TOTAL] = vcpus_util
return result
def produce_iperf_output(basevm,
guest_cmd_builder,
current_avail_cpu,
runtime,
omit,
load_factor,
modes):
"""Produce iperf raw output from server-client connection."""
# Check if we have enough CPUs to pin the servers on the host.
# The available CPUs are the total minus vcpus, vmm and API threads.
assert load_factor * basevm.vcpus_count < CpuMap.len() - \
basevm.vcpus_count - 2
# Start the servers.
for server_idx in range(load_factor*basevm.vcpus_count):
assigned_cpu = CpuMap(current_avail_cpu)
iperf_server = \
CmdBuilder(f"taskset --cpu-list {assigned_cpu}") \
.with_arg(basevm.jailer.netns_cmd_prefix()) \
.with_arg(IPERF3) \
.with_arg("-sD") \
.with_arg("-p", f"{BASE_PORT + server_idx}") \
.with_arg("-1") \
.build()
run_cmd(iperf_server)
current_avail_cpu += 1
# Wait for iperf3 server to start.
time.sleep(2)
# Start `vcpus` iperf3 clients. We can not use iperf3 parallel streams
# due to non deterministic results and lack of scaling.
def spawn_iperf_client(conn, client_idx, mode):
# Add the port where the iperf3 client is going to send/receive.
cmd = guest_cmd_builder \
.with_arg("-p", f"{BASE_PORT + client_idx}") \
.with_arg(mode) \
.build()
pinned_cmd = f"taskset --cpu-list {client_idx % basevm.vcpus_count}" \
f" {cmd}"
_, stdout, _ = conn.execute_command(pinned_cmd)
return stdout.read()
# Subtract 2 to remove the readings from the workloads end.
cpu_load_runtime = runtime - 2
with concurrent.futures.ThreadPoolExecutor() as executor:
futures = list()
cpu_load_future = executor.submit(get_cpu_percent,
basevm.jailer_clone_pid,
cpu_load_runtime,
omit)
modes_len = len(modes)
ssh_connection = net_tools.SSHConnection(basevm.ssh_config)
for client_idx in range(load_factor*basevm.vcpus_count):
futures.append(executor.submit(spawn_iperf_client,
ssh_connection,
client_idx,
# Distribute the modes evenly.
modes[client_idx % modes_len]))
cpu_load = cpu_load_future.result()
for future in futures[:-1]:
res = json.loads(future.result())
res[IPERF3_END_RESULTS_TAG][
IPERF3_CPU_UTILIZATION_PERCENT_OUT_TAG] = None
yield res
# Attach the real CPU utilization vmm/vcpus to
# the last iperf3 server-client pair measurements.
res = json.loads(futures[-1].result())
# We expect a single emulation thread tagged with `firecracker` name.
tag = "firecracker"
assert tag in cpu_load and len(cpu_load[tag]) == 1
for thread_id in cpu_load[tag]:
data = cpu_load[tag][thread_id]
data_len = len(data)
assert data_len == cpu_load_runtime
vmm_util = sum(data)/data_len
cpu_util_perc = res[IPERF3_END_RESULTS_TAG][
IPERF3_CPU_UTILIZATION_PERCENT_OUT_TAG] = dict()
cpu_util_perc[CPU_UTILIZATION_VMM_TAG] = vmm_util
if DEBUG:
res[IPERF3_END_RESULTS_TAG][
DEBUG_CPU_UTILIZATION_VMM_SAMPLES_TAG] \
= data
vcpus_util = 0
for vcpu in range(basevm.vcpus_count):
# We expect a single fc_vcpu thread tagged with
# f`fc_vcpu {vcpu}`.
tag = f"fc_vcpu {vcpu}"
assert tag in cpu_load and len(cpu_load[tag]) == 1
for thread_id in cpu_load[tag]:
data = cpu_load[tag][thread_id]
data_len = len(data)
assert data_len == cpu_load_runtime
if DEBUG:
res[IPERF3_END_RESULTS_TAG][f"cpu_utilization_fc_vcpu"
f"_{vcpu}_samples"] = \
data
vcpus_util += sum(data)/data_len
cpu_util_perc[CPU_UTILIZATION_VCPUS_TOTAL_TAG] = vcpus_util
yield res
def create_pipes_generator(basevm,
mode,
current_avail_cpu,
protocol,
host_ip,
env_id):
"""Create producer/consumer pipes."""
host_cpu_model_name = get_cpu_model_name()
cpus_baselines = test_cfg.CONFIG["hosts"]["instances"]["m5d.metal"]["cpus"]
stats = no_criteria_stats()
baselines = list(filter(
lambda baseline: baseline["model"] == host_cpu_model_name,
cpus_baselines))
for payload_length in protocol["payload_length"]:
for ws in protocol["window_size"]:
iperf_guest_cmd_builder = CmdBuilder(test_cfg.IPERF3) \
.with_arg("--verbose") \
.with_arg("--client", host_ip) \
.with_arg("--time", test_cfg.CONFIG["time"]) \
.with_arg("--json") \
.with_arg("--omit", protocol["omit"])
if ws:
iperf_guest_cmd_builder = iperf_guest_cmd_builder \
.with_arg("--window", f"{ws}")
iperf3_id_ws = ws
else:
iperf3_id_ws = "DEFAULT"
if payload_length:
iperf_guest_cmd_builder = iperf_guest_cmd_builder \
.with_arg("--len", f"{payload_length}")
iperf3_id_payload_len = payload_length
else:
iperf3_id_payload_len = "DEFAULT"
iperf3_id = f"tcp-p{iperf3_id_payload_len}" \
f"-ws{iperf3_id_ws}-{basevm.vcpus_count}vcpu-{mode}"
cons = consumer.LambdaConsumer(
consume_stats=False,
func=consume_iperf_tcp_output,
func_kwargs={
"vcpus_count": basevm.vcpus_count
}
)
if len(baselines) > 0:
stats = criteria_stats(baselines[0], iperf3_id, env_id)
eager_map(cons.set_measurement_def, measurements())
eager_map(cons.set_stat_def, stats)
prod_kwargs = {
"guest_cmd_builder": iperf_guest_cmd_builder,
"basevm": basevm,
"current_avail_cpu": current_avail_cpu,
"runtime": test_cfg.CONFIG["time"],
"omit": protocol["omit"],
"load_factor": test_cfg.CONFIG["load_factor"],
"modes": test_cfg.CONFIG["modes"][mode]
}
prod = producer.LambdaProducer(produce_iperf_output,
prod_kwargs)
yield cons, prod, f"{env_id}/{iperf3_id}"
