def pipes(basevm, current_avail_cpu, env_id): """Producer/Consumer pipes generator.""" host_cpu_model_name = get_cpu_model_name() cpus_baselines = test_cfg.CONFIG["hosts"]["instances"]["m5d.metal"]["cpus"] stats = no_criteria_stats() baselines = list( filter(lambda baseline: baseline["model"] == host_cpu_model_name, cpus_baselines)) for mode in test_cfg.CONFIG["modes"]: # We run bi-directional tests only on uVM with more than 2 vCPus # because we need to pin one iperf3/direction per vCPU, and since we # have two directions, we need at least two vCPUs. if mode == "bd" and basevm.vcpus_count < 2: continue for protocol in test_cfg.CONFIG["protocols"]: host_cpu_model_name = get_cpu_model_name() for payload_length in protocol["payload_length"]: iperf_guest_cmd_builder = CmdBuilder(test_cfg.IPERF3) \ .with_arg("--vsock") \ .with_arg("-c", 2) \ .with_arg("--json") \ .with_arg("--omit", protocol["omit"]) \ .with_arg("--time", test_cfg.CONFIG["time"]) if payload_length: iperf_guest_cmd_builder = iperf_guest_cmd_builder \ .with_arg("--len", f"{payload_length}") iperf3_id_payload_len = payload_length else: iperf3_id_payload_len = "DEFAULT" iperf3_id = f"vsock-p{iperf3_id_payload_len}" \ f"-{basevm.vcpus_count}vcpu-{mode}" cons = consumer.LambdaConsumer(consume_stats=False, func=consume_iperf_output) if len(baselines) > 0: stats = criteria_stats(baselines[0], iperf3_id, env_id) eager_map(cons.set_measurement_def, measurements()) eager_map(cons.set_stat_def, stats) prod_kwargs = { "guest_cmd_builder": iperf_guest_cmd_builder, "basevm": basevm, "current_avail_cpu": current_avail_cpu, "runtime": test_cfg.CONFIG["time"], "omit": protocol["omit"], "load_factor": test_cfg.CONFIG["load_factor"], "modes": test_cfg.CONFIG["modes"][mode], } prod = producer.LambdaProducer(produce_iperf_output, prod_kwargs) yield cons, prod, f"{env_id}/{iperf3_id}"
def create_pipes_generator(basevm, mode, current_avail_cpu, protocol, host_ip, env_id): """Create producer/consumer pipes.""" host_cpu_model_name = get_cpu_model_name() for payload_length in protocol["payload_length"]: for ws in protocol["window_size"]: iperf_guest_cmd_builder = CmdBuilder(IPERF3) \ .with_arg("--verbose") \ .with_arg("--client", host_ip) \ .with_arg("--time", CONFIG["time"]) \ .with_arg("--json") \ .with_arg("--omit", protocol["omit"]) if ws: iperf_guest_cmd_builder = iperf_guest_cmd_builder \ .with_arg("--window", f"{ws}") iperf3_id_ws = ws else: iperf3_id_ws = "DEFAULT" if payload_length: iperf_guest_cmd_builder = iperf_guest_cmd_builder \ .with_arg("--len", f"{payload_length}") iperf3_id_payload_len = payload_length else: iperf3_id_payload_len = "DEFAULT" iperf3_id = f"tcp-p{iperf3_id_payload_len}" \ f"-ws{iperf3_id_ws}-{basevm.vcpus_count}vcpu-{mode}" cons = consumer.LambdaConsumer( consume_stats=True, func=consume_iperf_tcp_output, func_kwargs={ "vcpus_count": basevm.vcpus_count } ) eager_map(cons.set_measurement_def, measurements_tcp()) eager_map(cons.set_stat_def, stats_tcp(host_cpu_model_name, iperf3_id, env_id)) prod_kwargs = { "guest_cmd_builder": iperf_guest_cmd_builder, "basevm": basevm, "current_avail_cpu": current_avail_cpu, "runtime": CONFIG["time"], "omit": protocol["omit"], "load_factor": CONFIG["load_factor"], "modes": CONFIG["modes"][mode] } prod = producer.LambdaProducer(produce_iperf_output, prod_kwargs) yield cons, prod, f"{env_id}/{iperf3_id}"
def consume_ping_output(cons, raw_data, requests): """Consume ping output. Output example: PING 8.8.8.8 (8.8.8.8) 56(84) bytes of data. 64 bytes from 8.8.8.8: icmp_seq=1 ttl=118 time=17.7 ms 64 bytes from 8.8.8.8: icmp_seq=2 ttl=118 time=17.7 ms 64 bytes from 8.8.8.8: icmp_seq=3 ttl=118 time=17.4 ms 64 bytes from 8.8.8.8: icmp_seq=4 ttl=118 time=17.8 ms --- 8.8.8.8 ping statistics --- 4 packets transmitted, 4 received, 0% packet loss, time 3005ms rtt min/avg/max/mdev = 17.478/17.705/17.808/0.210 ms """ eager_map(cons.set_measurement_def, measurements()) st_keys = ["Min", "Avg", "Max", "Stddev"] output = raw_data.strip().split("\n") assert len(output) > 2 # E.g: round-trip min/avg/max/stddev = 17.478/17.705/17.808/0.210 ms stat_values = output[-1] pattern_stats = "min/avg/max/[a-z]+dev = (.+)/(.+)/(.+)/(.+) ms" stat_values = re.findall(pattern_stats, stat_values)[0] assert len(stat_values) == 4 for index, stat_value in enumerate(stat_values[:4]): cons.consume_stat( st_name=st_keys[index], ms_name=LATENCY, value=float(stat_value) ) # E.g: 4 packets transmitted, 4 received, 0% packet loss packet_stats = output[-2] pattern_packet = ".+ packet.+transmitted, .+ received," " (.+)% packet loss" pkt_loss = re.findall(pattern_packet, packet_stats)[0] assert len(pkt_loss) == 1 cons.consume_stat(st_name=PKT_LOSS_STAT_KEY, ms_name=PKT_LOSS, value=pkt_loss[0]) # Compute percentiles. seqs = output[1 : requests + 1] times = [] pattern_time = ".+ bytes from .+: icmp_seq=.+ ttl=.+ time=(.+) ms" for index, seq in enumerate(seqs): time = re.findall(pattern_time, seq) assert len(time) == 1 times.append(time[0]) times.sort() cons.consume_stat( st_name="Percentile50", ms_name=LATENCY, value=times[int(requests * 0.5)] ) cons.consume_stat( st_name="Percentile90", ms_name=LATENCY, value=times[int(requests * 0.9)] ) cons.consume_stat( st_name="Percentile99", ms_name=LATENCY, value=times[int(requests * 0.99)] )
def _test_older_snapshot_resume_latency(context): builder = context.custom["builder"] logger = context.custom["logger"] snapshot_type = context.custom['snapshot_type'] file_dumper = context.custom["results_file_dumper"] firecracker = context.firecracker jailer = firecracker.jailer() jailer.download() fc_version = firecracker.base_name()[1:] logger.info("Firecracker version: %s", fc_version) logger.info("Source Firecracker: %s", firecracker.local_path()) logger.info("Source Jailer: %s", jailer.local_path()) # Create a fresh microvm with the binary artifacts. vm_instance = builder.build_vm_micro(firecracker.local_path(), jailer.local_path()) basevm = vm_instance.vm basevm.start() ssh_connection = net_tools.SSHConnection(basevm.ssh_config) # Check if guest works. exit_code, _, _ = ssh_connection.execute_command("ls") assert exit_code == 0 # The snapshot builder expects disks as paths, not artifacts. disks = [] for disk in vm_instance.disks: disks.append(disk.local_path()) # Create a snapshot builder from a microvm. snapshot_builder = SnapshotBuilder(basevm) snapshot = snapshot_builder.create(disks, vm_instance.ssh_key, snapshot_type) basevm.kill() st_core = core.Core(name="older_snapshot_resume_latency", iterations=SAMPLE_COUNT) prod = producer.LambdaProducer(func=snapshot_resume_producer, func_kwargs={ "logger": logger, "vm_builder": builder, "snapshot": snapshot, "snapshot_type": snapshot_type, "use_ramdisk": False }) cons = consumer.LambdaConsumer(func=lambda cons, result: cons.consume_stat( st_name="max", ms_name="latency", value=result), func_kwargs={}) eager_map(cons.set_measurement_def, snapshot_resume_measurements(context.microvm.name())) st_core.add_pipe(producer=prod, consumer=cons, tag=context.microvm.name()) # Gather results and verify pass criteria. try: result = st_core.run_exercise() except core.CoreException as err: handle_failure(file_dumper, err) dump_test_result(file_dumper, result)
def _test_snapshot_resume_latency(context): logger = context.custom['logger'] vm_builder = context.custom['builder'] snapshot_type = context.custom['snapshot_type'] file_dumper = context.custom['results_file_dumper'] diff_snapshots = snapshot_type == SnapshotType.DIFF logger.info("""Measuring snapshot resume({}) latency for microvm: \"{}\", kernel {}, disk {} """.format(snapshot_type, context.microvm.name(), context.kernel.name(), context.disk.name())) # Create a rw copy artifact. rw_disk = context.disk.copy() # Get ssh key from read-only artifact. ssh_key = context.disk.ssh_key() # Create a fresh microvm from aftifacts. vm_instance = vm_builder.build(kernel=context.kernel, disks=[rw_disk], ssh_key=ssh_key, config=context.microvm, diff_snapshots=diff_snapshots, use_ramdisk=True) basevm = vm_instance.vm basevm.start() ssh_connection = net_tools.SSHConnection(basevm.ssh_config) # Check if guest works. exit_code, _, _ = ssh_connection.execute_command("ls") assert exit_code == 0 logger.info("Create {}.".format(snapshot_type)) # Create a snapshot builder from a microvm. snapshot_builder = SnapshotBuilder(basevm) snapshot = snapshot_builder.create([rw_disk.local_path()], ssh_key, snapshot_type, use_ramdisk=True) basevm.kill() st_core = core.Core(name="snapshot_resume_latency", iterations=SAMPLE_COUNT) prod = producer.LambdaProducer(func=snapshot_resume_producer, func_kwargs={ "logger": logger, "vm_builder": vm_builder, "snapshot": snapshot, "snapshot_type": snapshot_type, "use_ramdisk": True }) cons = consumer.LambdaConsumer(func=lambda cons, result: cons.consume_stat( st_name="max", ms_name="latency", value=result), func_kwargs={}) eager_map(cons.set_measurement_def, snapshot_resume_measurements(context.microvm.name())) st_core.add_pipe(producer=prod, consumer=cons, tag=context.microvm.name()) # Gather results and verify pass criteria. try: result = st_core.run_exercise() except core.CoreException as err: handle_failure(file_dumper, err) dump_test_result(file_dumper, result)
def _test_snapshot_create_latency(context): logger = context.custom['logger'] vm_builder = context.custom['builder'] snapshot_type = context.custom['snapshot_type'] file_dumper = context.custom['results_file_dumper'] diff_snapshots = snapshot_type == SnapshotType.DIFF # Create a rw copy artifact. rw_disk = context.disk.copy() # Get ssh key from read-only artifact. ssh_key = context.disk.ssh_key() logger.info("Fetching firecracker/jailer versions from {}.".format( DEFAULT_TEST_IMAGES_S3_BUCKET)) artifacts = ArtifactCollection(_test_images_s3_bucket()) firecracker_versions = artifacts.firecracker_versions( # v1.0.0 breaks snapshot compatibility with older versions. min_version="1.0.0", max_version=get_firecracker_version_from_toml()) assert len(firecracker_versions) > 0 # Test snapshot creation for every supported target version. for target_version in firecracker_versions: logger.info("""Measuring snapshot create({}) latency for target version: {} and microvm: \"{}\", kernel {}, disk {} """.format( snapshot_type, target_version, context.microvm.name(), context.kernel.name(), context.disk.name())) # Create a fresh microVM from artifacts. vm_instance = vm_builder.build(kernel=context.kernel, disks=[rw_disk], ssh_key=ssh_key, config=context.microvm, diff_snapshots=diff_snapshots, use_ramdisk=True) vm = vm_instance.vm # Configure metrics system. metrics_fifo_path = os.path.join(vm.path, 'metrics_fifo') metrics_fifo = log_tools.Fifo(metrics_fifo_path) response = vm.metrics.put( metrics_path=vm.create_jailed_resource(metrics_fifo.path)) assert vm.api_session.is_status_no_content(response.status_code) vm.start() # Check if the needed CPU cores are available. We have the API # thread, VMM thread and then one thread for each configured vCPU. assert CpuMap.len() >= 2 + vm.vcpus_count # Pin uVM threads to physical cores. current_cpu_id = 0 assert vm.pin_vmm(current_cpu_id), \ "Failed to pin firecracker thread." current_cpu_id += 1 assert vm.pin_api(current_cpu_id), \ "Failed to pin fc_api thread." for idx_vcpu in range(vm.vcpus_count): current_cpu_id += 1 assert vm.pin_vcpu(idx_vcpu, current_cpu_id + idx_vcpu), \ f"Failed to pin fc_vcpu {idx_vcpu} thread." st_core = core.Core( name="snapshot_create_full_latency" if snapshot_type == SnapshotType.FULL else "snapshot_create_diff_latency", iterations=SAMPLE_COUNT) prod = producer.LambdaProducer(func=snapshot_create_producer, func_kwargs={ "logger": logger, "vm": vm, "disks": [rw_disk], "ssh_key": ssh_key, "target_version": target_version, "metrics_fifo": metrics_fifo, "snapshot_type": snapshot_type }) cons = consumer.LambdaConsumer( func=lambda cons, result: cons.consume_stat( st_name="max", ms_name="latency", value=result), func_kwargs={}) eager_map( cons.set_measurement_def, snapshot_create_measurements(context.microvm.name(), snapshot_type)) st_core.add_pipe(producer=prod, consumer=cons, tag=context.microvm.name()) # Gather results and verify pass criteria. try: result = st_core.run_exercise() except core.CoreException as err: handle_failure(file_dumper, err) dump_test_result(file_dumper, result)
def fio_workload(context): """Execute block device emulation benchmarking scenarios.""" vm_builder = context.custom['builder'] logger = context.custom["logger"] # Create a rw copy artifact. rw_disk = context.disk.copy() # Get ssh key from read-only artifact. ssh_key = context.disk.ssh_key() # Create a fresh microvm from artifacts. basevm = vm_builder.build(kernel=context.kernel, disks=[rw_disk], ssh_key=ssh_key, config=context.microvm) # Add a secondary block device for benchmark tests. fs = drive_tools.FilesystemFile( os.path.join(basevm.fsfiles, 'scratch'), CONFIG["block_device_size"] ) basevm.add_drive('scratch', fs.path) basevm.start() # Get names of threads in Firecracker. current_cpu_id = 0 basevm.pin_vmm(current_cpu_id) current_cpu_id += 1 basevm.pin_api(current_cpu_id) for vcpu_id in range(basevm.vcpus_count): current_cpu_id += 1 basevm.pin_vcpu(vcpu_id, current_cpu_id) st_core = core.Core(name=TEST_ID, iterations=1, custom={"microvm": context.microvm.name(), "kernel": context.kernel.name(), "disk": context.disk.name()}) logger.info("Testing with microvm: \"{}\", kernel {}, disk {}" .format(context.microvm.name(), context.kernel.name(), context.disk.name())) ssh_connection = net_tools.SSHConnection(basevm.ssh_config) env_id = f"{context.kernel.name()}/{context.disk.name()}" for mode in CONFIG["fio_modes"]: ms_defs = measurements(mode) for bs in CONFIG["fio_blk_sizes"]: fio_id = f"{mode}-bs{bs}-{basevm.vcpus_count}vcpu" st_defs = statistics(mode, env_id, fio_id) st_prod = st.producer.LambdaProducer( func=run_fio, func_kwargs={ "env_id": env_id, "basevm": basevm, "ssh_conn": ssh_connection, "mode": mode, "bs": bs } ) numjobs = CONFIG['load_factor'] * basevm.vcpus_count st_cons = st.consumer.LambdaConsumer( consume_stats=False, func=consume_fio_output, func_kwargs={"numjobs": numjobs, "mode": mode, "bs": bs, "env_id": env_id}) eager_map(st_cons.set_measurement_def, ms_defs) eager_map(st_cons.set_stat_def, st_defs) st_core.add_pipe(st_prod, st_cons, tag=f"{env_id}/{fio_id}") st_core.run_exercise() basevm.kill()
def create_pipes_generator(basevm, mode, current_avail_cpu, protocol, host_ip, env_id): """Create producer/consumer pipes.""" host_cpu_model_name = get_cpu_model_name() cpus_baselines = test_cfg.CONFIG["hosts"]["instances"]["m5d.metal"]["cpus"] stats = no_criteria_stats() baselines = list(filter( lambda baseline: baseline["model"] == host_cpu_model_name, cpus_baselines)) for payload_length in protocol["payload_length"]: for ws in protocol["window_size"]: iperf_guest_cmd_builder = CmdBuilder(test_cfg.IPERF3) \ .with_arg("--verbose") \ .with_arg("--client", host_ip) \ .with_arg("--time", test_cfg.CONFIG["time"]) \ .with_arg("--json") \ .with_arg("--omit", protocol["omit"]) if ws: iperf_guest_cmd_builder = iperf_guest_cmd_builder \ .with_arg("--window", f"{ws}") iperf3_id_ws = ws else: iperf3_id_ws = "DEFAULT" if payload_length: iperf_guest_cmd_builder = iperf_guest_cmd_builder \ .with_arg("--len", f"{payload_length}") iperf3_id_payload_len = payload_length else: iperf3_id_payload_len = "DEFAULT" iperf3_id = f"tcp-p{iperf3_id_payload_len}" \ f"-ws{iperf3_id_ws}-{basevm.vcpus_count}vcpu-{mode}" cons = consumer.LambdaConsumer( consume_stats=False, func=consume_iperf_tcp_output, func_kwargs={ "vcpus_count": basevm.vcpus_count } ) if len(baselines) > 0: stats = criteria_stats(baselines[0], iperf3_id, env_id) eager_map(cons.set_measurement_def, measurements()) eager_map(cons.set_stat_def, stats) prod_kwargs = { "guest_cmd_builder": iperf_guest_cmd_builder, "basevm": basevm, "current_avail_cpu": current_avail_cpu, "runtime": test_cfg.CONFIG["time"], "omit": protocol["omit"], "load_factor": test_cfg.CONFIG["load_factor"], "modes": test_cfg.CONFIG["modes"][mode] } prod = producer.LambdaProducer(produce_iperf_output, prod_kwargs) yield cons, prod, f"{env_id}/{iperf3_id}"