def run(self, result): """execute the benchmark""" if "options" in self.scenario_cfg: options = "-s %s" % \ self.scenario_cfg['options'].get("packetsize", '56') else: options = "" destination = self.context_cfg['target'].get('ipaddr', '127.0.0.1') dest_list = [s.strip() for s in destination.split(',')] rtt_result = {} ping_result = {"rtt": rtt_result} sla_max_rtt = self.scenario_cfg.get("sla", {}).get("max_rtt") for pos, dest in enumerate(dest_list): if 'targets' in self.scenario_cfg: target_vm = self.scenario_cfg['targets'][pos] else: target_vm = self.scenario_cfg['target'] LOG.debug("ping %s %s", options, dest) with open(self.target_script, "r") as stdin_file: exit_status, stdout, stderr = self.connection.execute( "/bin/sh -s {0} {1}".format(dest, options), stdin=stdin_file) if exit_status != 0: raise RuntimeError(stderr) if isinstance(target_vm, dict): target_vm_name = target_vm.get("name") else: target_vm_name = target_vm.split('.')[0] if stdout: rtt_result[target_vm_name] = float(stdout.strip()) # store result before potential AssertionError result.update(utils.flatten_dict_key(ping_result)) if sla_max_rtt is not None: sla_max_rtt = float(sla_max_rtt) assert rtt_result[target_vm_name] <= sla_max_rtt,\ "rtt %f > sla: max_rtt(%f); " % \ (rtt_result[target_vm_name], sla_max_rtt) else: LOG.error("ping '%s' '%s' timeout", options, target_vm) # we need to specify a result to satisfy influxdb schema # choose a very large number to inidcate timeout # in this case choose an order of magnitude greater than the SLA rtt_result[target_vm_name] = float(self.PING_ERROR_RTT) # store result before potential AssertionError result.update(utils.flatten_dict_key(ping_result)) if sla_max_rtt is not None: raise AssertionError( "packet dropped rtt {:f} > sla: max_rtt({:f})".format( rtt_result[target_vm_name], sla_max_rtt)) else: raise AssertionError("packet dropped rtt {:f}".format( rtt_result[target_vm_name]))
def test_ramspeed_successful_run_sla(self, mock_ssh): options = {"test_id": 1, "load": 16, "block_size": 32} args = {"options": options, "sla": {"min_bandwidth": 6000}} r = ramspeed.Ramspeed(args, self.ctx) sample_output = '{"Result": [{"Test_type": "INTEGER & WRITING",\ "Block_size(kb)": 1, "Bandwidth(MBps)": 19909.18}, {"Test_type":\ "INTEGER & WRITING", "Block_size(kb)": 2, "Bandwidth(MBps)": 19873.89},\ {"Test_type": "INTEGER & WRITING", "Block_size(kb)": 4, "Bandwidth(MBps)":\ 19907.56}, {"Test_type": "INTEGER & WRITING", "Block_size(kb)": 8,\ "Bandwidth(MBps)": 19906.94}, {"Test_type": "INTEGER & WRITING",\ "Block_size(kb)": 16, "Bandwidth(MBps)": 19881.74}, {"Test_type":\ "INTEGER & WRITING", "Block_size(kb)": 32, "Bandwidth(MBps)": 19395.65},\ {"Test_type": "INTEGER & WRITING", "Block_size(kb)": 64, "Bandwidth(MBps)":\ 17623.14}, {"Test_type": "INTEGER & WRITING", "Block_size(kb)": 128,\ "Bandwidth(MBps)": 17677.36}, {"Test_type": "INTEGER & WRITING",\ "Block_size(kb)": 256, "Bandwidth(MBps)": 16113.49}, {"Test_type":\ "INTEGER & WRITING", "Block_size(kb)": 512, "Bandwidth(MBps)": 14659.19},\ {"Test_type": "INTEGER & WRITING", "Block_size(kb)": 1024, "Bandwidth(MBps)":\ 14680.75}, {"Test_type": "INTEGER & WRITING", "Block_size(kb)": 2048,\ "Bandwidth(MBps)": 14756.45}, {"Test_type": "INTEGER & WRITING",\ "Block_size(kb)": 4096, "Bandwidth(MBps)": 14604.44}, {"Test_type":\ "INTEGER & WRITING", "Block_size(kb)": 8192, "Bandwidth(MBps)": 14159.86},\ {"Test_type": "INTEGER & WRITING", "Block_size(kb)": 16384,\ "Bandwidth(MBps)": 14128.94}, {"Test_type": "INTEGER & WRITING",\ "Block_size(kb)": 32768, "Bandwidth(MBps)": 8340.85}]}' mock_ssh.SSH.from_node().execute.return_value = (0, sample_output, '') r.run(self.result) expected_result = utils.flatten_dict_key( jsonutils.loads(sample_output)) self.assertEqual(self.result, expected_result)
def test__dict_key_flatten(self): line = 'mpstat.loadavg1=0.29,rtt=1.03,mpstat.loadavg0=1.09,' \ 'mpstat.cpu0.%idle=99.00,mpstat.cpu0.%sys=0.00' # need to sort for assert to work line = ",".join(sorted(line.split(','))) flattened_data = utils.flatten_dict_key(self.data['benchmark']['data']) result = ",".join( ("=".join(item) for item in sorted(flattened_data.items()))) self.assertEqual(result, line)
def test_iperf_successful_sla(self, mock_ssh): options = {} args = {'options': options, 'sla': {'bytes_per_second': 15000000}} result = {} p = iperf3.Iperf(args, self.ctx) mock_ssh.SSH.from_node().execute.return_value = (0, '', '') p.host = mock_ssh.SSH.from_node() sample_output = self._read_sample_output(self.output_name_tcp) mock_ssh.SSH.from_node().execute.return_value = (0, sample_output, '') expected_result = utils.flatten_dict_key( jsonutils.loads(sample_output)) p.run(result) self.assertEqual(result, expected_result)
def test_iperf_successful_sla_jitter(self, mock_ssh): options = {"protocol": "udp", "bandwidth": "20m"} args = {'options': options, 'sla': {'jitter': 10}} result = {} p = iperf3.Iperf(args, self.ctx) mock_ssh.SSH.from_node().execute.return_value = (0, '', '') p.host = mock_ssh.SSH.from_node() sample_output = self._read_sample_output(self.output_name_udp) mock_ssh.SSH.from_node().execute.return_value = (0, sample_output, '') expected_result = utils.flatten_dict_key( jsonutils.loads(sample_output)) p.run(result) self.assertEqual(result, expected_result)
def test_ramspeed_mem_successful_run_sla(self, mock_ssh): options = {"test_id": 3, "load": 16, "block_size": 32, "iteration": 1} args = {"options": options, "sla": {"min_bandwidth": 6000}} r = ramspeed.Ramspeed(args, self.ctx) sample_output = '{"Result": [{"Test_type": "INTEGER Copy:",\ "Bandwidth(MBps)": 8353.97}, {"Test_type": "INTEGER Scale:",\ "Bandwidth(MBps)": 9078.59}, {"Test_type": "INTEGER Add:",\ "Bandwidth(MBps)": 10057.48}, {"Test_type": "INTEGER Triad:",\ "Bandwidth(MBps)": 10116.27}, {"Test_type": "INTEGER AVERAGE:",\ "Bandwidth(MBps)": 9401.58}]}' mock_ssh.SSH.from_node().execute.return_value = (0, sample_output, '') r.run(self.result) expected_result = utils.flatten_dict_key( jsonutils.loads(sample_output)) self.assertEqual(self.result, expected_result)
def run(self, result): """execute the benchmark""" if "options" in self.scenario_cfg: options = "-s %s" % \ self.scenario_cfg['options'].get("packetsize", '56') else: options = "" destination = self.context_cfg['target'].get('ipaddr', '127.0.0.1') dest_list = [s.strip() for s in destination.split(',')] rtt_result = {} ping_result = {"rtt": rtt_result} for pos, dest in enumerate(dest_list): if 'targets' in self.scenario_cfg: target_vm = self.scenario_cfg['targets'][pos] else: target_vm = self.scenario_cfg['target'] LOG.debug("ping '%s' '%s'", options, dest) with open(self.target_script, "r") as stdin_file: exit_status, stdout, stderr = self.connection.execute( "/bin/sh -s {0} {1}".format(dest, options), stdin=stdin_file) if exit_status != 0: raise RuntimeError(stderr) if stdout: if isinstance(target_vm, dict): target_vm_name = target_vm.get("name") else: target_vm_name = target_vm.split('.')[0] rtt_result[target_vm_name] = float(stdout) if "sla" in self.scenario_cfg: sla_max_rtt = int(self.scenario_cfg["sla"]["max_rtt"]) assert rtt_result[target_vm_name] <= sla_max_rtt,\ "rtt %f > sla: max_rtt(%f); " % \ (rtt_result[target_vm_name], sla_max_rtt) else: LOG.error("ping '%s' '%s' timeout", options, target_vm) result.update(utils.flatten_dict_key(ping_result))
def run(self, result): """execute the benchmark""" if not self.setup_done: self.setup() options = self.scenario_cfg['options'] test_id = options.get('type_id', 1) load = options.get('load', 8) block_size = options.get('block_size', 32) if test_id == 3 or test_id == 6: iteration = options.get('iteration', 1) cmd = "sudo bash ramspeed_mem_benchmark.sh %d %d %d %d" % \ (test_id, load, block_size, iteration) elif 0 < test_id <= 5: cmd = "sudo bash ramspeed_mark_benchmark.sh %d %d %d" % \ (test_id, load, block_size) # only the test_id 1-6 will be used in this scenario else: raise RuntimeError("No such type_id: %s for Ramspeed scenario", test_id) LOG.debug("Executing command: %s", cmd) status, stdout, stderr = self.client.execute(cmd) if status: raise RuntimeError(stderr) ramspeed_result = jsonutils.loads(stdout) result.update(utils.flatten_dict_key(ramspeed_result)) if "sla" in self.scenario_cfg: sla_error = "" sla_min_bw = int(self.scenario_cfg['sla']['min_bandwidth']) for i in ramspeed_result["Result"]: bw = i["Bandwidth(MBps)"] if bw < sla_min_bw: sla_error += "Bandwidth %f < " \ "sla:min_bandwidth(%f)" % (bw, sla_min_bw) assert sla_error == "", sla_error
def _data_to_line_protocol(self, data, case, criteria): msg = {} if not self.tags: self.tags = { 'deploy_scenario': os.environ.get('DEPLOY_SCENARIO', 'unknown'), 'installer': os.environ.get('INSTALLER_TYPE', 'unknown'), 'pod_name': os.environ.get('NODE_NAME', 'unknown'), 'version': os.environ.get('YARDSTICK_BRANCH', 'unknown') } point = { "measurement": case, "fields": utils.flatten_dict_key(data["data"]), "time": self._get_nano_timestamp(data), "tags": self._get_extended_tags(criteria), } msg["points"] = [point] msg["tags"] = self.tags return make_lines(msg).encode('utf-8') def _get_nano_timestamp(self, results): try: timestamp = results["timestamp"] except KeyError: timestamp = time.time() return str(int(float(timestamp) * 1000000000))
def run(self, result): """execute the benchmark""" if not self.setup_done: self.setup() # if run by a duration runner, get the duration time and setup as arg time = self.scenario_cfg["runner"].get("duration", None) \ if "runner" in self.scenario_cfg else None options = self.scenario_cfg['options'] cmd = "iperf3 -c %s --json" % (self.context_cfg['target']['ipaddr']) # If there are no options specified if not options: options = {} use_UDP = False try: protocol = options.get("protocol") bandwidth = options.get('bandwidth') use_UDP = protocol == 'udp' if protocol: cmd += " --" + protocol if use_UDP and bandwidth: cmd += " --bandwidth " + bandwidth # if nodelay in the option, protocal maybe null or 'tcp' if "nodelay" in options: cmd += " --nodelay" except AttributeError: LOG.warning("Can't parser the options in your config file!!!") # these options are mutually exclusive in iperf3 if time: cmd += " %d" % time elif "bytes" in options: # number of bytes to transmit (instead of --time) cmd += " --bytes %d" % options["bytes"] elif "blockcount" in options: cmd += " --blockcount %d" % options["blockcount"] if "length" in options: cmd += " --length %s" % options["length"] if "window" in options: cmd += " --window %s" % options["window"] LOG.debug("Executing command: %s", cmd) status, stdout, stderr = self.host.execute(cmd) if status: # error cause in json dict on stdout raise RuntimeError(stdout) # Note: convert all ints to floats in order to avoid # schema conflicts in influxdb. We probably should add # a format func in the future. iperf_result = jsonutils.loads(stdout, parse_int=float) result.update(utils.flatten_dict_key(iperf_result)) if "sla" in self.scenario_cfg: sla_iperf = self.scenario_cfg["sla"] if not use_UDP: sla_bytes_per_second = int(sla_iperf["bytes_per_second"]) # convert bits per second to bytes per second bit_per_second = \ int(iperf_result["end"]["sum_received"]["bits_per_second"]) bytes_per_second = bit_per_second / 8 assert bytes_per_second >= sla_bytes_per_second, \ "bytes_per_second %d < sla:bytes_per_second (%d); " % \ (bytes_per_second, sla_bytes_per_second) else: sla_jitter = float(sla_iperf["jitter"]) jitter_ms = float(iperf_result["end"]["sum"]["jitter_ms"]) assert jitter_ms <= sla_jitter, \ "jitter_ms %f > sla:jitter %f; " % \ (jitter_ms, sla_jitter)
def run(self, result): """execute the benchmark""" if not self.setup_done: self.setup() options = self.scenario_cfg['options'] test_type = options.get('test_type', 'latency') if test_type == 'latency': stride = options.get('stride', 128) stop_size = options.get('stop_size', 16.0) cmd = "sudo bash lmbench_latency.sh %f %d" % (stop_size, stride) elif test_type == 'bandwidth': size = options.get('size', 128) benchmark = options.get('benchmark', 'rd') warmup_repetitions = options.get('warmup', 0) cmd = "sudo bash lmbench_bandwidth.sh %d %s %d" % \ (size, benchmark, warmup_repetitions) elif test_type == 'latency_for_cache': repetition = options.get('repetition', 1) warmup = options.get('warmup', 0) cmd = "sudo bash lmbench_latency_for_cache.sh %d %d" % \ (repetition, warmup) else: raise RuntimeError("No such test_type: %s for Lmbench scenario" % test_type) LOG.debug("Executing command: %s", cmd) status, stdout, stderr = self.client.execute(cmd) if status: raise RuntimeError(stderr) lmbench_result = {} if test_type == 'latency': lmbench_result.update({"latencies": jsonutils.loads(stdout)}) else: lmbench_result.update(jsonutils.loads(stdout)) result.update(utils.flatten_dict_key(lmbench_result)) if "sla" in self.scenario_cfg: sla_error = "" if test_type == 'latency': sla_max_latency = int(self.scenario_cfg['sla']['max_latency']) for t_latency in lmbench_result["latencies"]: latency = t_latency['latency'] if latency > sla_max_latency: sla_error += "latency %f > sla:max_latency(%f); " \ % (latency, sla_max_latency) elif test_type == 'bandwidth': sla_min_bw = int(self.scenario_cfg['sla']['min_bandwidth']) bw = lmbench_result["bandwidth(MBps)"] if bw < sla_min_bw: sla_error += "bandwidth %f < " \ "sla:min_bandwidth(%f)" % (bw, sla_min_bw) elif test_type == 'latency_for_cache': sla_latency = float(self.scenario_cfg['sla']['max_latency']) cache_latency = float(lmbench_result['L1cache']) if sla_latency < cache_latency: sla_error += "latency %f > sla:max_latency(%f); " \ % (cache_latency, sla_latency) self.verify_SLA(sla_error == "", sla_error)