def test_check_and_parse_execution_results(): stdout = "[0.000403]" stderr = "[iterations_runner.py] iteration 1/1" assert check_and_parse_execution_results(stdout, stderr, 0) == json.loads(stdout) # Non-zero return code. with pytest.raises(ExecutionFailed) as excinfo: check_and_parse_execution_results(stdout, stderr, 1) expected = """Benchmark returned non-zero or didn't emit JSON list. return code: 1 stdout: -------------------------------------------------- [0.000403] -------------------------------------------------- stderr: -------------------------------------------------- [iterations_runner.py] iteration 1/1 -------------------------------------------------- """ assert excinfo.value.message == expected # Corrupt Json in STDOUT. with pytest.raises(ExecutionFailed) as excinfo: check_and_parse_execution_results("[0.000403[", stderr, 0) expected = """Benchmark returned non-zero or didn't emit JSON list. Exception string: Expecting , delimiter: line 1 column 10 (char 9) return code: 0 stdout: -------------------------------------------------- [0.000403[ -------------------------------------------------- stderr: -------------------------------------------------- [iterations_runner.py] iteration 1/1 -------------------------------------------------- """ assert excinfo.value.message == expected
def test_check_and_parse_execution_results0005(caplog): stdout = json.dumps({ "wallclock_times": [.5, .5], "core_cycle_counts": [[1000, 500], [50, 50], [30, 30], [40, 40]], "aperf_counts": [[20, 15], [5, 5], [5, 5], [5, 5]], "mperf_counts": [[20, 20], [5, 5], [5, 5], [5, 5]], }) stderr = "[iterations_runner.py] iteration 1/1" with pytest.raises(RerunExecution) as e: check_and_parse_execution_results(stdout, stderr, 0, DUMMY_CONFIG) assert e.value.message == \ "APERF/MPERF ratio badness detected\n" \ " in_proc_iter=1, core=0, type=throttle, ratio=0.75\n\n" \ "The process execution will be retried until the ratios are OK."
def test_check_and_parse_execution_results0005(caplog): stdout = json.dumps({ "wallclock_times": [.5, .5], "core_cycle_counts": [[1000, 500], [50, 50], [30, 30], [40, 40]], "aperf_counts": [[20, 15], [5, 5], [5, 5], [5, 5]], "mperf_counts": [[20, 20], [5, 5], [5, 5], [5, 5]], }) stderr = "[iterations_runner.py] iteration 1/1" with pytest.raises(RerunExecution) as e: check_and_parse_execution_results(stdout, stderr, 0, DUMMY_CONFIG) assert e.value.message == \ "APERF/MPERF ratio badness detected\n" \ " in_proc_iter=1, core=0, type=throttle, ratio=0.75\n\n" \ "The process execution will be retried until the ratios are OK."
def run(self, mailer, dry_run=False): """Runs this job (execution)""" entry_point = self.sched.config.VARIANTS[self.variant] vm_def = self.vm_info["vm_def"] vm_def.dry_run = dry_run # Set heap limit heap_limit_kb = self.sched.config.HEAP_LIMIT stack_limit_kb = self.sched.config.STACK_LIMIT stdout, stderr, rc = vm_def.run_exec(entry_point, self.benchmark, self.vm_info["n_iterations"], self.parameter, heap_limit_kb, stack_limit_kb) if not dry_run: try: iterations_results = util.check_and_parse_execution_results( stdout, stderr, rc) except util.ExecutionFailed as e: util.log_and_mail(mailer, error, "Benchmark failure: %s" % self.key, e.message) iterations_results = [] else: iterations_results = [] # We print the status *after* benchmarking, so that I/O cannot be # commited during benchmarking. In production, we will be rebooting # before the next execution, so we are grand. info("Finished '%s(%d)' (%s variant) under '%s'" % (self.benchmark, self.parameter, self.variant, self.vm_name)) return iterations_results
def run(self, mailer, dry_run=False): """Runs this job (execution)""" entry_point = self.sched.config.VARIANTS[self.variant] vm_def = self.vm_info["vm_def"] vm_def.dry_run = dry_run # Set heap limit heap_limit_kb = self.sched.config.HEAP_LIMIT stack_limit_kb = self.sched.config.STACK_LIMIT stdout, stderr, rc = vm_def.run_exec( entry_point, self.benchmark, self.vm_info["n_iterations"], self.parameter, heap_limit_kb, stack_limit_kb) if not dry_run: try: iterations_results = util.check_and_parse_execution_results(stdout, stderr, rc) except util.ExecutionFailed as e: util.log_and_mail(mailer, error, "Benchmark failure: %s" % self.key, e.message) iterations_results = [] else: iterations_results = [] # We print the status *after* benchmarking, so that I/O cannot be # commited during benchmarking. In production, we will be rebooting # before the next execution, so we are grand. info("Finished '%s(%d)' (%s variant) under '%s'" % (self.benchmark, self.parameter, self.variant, self.vm_name)) return iterations_results
def test_check_and_parse_execution_results0004(caplog): stdout = json.dumps({ "wallclock_times": [1], "core_cycle_counts": [[1], [1], [1], [1]], "aperf_counts": [[30], [20], [5], [3]], "mperf_counts": [[20], [30], [5], [5]], }) stderr = "[iterations_runner.py] iteration 1/1" with pytest.raises(RerunExecution) as e: check_and_parse_execution_results(stdout, stderr, 0, DUMMY_CONFIG, "a:b:c") assert e.value.args[0] == \ "APERF/MPERF ratio badness detected\n" \ " in_proc_iter=0, core=0, type=turbo, ratio=1.5\n" \ " in_proc_iter=0, core=1, type=throttle, ratio=0.666666666667\n\n" \ "The process execution will be retried until the ratios are OK."
def test_check_and_parse_execution_results0003(): stderr = "[iterations_runner.py] iteration 1/1" stdout = "[0.000403[" # Corrupt JSON in STDOUT. with pytest.raises(ExecutionFailed) as excinfo: check_and_parse_execution_results(stdout, stderr, 0, DUMMY_CONFIG) expected = """Benchmark returned non-zero or emitted invalid JSON.\nException string: Expecting , delimiter: line 1 column 10 (char 9) return code: 0 stdout: -------------------------------------------------- [0.000403[ -------------------------------------------------- stderr: -------------------------------------------------- [iterations_runner.py] iteration 1/1 -------------------------------------------------- """ assert excinfo.value.message == expected
def test_check_and_parse_execution_results0001(): stdout = json.dumps({ "wallclock_times": [1], "core_cycle_counts": [[1], [2], [3], [4]], "aperf_counts": [[5], [5], [5], [5]], "mperf_counts": [[5], [5], [5], [5]], }) stderr = "[iterations_runner.py] iteration 1/1" js = check_and_parse_execution_results(stdout, stderr, 0, DUMMY_CONFIG) assert js == json.loads(stdout)
def test_check_and_parse_execution_results0003(): stderr = "[iterations_runner.py] iteration 1/1" stdout = "[0.000403[" # Corrupt JSON in STDOUT. with pytest.raises(ExecutionFailed) as excinfo: check_and_parse_execution_results(stdout, stderr, 0, DUMMY_CONFIG) expected = """Benchmark returned non-zero or emitted invalid JSON.\nException string: Expecting , delimiter: line 1 column 10 (char 9) return code: 0 stdout: -------------------------------------------------- [0.000403[ -------------------------------------------------- stderr: -------------------------------------------------- [iterations_runner.py] iteration 1/1 -------------------------------------------------- """ assert excinfo.value.message == expected
def test_check_and_parse_execution_results0001(): stdout = json.dumps({ "wallclock_times": [1], "core_cycle_counts": [[1], [2], [3], [4]], "aperf_counts": [[5], [5], [5], [5]], "mperf_counts": [[5], [5], [5], [5]], }) stderr = "[iterations_runner.py] iteration 1/1" js = check_and_parse_execution_results(stdout, stderr, 0, DUMMY_CONFIG) assert js == json.loads(stdout)
def test_check_and_parse_execution_results(): stdout = "[0.000403]" stderr = "[iterations_runner.py] iteration 1/1" assert check_and_parse_execution_results(stdout, stderr, 0) == json.loads(stdout) # Non-zero return code. with pytest.raises(ExecutionFailed) as excinfo: check_and_parse_execution_results(stdout, stderr, 1) expected = """Benchmark returned non-zero or didn't emit JSON list. return code: 1 stdout: -------------------------------------------------- [0.000403] -------------------------------------------------- stderr: -------------------------------------------------- [iterations_runner.py] iteration 1/1 -------------------------------------------------- """ assert excinfo.value.message == expected # Corrupt Json in STDOUT. with pytest.raises(ExecutionFailed) as excinfo: check_and_parse_execution_results("[0.000403[", stderr, 0) expected = """Benchmark returned non-zero or didn't emit JSON list. Exception string: Expecting , delimiter: line 1 column 10 (char 9) return code: 0 stdout: -------------------------------------------------- [0.000403[ -------------------------------------------------- stderr: -------------------------------------------------- [iterations_runner.py] iteration 1/1 -------------------------------------------------- """ assert excinfo.value.message == expected
def test_check_and_parse_execution_results0002(): stdout = json.dumps({ "wallclock_times": [123.4], "core_cycle_counts": [[1], [2], [3], [4]], "aperf_counts": [[5], [6], [7], [8]], "mperf_counts": [[9], [10], [11], [12]], }) stderr = "[iterations_runner.py] iteration 1/1" # Non-zero return code. with pytest.raises(ExecutionFailed) as excinfo: check_and_parse_execution_results(stdout, stderr, 1, DUMMY_CONFIG) expected = """Benchmark returned non-zero or emitted invalid JSON. return code: 1 stdout: -------------------------------------------------- %s -------------------------------------------------- stderr: -------------------------------------------------- [iterations_runner.py] iteration 1/1 -------------------------------------------------- """ % stdout assert excinfo.value.message == expected
def test_check_and_parse_execution_results0002(): stdout = json.dumps({ "wallclock_times": [123.4], "core_cycle_counts": [[1], [2], [3], [4]], "aperf_counts": [[5], [6], [7], [8]], "mperf_counts": [[9], [10], [11], [12]], }) stderr = "[iterations_runner.py] iteration 1/1" # Non-zero return code. with pytest.raises(ExecutionFailed) as excinfo: check_and_parse_execution_results(stdout, stderr, 1, DUMMY_CONFIG) expected = """Benchmark returned non-zero or emitted invalid JSON. return code: 1 stdout: -------------------------------------------------- %s -------------------------------------------------- stderr: -------------------------------------------------- [iterations_runner.py] iteration 1/1 -------------------------------------------------- """ % stdout assert excinfo.value.message == expected
def run(self, mailer, dry_run=False): """Runs this job (execution)""" flag = None entry_point = self.sched.config.VARIANTS[self.variant] vm_def = self.vm_info["vm_def"] vm_def.dry_run = dry_run # Set heap limit heap_limit_kb = self.sched.config.HEAP_LIMIT stack_limit_kb = self.sched.config.STACK_LIMIT in_proc_iters = self.vm_info["n_iterations"] if not dry_run: self.sched.platform.collect_starting_throttle_counts() stdout, stderr, rc, envlog_filename, timed_out = \ vm_def.run_exec(entry_point, in_proc_iters, self.parameter, heap_limit_kb, stack_limit_kb, self.key, self.key_pexec_idx) if timed_out: measurements = self.empty_measurements instr_data = {} flag = "T" elif not dry_run: try: self.sched.platform.check_throttle_counts(self.sched.manifest) measurements = util.check_and_parse_execution_results( stdout, stderr, rc, self.sched.config, self.key, instrument=vm_def.instrument) flag = "C" except util.RerunExecution as e: subject = ("Benchmark needs to be re-run: %s (exec_idx=%s)" % (self.key, self.sched.manifest.next_exec_idx)) util.log_and_mail(mailer, warn, subject, e.message, manifest=self.sched.manifest, bypass_limiter=True) measurements = self.empty_measurements flag = "O" # i.e. still outstanding except util.ExecutionFailed as e: util.log_and_mail(mailer, error, "Benchmark failure: %s" % self.key, e.message, manifest=self.sched.manifest) measurements = self.empty_measurements flag = "E" # Collect instrumentation data if vm_def.instrument and flag == "C": instr_data = vm_def.get_instr_data() for k, v in instr_data.iteritems(): assert len(instr_data[k]) == in_proc_iters else: # The benchmark either failed, needs to be re-run, or had # instrumentation turned off. instr_data = {} else: measurements = self.empty_measurements instr_data = {} flag = "C" # We print the status *after* benchmarking, so that I/O cannot be # committed during benchmarking. In production, we will be rebooting # before the next execution, so we are grand. info("Finished '%s(%d)' (%s variant) under '%s'" % (self.benchmark, self.parameter, self.variant, self.vm_name)) # Move the environment log out of /tmp. # # We don't do this for re-runs (O) as the log for the re-run pexec is # the one we want. # # We don't do this for timeouts (T) because the wrapper script is # killed upon timeout, and thus doesn't get a chance to log the # environment. if not dry_run and flag not in ("O", "T"): key_exec_num = self.sched.manifest.completed_exec_counts[self.key] util.stash_envlog(envlog_filename, self.sched.config, self.sched.platform, self.key, key_exec_num) assert flag is not None return measurements, instr_data, flag