def ComparePerfViaSingleFile(self, before_dir, after_dir): # Convert to the single-file JSON format. json_file = os.path.join(self.MakeTempDir(), 'dataset.json') with open(json_file, 'w') as fh: perfcompare.Main( ['make_combined_perf_dataset_file', before_dir, after_dir], fh) # Run compare_perf on the JSON file. stdout = StringIO.StringIO() perfcompare.Main(['compare_perf', '--dataset_file', json_file], stdout) return stdout.getvalue()
def test_display_single_boot_two_datasets(self): dataset_dirs = [ self.ExampleDataDir(mean=1000, single_boot=True), self.ExampleDataDir(mean=2000, single_boot=True, drop_one=True) ] stdout = StringIO.StringIO() perfcompare.Main(['compare_perf'] + dataset_dirs, stdout) output = stdout.getvalue() GOLDEN.AssertCaseEq('display_single_boot_two_datasets', output)
def test_validate_perfcompare(self): # This is an example input dataset that gives a high mismatch rate, # because the data is drawn from two very different distributions. results_dirs = ([self.ExampleDataDir(mean=100, stddev=10)] * 10 + [self.ExampleDataDir(mean=200, stddev=10)] * 10) stdout = StringIO.StringIO() perfcompare.Main(['validate_perfcompare'] + results_dirs, stdout) output = stdout.getvalue() GOLDEN.AssertCaseEq('validate_perfcompare', output)
def ComparePerf(self, before_dir, after_dir): stdout = StringIO.StringIO() perfcompare.Main(['compare_perf', before_dir, after_dir], stdout) output = stdout.getvalue() # Check that going via the single-file JSON format produces the # same result. self.assertEquals(self.ComparePerfViaSingleFile(before_dir, after_dir), output) return output
def test_error_if_dest_files_already_exist(self): dest_dir = os.path.join(self.MakeTempDir(), 'new_dir') iter_temp_file = os.path.join(self.MakeTempDir(), 'result.fuchsiaperf.json') WriteJsonFile(iter_temp_file, []) args = [ 'run_local', '--boots=4', '--iter_file', iter_temp_file, '--iter_cmd', 'my_iter_cmd', '--reboot_cmd', 'my_reboot_cmd', '--dest', dest_dir ] self.assertRaises(AssertionError, lambda: perfcompare.Main(args, sys.stdout))
def test_errexit_error_checking_in_shell_commands(self): iter_temp_file = os.path.join(self.MakeTempDir(), 'result.fuchsiaperf.json') stdout = StringIO.StringIO() def get_args(): dest_dir = os.path.join(self.MakeTempDir(), 'new_dir') return [ 'run_local', '--boots=4', '--iter_file', iter_temp_file, '--dest', dest_dir ] perfcompare.Main( get_args() + ['--iter_cmd', 'true', '--reboot_cmd', 'true'], stdout) # Check that the failure of the "false" command gets caught. self.assertRaises( subprocess.CalledProcessError, lambda: perfcompare.Main( get_args() + ['--iter_cmd', 'false; true', '--reboot_cmd', 'true'], stdout)) self.assertRaises( subprocess.CalledProcessError, lambda: perfcompare.Main( get_args() + ['--iter_cmd', 'true', '--reboot_cmd', 'false; true'], stdout))
def test_validate_perfcompare(self): def MakeExampleDirs(**kwargs): by_boot_dir = os.path.join(self.ExampleDataDir(**kwargs), 'by_boot') return [os.path.join(by_boot_dir, name) for name in sorted(os.listdir(by_boot_dir))] # This is an example input dataset that gives a high mismatch rate, # because the data is drawn from two very different distributions. results_dirs = (MakeExampleDirs(mean=100, stddev=10) + MakeExampleDirs(mean=200, stddev=10)) stdout = StringIO.StringIO() perfcompare.Main(['validate_perfcompare', '--group_size=5'] + results_dirs, stdout) output = stdout.getvalue() GOLDEN.AssertCaseEq('validate_perfcompare', output)
def test_run_local(self): # Destination directory for the full multiboot dataset. Use a # destination path that does not exist yet. dest_dir = os.path.join(self.MakeTempDir(), 'new_dir') # Destination pathnames for process dataset files. iter_temp_dir = self.MakeTempDir() iter_temp_file = os.path.join(iter_temp_dir, 'result.fuchsiaperf.json') iter_temp_glob = os.path.join(iter_temp_dir, '*.fuchsiaperf.json') data = GenerateData(mean=1000, stddev_across_boots=10, stddev_across_processes=10, stddev_across_iters=10) commands = [] # Dummy version of subprocess.check_call() for testing. def DummyRunCmd(cmd, shell=False): self.assertEquals(shell, True) commands.append(cmd) if cmd == 'set -o errexit -o nounset; my_iter_cmd': WriteJsonFile(iter_temp_file, [{ 'label': 'MyTest', 'test_suite': 'example_suite', 'unit': 'nanoseconds', 'values': data.pop(0)[0] }]) stdout = StringIO.StringIO() perfcompare.Main([ 'run_local', '--boots=4', '--iter_file', iter_temp_glob, '--iter_cmd', 'my_iter_cmd', '--reboot_cmd', 'my_reboot_cmd', '--dest', dest_dir ], stdout, run_cmd=DummyRunCmd) self.assertEquals(commands, [ 'set -o errexit -o nounset; my_reboot_cmd', 'set -o errexit -o nounset; my_iter_cmd' ] * 4) GOLDEN.AssertCaseEq('run_local', stdout.getvalue())
def ComparePerf(self, before_dir, after_dir): stdout = StringIO.StringIO() perfcompare.Main(['compare_perf', before_dir, after_dir], stdout) return stdout.getvalue()
def test_comparing_equal_zero_width_confidence_intervals(self): dir_path = self.DirOfData([[[200]], [[200]]]) stdout = StringIO.StringIO() perfcompare.Main(['compare_perf', dir_path, dir_path], stdout) output = stdout.getvalue() GOLDEN.AssertCaseEq('comparison_no_change_zero_width_ci', output)
def test_display_single_boot_single_dataset(self): dataset_dir = self.ExampleDataDir(single_boot=True) stdout = StringIO.StringIO() perfcompare.Main(['compare_perf', dataset_dir], stdout) output = stdout.getvalue() GOLDEN.AssertCaseEq('display_single_boot_single_dataset', output)