def main(): rest_args = sys.argv[1:] FailIfScreenLockedOnMac() parser = argparse.ArgumentParser(description='Extra argument parser', add_help=False) rest_args_filtered = ProcessArgs(rest_args, parser) retval = browser_test_runner.Run(gpu_project_config.CONFIG, rest_args_filtered) # We're not relying on argparse to print the help in the normal way, because # we need the help output from both the argument parser here and the argument # parser in browser_test_runner. if '--help' in rest_args: print('\n\nCommand line arguments handed by run_gpu_integration_test:') parser.print_help() return retval # This duplicates an argument of browser_test_runner. parser.add_argument( '--write-full-results-to', metavar='FILENAME', action='store', help=('If specified, writes the full results to that path.')) option, _ = parser.parse_known_args(rest_args) # Postprocess the outputted JSON to add test arguments. if option.write_run_test_arguments and option.write_full_results_to: PostprocessJSON(option.write_full_results_to, rest_args) return retval
def baseTest(self, mockInitDependencyManager, test_filter, failures, successes): options = browser_test_runner.TestRunOptions() options.verbosity = 0 config = project_config.ProjectConfig( top_level_dir=os.path.join(util.GetTelemetryDir(), 'examples'), client_configs=['a', 'b', 'c'], benchmark_dirs=[ os.path.join(util.GetTelemetryDir(), 'examples', 'browser_tests')] ) temp_file = tempfile.NamedTemporaryFile(delete=False) temp_file.close() temp_file_name = temp_file.name try: browser_test_runner.Run( config, options, ['SimpleTest', '--write-abbreviated-json-results-to=%s' % temp_file_name, '--test-filter=%s' % test_filter]) mockInitDependencyManager.assert_called_with(['a', 'b', 'c']) with open(temp_file_name) as f: test_result = json.load(f) self.assertEquals(test_result['failures'], failures) self.assertEquals(test_result['successes'], successes) self.assertEquals(test_result['valid'], True) finally: os.remove(temp_file_name)
def testSimpleIntegrationUnittest(self, mockInitDependencyManager): options = browser_test_runner.TestRunOptions() # Suppress printing out information for passing tests. options.verbosity = 0 config = gpu_project_config.CONFIG temp_file = tempfile.NamedTemporaryFile(delete=False) temp_file.close() temp_file_name = temp_file.name try: browser_test_runner.Run(config, options, [ 'simple_integration_unittest', '--write-abbreviated-json-results-to=%s' % temp_file_name ]) with open(temp_file_name) as f: test_result = json.load(f) self.assertEquals(test_result['failures'], ['unexpected_error', 'unexpected_failure']) self.assertEquals(test_result['successes'], ['expected_failure', 'expected_flaky']) self.assertEquals(test_result['valid'], True) # It might be nice to be more precise about the order of operations # with these browser restarts, but this is at least a start. self.assertEquals(SimpleIntegrationUnittest._num_browser_starts, 5) finally: os.remove(temp_file_name)
def testJsonOutputFormat(self, mockInitDependencyManager): options = browser_test_runner.TestRunOptions() config = project_config.ProjectConfig( top_level_dir=os.path.join(util.GetTelemetryDir(), 'examples'), client_configs=['a', 'b', 'c'], benchmark_dirs=[ os.path.join(util.GetTelemetryDir(), 'examples', 'browser_tests') ]) temp_file = tempfile.NamedTemporaryFile(delete=False) temp_file.close() temp_file_name = temp_file.name try: browser_test_runner.Run(config, options, [ 'SimpleTest', '--write-abbreviated-json-results-to=%s' % temp_file_name ]) mockInitDependencyManager.assert_called_with(['a', 'b', 'c']) with open(temp_file_name) as f: test_result = json.load(f) self.assertEquals(test_result['failures'], [ 'browser_tests.simple_numeric_test.SimpleTest.multiplier_simple_2', 'browser_tests.simple_numeric_test.SimpleTest.add_1_and_2', 'browser_tests.simple_numeric_test.SimpleTest.add_7_and_3', 'browser_tests.simple_numeric_test.SimpleTest.testSimple' ]) self.assertEquals(test_result['valid'], True) finally: os.remove(temp_file_name)
def _RunIntegrationTest(self, test_name, failures, successes, skips, additional_args): config = chromium_config.ChromiumConfig( top_level_dir=path_util.GetGpuTestDir(), benchmark_dirs=[ os.path.join(path_util.GetGpuTestDir(), 'unittest_data') ]) temp_dir = tempfile.mkdtemp() test_results_path = os.path.join(temp_dir, 'test_results.json') test_state_path = os.path.join(temp_dir, 'test_state.json') try: browser_test_runner.Run(config, [ test_name, '--write-full-results-to=%s' % test_results_path, '--test-state-json-path=%s' % test_state_path ] + additional_args) with open(test_results_path) as f: self._test_result = json.load(f) with open(test_state_path) as f: self._test_state = json.load(f) actual_successes, actual_failures, actual_skips = ( _ExtractTestResults(self._test_result)) self.assertEquals(set(actual_failures), set(failures)) self.assertEquals(set(actual_successes), set(successes)) self.assertEquals(set(actual_skips), set(skips)) finally: shutil.rmtree(temp_dir)
def baseShardingTest(self, total_shards, shard_index, failures, successes): options = browser_test_runner.TestRunOptions() options.verbosity = 0 config = project_config.ProjectConfig( top_level_dir=os.path.join(util.GetTelemetryDir(), 'examples'), client_configs=['a', 'b', 'c'], benchmark_dirs=[ os.path.join(util.GetTelemetryDir(), 'examples', 'browser_tests')] ) temp_file = tempfile.NamedTemporaryFile(delete=False) temp_file.close() temp_file_name = temp_file.name try: browser_test_runner.Run( config, options, ['SimpleShardingTest', '--write-abbreviated-json-results-to=%s' % temp_file_name, '--total-shards=%d' % total_shards, '--shard-index=%d' % shard_index]) with open(temp_file_name) as f: test_result = json.load(f) self.assertEquals(test_result['failures'], failures) self.assertEquals(test_result['successes'], successes) self.assertEquals(test_result['valid'], True) finally: os.remove(temp_file_name)
def _RunTest(self, test_filter, expected_failures, expected_successes, expected_skips=None, test_name='SimpleTest', extra_args=None): expected_skips = expected_skips or [] extra_args = extra_args or [] config = project_config.ProjectConfig( top_level_dir=os.path.join(util.GetTelemetryDir(), 'examples'), client_configs=[], benchmark_dirs=[ os.path.join(util.GetTelemetryDir(), 'examples', 'browser_tests') ]) temp_file = tempfile.NamedTemporaryFile(delete=False) temp_file.close() temp_file_name = temp_file.name try: browser_test_runner.Run(config, [ test_name, '--write-full-results-to=%s' % temp_file_name, '--test-filter=%s' % test_filter ] + extra_args) with open(temp_file_name) as f: self._test_result = json.load(f) (actual_successes, actual_failures, actual_skips) = self._ExtractTestResults(self._test_result) self.assertEquals(set(actual_failures), set(expected_failures)) self.assertEquals(set(actual_successes), set(expected_successes)) self.assertEquals(set(actual_skips), set(expected_skips)) finally: os.remove(temp_file_name)
def main(): options = browser_test_runner.TestRunOptions() config = project_config.ProjectConfig( top_level_dir=os.path.dirname(__file__), benchmark_dirs=[ os.path.join(os.path.dirname(__file__), 'browser_tests') ]) return browser_test_runner.Run(config, options, sys.argv[1:])
def main(): FailIfScreenLockedOnMac() rest_args = sys.argv[1:] parser = argparse.ArgumentParser(description='Extra argument parser', add_help=False) parser.add_argument( '--write-run-test-arguments', action='store_true', help=('Write the test script arguments to the results file.')) option, rest_args_filtered = parser.parse_known_args(rest_args) parser.add_argument('test', nargs='*', type=str, help=argparse.SUPPRESS) option, _ = parser.parse_known_args(rest_args_filtered) if option.test: test_class = FindTestCase(option.test[0]) else: test_class = None if test_class: rest_args_filtered.extend([ '--test-name-prefix=%s.%s.' % (test_class.__module__, test_class.__name__) ]) if not any(arg.startswith('--retry-limit') for arg in rest_args_filtered): if '--retry-only-retry-on-failure-tests' not in rest_args_filtered: rest_args_filtered.append('--retry-only-retry-on-failure-tests') rest_args_filtered.append('--retry-limit=2') rest_args_filtered.extend( ['--repository-absolute-path', path_util.GetChromiumSrcDir()]) retval = browser_test_runner.Run(gpu_project_config.CONFIG, rest_args_filtered) # We're not relying on argparse to print the help in the normal way, because # we need the help output from both the argument parser here and the argument # parser in browser_test_runner. if '--help' in rest_args: print('\n\nCommand line arguments handed by run_gpu_integration_test:') parser.print_help() return retval # This duplicates an argument of browser_test_runner. parser.add_argument( '--write-full-results-to', metavar='FILENAME', action='store', help=('If specified, writes the full results to that path.')) option, _ = parser.parse_known_args(rest_args) # Postprocess the outputted JSON to add test arguments. if option.write_run_test_arguments and option.write_full_results_to: PostprocessJSON(option.write_full_results_to, rest_args) return retval
def main(): rest_args = sys.argv[1:] retval = browser_test_runner.Run(gpu_project_config.CONFIG, rest_args) # Postprocess the outputted JSON to trim all of the prefixes from # the test names, to keep them as similar to the old form as # possible -- and keep them from getting crazily long. parser = argparse.ArgumentParser(description='Temporary argument parser') parser.add_argument('--write-abbreviated-json-results-to', metavar='FILENAME', action='store', help=('Full path for json results')) option, _ = parser.parse_known_args(rest_args) if option.write_abbreviated_json_results_to: PostprocessJSON(option.write_abbreviated_json_results_to, rest_args) return retval
def _RunBrowserTest(self, modulename, classname, test_name, expectation='Pass', test_tags='foo', extra_args=None, expected_exit_code=0, include_expectations=True): extra_args = extra_args or [] if include_expectations: expectations = ('# tags: [ foo bar mac ]\n' 'crbug.com/123 [ %s ] ' 'browser_tests.%s.%s.%s' ' [ %s ]') expectations = expectations % (test_tags, modulename, classname, test_name, expectation) expectations_file = tempfile.NamedTemporaryFile(delete=False) expectations_file.write(expectations) expectations_file.close() expectations_file_paths = [expectations_file.name] else: expectations_file_paths = [] results = tempfile.NamedTemporaryFile(delete=False) results.close() config = project_config.ProjectConfig( top_level_dir=os.path.join(util.GetTelemetryDir(), 'examples'), client_configs=[], expectations_files=expectations_file_paths, benchmark_dirs=[ os.path.join(util.GetTelemetryDir(), 'examples', 'browser_tests') ]) try: ret = browser_test_runner.Run(config, [ '%s' % classname, ('--write-full-results-to=%s' % results.name), ('--test-filter=.*%s.*' % test_name) ] + extra_args) self.assertEqual(ret, expected_exit_code) with open(results.name) as f: test_result = json.load(f) finally: if expectations_file_paths: os.remove(expectations_file.name) os.remove(results.name) return test_result
def BaseShardingTest(self, total_shards, shard_index, failures, successes, opt_abbr_input_json_file=None, opt_test_filter='', opt_filter_tests_after_sharding=False, opt_test_name_prefix=''): config = project_config.ProjectConfig( top_level_dir=os.path.join(util.GetTelemetryDir(), 'examples'), client_configs=[], benchmark_dirs=[ os.path.join(util.GetTelemetryDir(), 'examples', 'browser_tests') ]) temp_file = tempfile.NamedTemporaryFile(delete=False) temp_file.close() temp_file_name = temp_file.name opt_args = [] if opt_abbr_input_json_file: opt_args += [ '--read-abbreviated-json-results-from=%s' % opt_abbr_input_json_file ] if opt_test_filter: opt_args += ['--test-filter=%s' % opt_test_filter] if opt_filter_tests_after_sharding: opt_args += ['--filter-tests-after-sharding'] if opt_test_name_prefix: opt_args += ['--test-name-prefix=%s' % opt_test_name_prefix] try: browser_test_runner.Run(config, [ 'SimpleShardingTest', '--write-full-results-to=%s' % temp_file_name, '--total-shards=%d' % total_shards, '--shard-index=%d' % shard_index ] + opt_args) with open(temp_file_name) as f: test_result = json.load(f) (actual_successes, actual_failures, _) = self._ExtractTestResults(test_result) self.assertEquals(set(actual_failures), set(failures)) self.assertEquals(set(actual_successes), set(successes)) finally: os.remove(temp_file_name)
def main(): FailIfScreenLockedOnMac() rest_args = sys.argv[1:] parser = argparse.ArgumentParser(description='Extra argument parser', add_help=False) parser.add_argument( '--write-run-test-arguments', action='store_true', help=('Write the test script arguments to the results file.')) option, rest_args_filtered = parser.parse_known_args(rest_args) parser.add_argument('test', type=str, help='Name of the test suite to run') option, _ = parser.parse_known_args(rest_args_filtered) test_class = FindTestCase(option) assert test_class rest_args_filtered.extend([ '--test-name-prefix=%s.%s.' % (test_class.__module__, test_class.__name__) ]) retval = browser_test_runner.Run(gpu_project_config.CONFIG, rest_args_filtered) # We're not relying on argparse to print the help in the normal way, because # we need the help output from both the argument parser here and the argument # parser in browser_test_runner. if '--help' in rest_args: print '\n\nCommand line arguments handed by run_gpu_integration_test:' parser.print_help() return retval # This duplicates an argument of browser_test_runner. parser.add_argument( '--write-full-results-to', metavar='FILENAME', action='store', help=('If specified, writes the full results to that path.')) option, _ = parser.parse_known_args(rest_args) # Postprocess the outputted JSON to add test arguments. if option.write_run_test_arguments and option.write_full_results_to: PostprocessJSON(option.write_full_results_to, rest_args) return retval
def _RunIntegrationTest(self, test_name, failures, successes): options = browser_test_runner.TestRunOptions() # Suppress printing out information for passing tests. options.verbosity = 0 config = gpu_project_config.CONFIG temp_file = tempfile.NamedTemporaryFile(delete=False) temp_file.close() temp_file_name = temp_file.name try: browser_test_runner.Run( config, options, [test_name, '--write-abbreviated-json-results-to=%s' % temp_file_name]) with open(temp_file_name) as f: test_result = json.load(f) self.assertEquals(test_result['failures'], failures) self.assertEquals(test_result['successes'], successes) self.assertEquals(test_result['valid'], True) finally: os.remove(temp_file_name)
def _RunBrowserTest(self, modulename, classname, test_name, expectation, test_tags='foo'): expectations = ('# tags: [ foo bar mac ]\n' 'crbug.com/123 [ %s ] ' 'browser_tests.%s.%s.%s' ' [ %s ]') expectations = expectations % (test_tags, modulename, classname, test_name, expectation) expectations_file = tempfile.NamedTemporaryFile(delete=False) expectations_file.write(expectations) results = tempfile.NamedTemporaryFile(delete=False) results.close() expectations_file.close() config = project_config.ProjectConfig( top_level_dir=os.path.join(util.GetTelemetryDir(), 'examples'), client_configs=[], expectations_files=[expectations_file.name], benchmark_dirs=[ os.path.join(util.GetTelemetryDir(), 'examples', 'browser_tests') ]) try: browser_test_runner.Run(config, [ '%s' % classname, '--write-full-results-to=%s' % results.name, '--test-filter=.*%s.*' % test_name ]) with open(results.name) as f: test_result = json.load(f) finally: os.remove(expectations_file.name) os.remove(results.name) return test_result
def main(args): parser = argparse.ArgumentParser(description='Extra argument parser', add_help=False) parser.add_argument( '--output-directory', action='store', default=None, help='Sets the CHROMIUM_OUTPUT_DIR environment variable') known_options, rest_args = parser.parse_known_args(args) constants.SetOutputDirectory( os.path.realpath(known_options.output_directory or os.getcwd())) config = chromium_config.ChromiumConfig( top_level_dir=os.path.dirname(__file__), benchmark_dirs=[os.path.dirname(__file__)]) ret_val = browser_test_runner.Run(config, rest_args) if '--help' in rest_args or '-h' in rest_args: print('\n\nCommand line arguments used in ' 'run_webview_component_smoketest.py') parser.print_help() return ret_val
def main(): options = browser_test_runner.TestRunOptions() return browser_test_runner.Run(gpu_project_config.CONFIG, options, sys.argv[1:])
def main(args): config = chromium_config.ChromiumConfig( top_level_dir=os.path.dirname(__file__), benchmark_dirs=[os.path.dirname(__file__)]) return browser_test_runner.Run(config, args)
def main(): retval = browser_test_runner.Run(cast_project_config.CONFIG, sys.argv) return retval