def testAllArguments(self): arguments = dict(_BASE_ARGUMENTS) arguments['extra_test_args'] = '["--custom-arg", "custom value"]' quest = run_test.RunTest.FromDict(arguments) extra_args = ['--custom-arg', 'custom value'] expected = run_test.RunTest('server', [{'key': 'value'}], extra_args) self.assertEqual(quest, expected)
def testStringExtraTestArgs(self): arguments = dict(_BASE_ARGUMENTS) arguments['extra_test_args'] = '--custom-arg "custom value"' quest = run_test.RunTest.FromDict(arguments) extra_args = ['--custom-arg', 'custom value'] + run_test._DEFAULT_EXTRA_ARGS expected = run_test.RunTest('server', [{'key': 'value'}], extra_args) self.assertEqual(quest, expected)
def testMinimumArguments(self): arguments = { 'configuration': 'chromium-rel-mac11-pro', 'target': 'net_perftests', } expected = run_test.RunTest({'key': 'value'}, _MIN_GTEST_ARGUMENTS) self.assertEqual(run_test.RunTest.FromDict(arguments), expected)
def testStart_NoSwarmingTags(self, swarming_task_result, swarming_tasks_new): del swarming_task_result del swarming_tasks_new quest = run_test.RunTest('server', DIMENSIONS, ['arg'], None, None, None) quest.Start('change_1', 'isolate server', 'input isolate hash')
def testStringDimensions(self): arguments = dict(_BASE_ARGUMENTS) arguments['dimensions'] = '[{"key": "value"}]' quest = run_test.RunTest.FromDict(arguments) expected = run_test.RunTest('server', [{ 'key': 'value' }], run_test._DEFAULT_EXTRA_ARGS) self.assertEqual(quest, expected)
def testSuccess(self, swarming_task_result, swarming_tasks_new): # Goes through a full run of two Executions. # Call RunTest.Start() to create an Execution. quest = run_test.RunTest(_SWARMING_DIMENSIONS, _SWARMING_EXTRA_ARGS) execution = quest.Start('input isolate hash') swarming_task_result.assert_not_called() swarming_tasks_new.assert_not_called() # Call the first Poll() to start the swarming task. swarming_tasks_new.return_value = {'task_id': 'task id'} execution.Poll() swarming_task_result.assert_not_called() swarming_tasks_new.assert_called_once() self.assertNewTaskHasDimensions(swarming_tasks_new) self.assertFalse(execution.completed) self.assertFalse(execution.failed) # Call subsequent Poll()s to check the task status. swarming_task_result.return_value = {'state': 'PENDING'} execution.Poll() self.assertFalse(execution.completed) self.assertFalse(execution.failed) swarming_task_result.return_value = { 'bot_id': 'bot id', 'exit_code': 0, 'failure': False, 'outputs_ref': {'isolated': 'output isolate hash'}, 'state': 'COMPLETED', } execution.Poll() self.assertTrue(execution.completed) self.assertFalse(execution.failed) self.assertEqual(execution.result_values, (None,)) self.assertEqual(execution.result_arguments, {'isolate_hashes': ('output isolate hash',)}) self.assertEqual( { 'bot_ids': ['bot id'], 'input_isolate_hash': 'input isolate hash', 'task_ids': ['task id'], 'result_arguments': {'isolate_hashes': ('output isolate hash',)}, 'result_values': (None,), }, execution.AsDict()) # Start a second Execution to check bot_id handling. We get a bot_id from # Swarming from the first Execution and reuse it in subsequent Executions. execution = quest.Start('input isolate hash') execution.Poll() self.assertNewTaskHasBotId(swarming_tasks_new)
def testMinimumArguments(self): arguments = { 'configuration': 'chromium-rel-mac11-pro', 'target': 'telemetry_perf_tests', 'benchmark': 'speedometer', } expected = run_test.RunTest({'key': 'value'}, _MIN_TELEMETRY_ARGUMENTS) self.assertEqual(run_test.RunTest.FromDict(arguments), expected)
def testStringExtraTestArgs(self): arguments = dict(_BASE_ARGUMENTS) arguments['extra_test_args'] = '--custom-arg "custom value"' quest = run_test.RunTest.FromDict(arguments) extra_args = ['--custom-arg', 'custom value'] expected = run_test.RunTest('server', DIMENSIONS, extra_args, _BASE_SWARMING_TAGS) self.assertEqual(quest, expected)
def testWebviewFlag(self): arguments = { 'configuration': 'chromium-rel-mac11-pro', 'target': 'telemetry_perf_webview_tests', 'benchmark': 'speedometer', 'browser': 'android-webview', } expected = run_test.RunTest({'key': 'value'}, _WEBVIEW_ARGUMENTS) self.assertEqual(run_test.RunTest.FromDict(arguments), expected)
def testStartupBenchmarkRepeatCount(self): arguments = { 'configuration': 'chromium-rel-mac11-pro', 'target': 'telemetry_perf_tests', 'benchmark': 'start_with_url.warm.startup_pages', } expected = run_test.RunTest({'key': 'value'}, _STARTUP_BENCHMARK_ARGUMENTS) self.assertEqual(run_test.RunTest.FromDict(arguments), expected)
def testWebviewFlag(self): arguments = { 'target': 'telemetry_perf_webview_tests', 'dimensions': '{"key": "value"}', 'benchmark': 'speedometer', 'browser': 'android-webview', } expected = run_test.RunTest({'key': 'value'}, _WEBVIEW_ARGUMENTS) self.assertEqual(run_test.RunTest.FromDict(arguments), expected)
def testPagesetRepeat(self, swarming_tasks_new): quest = run_test.RunTest('chromium-rel-mac11-pro', 'test_suite', 'test', 10) execution = quest.Start('input isolate hash') execution.Poll() new_call_body = swarming_tasks_new.call_args[0][0] self.assertIn('--pageset-repeat', new_call_body['properties']['extra_args']) self.assertIn('10', new_call_body['properties']['extra_args'])
def testAllArguments(self): arguments = { 'target': 'net_perftests', 'dimensions': '{"key": "value"}', 'test': 'test_name', 'extra_test_args': '["--custom-arg", "custom value"]', } expected = run_test.RunTest({'key': 'value'}, _ALL_GTEST_ARGUMENTS) self.assertEqual(run_test.RunTest.FromDict(arguments), expected)
def testDictDimensions(self): arguments = { 'target': 'net_perftests', 'dimensions': { 'key': 'value' }, } expected = run_test.RunTest({'key': 'value'}, _MIN_GTEST_ARGUMENTS) self.assertEqual(run_test.RunTest.FromDict(arguments), expected)
def testMinimumArguments(self): arguments = { 'target': 'telemetry_perf_tests', 'dimensions': '{"key": "value"}', 'benchmark': 'speedometer', 'browser': 'release', } expected = run_test.RunTest({'key': 'value'}, _MIN_TELEMETRY_ARGUMENTS) self.assertEqual(run_test.RunTest.FromDict(arguments), expected)
def testWithNoConfiguration(self): arguments = { 'target': 'telemetry_perf_tests', 'dimensions': '{}', 'benchmark': 'speedometer', 'browser': 'release', } expected = run_test.RunTest({}, _MIN_TELEMETRY_RUN_TEST_ARGUMENTS) self.assertEqual(run_test.RunTest.FromDict(arguments), (arguments, expected))
def testAllArguments(self): quest = run_test.RunTest.FromDict({ 'dimensions': '{"key": "value"}', 'extra_test_args': '["--custom-arg", "custom value"]', }) extra_args = ['--custom-arg', 'custom value' ] + run_test._DEFAULT_EXTRA_ARGS expected = run_test.RunTest({'key': 'value'}, extra_args) self.assertEqual(quest, expected)
def testExecutionExpired(self, swarming_task_result, swarming_tasks_new): # If the Swarming task expires, the bots are overloaded or the dimensions # don't correspond to any bot. Raise an error that's fatal to the Job. swarming_tasks_new.return_value = {'task_id': 'task id'} swarming_task_result.return_value = {'state': 'EXPIRED'} quest = run_test.RunTest([{'key': 'value'}], ['arg']) execution = quest.Start('change_1', 'input isolate hash') execution.Poll() with self.assertRaises(run_test.SwarmingExpiredError): execution.Poll()
def testStartupBenchmarkRepeatCount(self): arguments = { 'target': 'telemetry_perf_tests', 'dimensions': '{"key": "value"}', 'benchmark': 'start_with_url.warm.startup_pages', 'browser': 'release', } expected = run_test.RunTest({'key': 'value'}, _STARTUP_BENCHMARK_ARGUMENTS) self.assertEqual(run_test.RunTest.FromDict(arguments), expected)
def testStringExtraTestArgs(self): arguments = { 'target': 'telemetry_perf_tests', 'dimensions': '{"key": "value"}', 'benchmark': 'speedometer', 'browser': 'release', 'story': 'http://www.fifa.com/', 'extra_test_args': '--custom-arg "custom value"', } expected = run_test.RunTest({'key': 'value'}, _ALL_TELEMETRY_ARGUMENTS) self.assertEqual(run_test.RunTest.FromDict(arguments), expected)
def testUnknownConfig(self, swarming_task_result, swarming_tasks_new): quest = run_test.RunTest('configuration', 'test_suite', 'test') execution = quest.Start('input isolate hash') execution.Poll() swarming_task_result.assert_not_called() swarming_tasks_new.assert_not_called() self.assertTrue(execution.completed) self.assertTrue(execution.failed) self.assertEqual(len(execution.result_values), 1) self.assertIsInstance(execution.result_values[0], run_test.UnknownConfigError)
def testSwarmingError(self, swarming_task_result, swarming_tasks_new): swarming_task_result.return_value = {'state': 'BOT_DIED'} swarming_tasks_new.return_value = {'task_id': 'task id'} quest = run_test.RunTest('server', DIMENSIONS, ['arg']) execution = quest.Start(None, 'isolate server', 'input isolate hash') execution.Poll() execution.Poll() self.assertTrue(execution.completed) self.assertTrue(execution.failed) last_exception_line = execution.exception.splitlines()[-1] self.assertTrue(last_exception_line.startswith('SwarmingTaskError'))
def testExecutionExpired(self, swarming_task_result, swarming_tasks_new): # If the Swarming task expires, the bots are overloaded or the dimensions # don't correspond to any bot. Raise an error that's fatal to the Job. swarming_tasks_new.return_value = {'task_id': 'task id'} swarming_task_result.return_value = {'state': 'EXPIRED'} quest = run_test.RunTest('server', DIMENSIONS, ['arg'], _BASE_SWARMING_TAGS, None, None) execution = quest.Start('change_1', 'isolate server', 'input isolate hash') execution.Poll() with self.assertRaises(errors.SwarmingExpired): execution.Poll()
def testSuccess(self, swarming_task_result, swarming_tasks_new): # Goes through a full run of two Executions. # Call RunTest.Start() to create an Execution. quest = run_test.RunTest('chromium-rel-mac11-pro', 'test_suite', 'test', 1) execution = quest.Start('input isolate hash') swarming_task_result.assert_not_called() swarming_tasks_new.assert_not_called() # Call the first Poll() to start the swarming task. swarming_tasks_new.return_value = {'task_id': 'task id'} execution.Poll() swarming_task_result.assert_not_called() swarming_tasks_new.assert_called_once() self.assertNewTaskHasDimensions(swarming_tasks_new) self.assertFalse(execution.completed) self.assertFalse(execution.failed) # Call subsequent Poll()s to check the task status. swarming_task_result.return_value = {'state': 'PENDING'} execution.Poll() self.assertFalse(execution.completed) self.assertFalse(execution.failed) swarming_task_result.return_value = { 'bot_id': 'bot id', 'exit_code': 0, 'failure': False, 'outputs_ref': { 'isolated': 'output isolate hash' }, 'state': 'COMPLETED', } execution.Poll() self.assertTrue(execution.completed) self.assertFalse(execution.failed) self.assertEqual(execution.result_values, (0, )) self.assertEqual(execution.result_arguments, {'isolate_hash': 'output isolate hash'}) # Start a second Execution to check bot_id handling. We get a bot_id from # Swarming from the first Execution and reuse it in subsequent Executions. execution = quest.Start('input isolate hash') execution.Poll() self.assertNewTaskHasBotId(swarming_tasks_new)
def testSwarmingError(self, swarming_task_result, swarming_tasks_new): swarming_task_result.return_value = {'state': 'BOT_DIED'} swarming_tasks_new.return_value = {'task_id': 'task id'} quest = run_test.RunTest('Mac Pro 10.11 Perf', 'test_suite', 'test') execution = quest.Start('input isolate hash') execution.Poll() execution.Poll() self.assertTrue(execution.completed) self.assertTrue(execution.failed) self.assertEqual(len(execution.result_values), 1) self.assertIsInstance(execution.result_values[0], run_test.SwarmingTaskError)
def testSwarmingError(self, swarming_task_result, swarming_tasks_new): swarming_task_result.return_value = {'state': 'BOT_DIED'} swarming_tasks_new.return_value = {'task_id': 'task id'} quest = run_test.RunTest(_SWARMING_DIMENSIONS, _SWARMING_EXTRA_ARGS) execution = quest.Start('input isolate hash') execution.Poll() execution.Poll() self.assertTrue(execution.completed) self.assertTrue(execution.failed) self.assertEqual(len(execution.result_values), 1) self.assertIsInstance(execution.result_values[0], basestring) last_exception_line = execution.result_values[0].splitlines()[-1] self.assertTrue(last_exception_line.startswith('SwarmingTaskError'))
def testTestError(self, swarming_task_result, swarming_tasks_new): swarming_task_result.return_value = { 'bot_id': 'bot id', 'exit_code': 1, 'failure': True, 'state': 'COMPLETED', } swarming_tasks_new.return_value = {'task_id': 'task id'} quest = run_test.RunTest(_SWARMING_DIMENSIONS, _SWARMING_EXTRA_ARGS) execution = quest.Start(None, 'isolate_hash') execution.Poll() execution.Poll() self.assertTrue(execution.completed) self.assertTrue(execution.failed) last_exception_line = execution.exception.splitlines()[-1] self.assertTrue(last_exception_line.startswith('SwarmingTestError'))
def testTestError(self, swarming_task_result, swarming_tasks_new): swarming_task_result.return_value = { 'bot_id': 'bot id', 'exit_code': 1, 'failure': True, 'state': 'COMPLETED', } swarming_tasks_new.return_value = {'task_id': 'task id'} quest = run_test.RunTest('Mac Pro 10.11 Perf', 'test_suite', 'test') execution = quest.Start('isolate_hash') execution.Poll() execution.Poll() self.assertTrue(execution.completed) self.assertTrue(execution.failed) self.assertEqual(len(execution.result_values), 1) self.assertIsInstance(execution.result_values[0], run_test.SwarmingTestError)
def testTestError(self, swarming_task_stdout, swarming_task_result, swarming_tasks_new): swarming_task_stdout.return_value = {'output': ''} swarming_task_result.return_value = { 'bot_id': 'bot id', 'exit_code': 1, 'failure': True, 'state': 'COMPLETED', } swarming_tasks_new.return_value = {'task_id': 'task id'} quest = run_test.RunTest('server', DIMENSIONS, ['arg'], _BASE_SWARMING_TAGS) execution = quest.Start(None, 'isolate server', 'isolate_hash') execution.Poll() execution.Poll() self.assertTrue(execution.completed) self.assertTrue(execution.failed) last_exception_line = execution.exception['traceback'].splitlines()[-1] self.assertTrue(last_exception_line.startswith('SwarmingTaskFailed'))
def testStringDimensions(self): arguments = dict(_BASE_ARGUMENTS) arguments['dimensions'] = json.dumps(DIMENSIONS) quest = run_test.RunTest.FromDict(arguments) expected = run_test.RunTest('server', DIMENSIONS, []) self.assertEqual(quest, expected)