def testAllArguments(self):
arguments = dict(_BASE_ARGUMENTS)
arguments['extra_test_args'] = '["--custom-arg", "custom value"]'
quest = run_test.RunTest.FromDict(arguments)
extra_args = ['--custom-arg', 'custom value']
expected = run_test.RunTest('server', [{'key': 'value'}], extra_args)
self.assertEqual(quest, expected)
def testStringExtraTestArgs(self):
arguments = dict(_BASE_ARGUMENTS)
arguments['extra_test_args'] = '--custom-arg "custom value"'
quest = run_test.RunTest.FromDict(arguments)
extra_args = ['--custom-arg', 'custom value'] + run_test._DEFAULT_EXTRA_ARGS
expected = run_test.RunTest('server', [{'key': 'value'}], extra_args)
self.assertEqual(quest, expected)
def testMinimumArguments(self):
arguments = {
'configuration': 'chromium-rel-mac11-pro',
'target': 'net_perftests',
}
expected = run_test.RunTest({'key': 'value'}, _MIN_GTEST_ARGUMENTS)
self.assertEqual(run_test.RunTest.FromDict(arguments), expected)
def testStart_NoSwarmingTags(self, swarming_task_result,
swarming_tasks_new):
del swarming_task_result
del swarming_tasks_new
quest = run_test.RunTest('server', DIMENSIONS, ['arg'], None, None,
None)
quest.Start('change_1', 'isolate server', 'input isolate hash')
def testStringDimensions(self):
arguments = dict(_BASE_ARGUMENTS)
arguments['dimensions'] = '[{"key": "value"}]'
quest = run_test.RunTest.FromDict(arguments)
expected = run_test.RunTest('server', [{
'key': 'value'
}], run_test._DEFAULT_EXTRA_ARGS)
self.assertEqual(quest, expected)
def testSuccess(self, swarming_task_result, swarming_tasks_new):
# Goes through a full run of two Executions.
# Call RunTest.Start() to create an Execution.
quest = run_test.RunTest(_SWARMING_DIMENSIONS, _SWARMING_EXTRA_ARGS)
execution = quest.Start('input isolate hash')
swarming_task_result.assert_not_called()
swarming_tasks_new.assert_not_called()
# Call the first Poll() to start the swarming task.
swarming_tasks_new.return_value = {'task_id': 'task id'}
execution.Poll()
swarming_task_result.assert_not_called()
swarming_tasks_new.assert_called_once()
self.assertNewTaskHasDimensions(swarming_tasks_new)
self.assertFalse(execution.completed)
self.assertFalse(execution.failed)
# Call subsequent Poll()s to check the task status.
swarming_task_result.return_value = {'state': 'PENDING'}
execution.Poll()
self.assertFalse(execution.completed)
self.assertFalse(execution.failed)
swarming_task_result.return_value = {
'bot_id': 'bot id',
'exit_code': 0,
'failure': False,
'outputs_ref': {'isolated': 'output isolate hash'},
'state': 'COMPLETED',
}
execution.Poll()
self.assertTrue(execution.completed)
self.assertFalse(execution.failed)
self.assertEqual(execution.result_values, (None,))
self.assertEqual(execution.result_arguments,
{'isolate_hashes': ('output isolate hash',)})
self.assertEqual(
{
'bot_ids': ['bot id'],
'input_isolate_hash': 'input isolate hash',
'task_ids': ['task id'],
'result_arguments': {'isolate_hashes': ('output isolate hash',)},
'result_values': (None,),
},
execution.AsDict())
# Start a second Execution to check bot_id handling. We get a bot_id from
# Swarming from the first Execution and reuse it in subsequent Executions.
execution = quest.Start('input isolate hash')
execution.Poll()
self.assertNewTaskHasBotId(swarming_tasks_new)
def testMinimumArguments(self):
arguments = {
'configuration': 'chromium-rel-mac11-pro',
'target': 'telemetry_perf_tests',
'benchmark': 'speedometer',
}
expected = run_test.RunTest({'key': 'value'}, _MIN_TELEMETRY_ARGUMENTS)
self.assertEqual(run_test.RunTest.FromDict(arguments), expected)
def testStringExtraTestArgs(self):
arguments = dict(_BASE_ARGUMENTS)
arguments['extra_test_args'] = '--custom-arg "custom value"'
quest = run_test.RunTest.FromDict(arguments)
extra_args = ['--custom-arg', 'custom value']
expected = run_test.RunTest('server', DIMENSIONS, extra_args,
_BASE_SWARMING_TAGS)
self.assertEqual(quest, expected)
def testWebviewFlag(self):
arguments = {
'configuration': 'chromium-rel-mac11-pro',
'target': 'telemetry_perf_webview_tests',
'benchmark': 'speedometer',
'browser': 'android-webview',
}
expected = run_test.RunTest({'key': 'value'}, _WEBVIEW_ARGUMENTS)
self.assertEqual(run_test.RunTest.FromDict(arguments), expected)
def testStartupBenchmarkRepeatCount(self):
arguments = {
'configuration': 'chromium-rel-mac11-pro',
'target': 'telemetry_perf_tests',
'benchmark': 'start_with_url.warm.startup_pages',
}
expected = run_test.RunTest({'key': 'value'},
_STARTUP_BENCHMARK_ARGUMENTS)
self.assertEqual(run_test.RunTest.FromDict(arguments), expected)
def testWebviewFlag(self):
arguments = {
'target': 'telemetry_perf_webview_tests',
'dimensions': '{"key": "value"}',
'benchmark': 'speedometer',
'browser': 'android-webview',
}
expected = run_test.RunTest({'key': 'value'}, _WEBVIEW_ARGUMENTS)
self.assertEqual(run_test.RunTest.FromDict(arguments), expected)
def testPagesetRepeat(self, swarming_tasks_new):
quest = run_test.RunTest('chromium-rel-mac11-pro', 'test_suite',
'test', 10)
execution = quest.Start('input isolate hash')
execution.Poll()
new_call_body = swarming_tasks_new.call_args[0][0]
self.assertIn('--pageset-repeat',
new_call_body['properties']['extra_args'])
self.assertIn('10', new_call_body['properties']['extra_args'])
def testAllArguments(self):
arguments = {
'target': 'net_perftests',
'dimensions': '{"key": "value"}',
'test': 'test_name',
'extra_test_args': '["--custom-arg", "custom value"]',
}
expected = run_test.RunTest({'key': 'value'}, _ALL_GTEST_ARGUMENTS)
self.assertEqual(run_test.RunTest.FromDict(arguments), expected)
def testDictDimensions(self):
arguments = {
'target': 'net_perftests',
'dimensions': {
'key': 'value'
},
}
expected = run_test.RunTest({'key': 'value'}, _MIN_GTEST_ARGUMENTS)
self.assertEqual(run_test.RunTest.FromDict(arguments), expected)
def testMinimumArguments(self):
arguments = {
'target': 'telemetry_perf_tests',
'dimensions': '{"key": "value"}',
'benchmark': 'speedometer',
'browser': 'release',
}
expected = run_test.RunTest({'key': 'value'}, _MIN_TELEMETRY_ARGUMENTS)
self.assertEqual(run_test.RunTest.FromDict(arguments), expected)
def testWithNoConfiguration(self):
arguments = {
'target': 'telemetry_perf_tests',
'dimensions': '{}',
'benchmark': 'speedometer',
'browser': 'release',
}
expected = run_test.RunTest({}, _MIN_TELEMETRY_RUN_TEST_ARGUMENTS)
self.assertEqual(run_test.RunTest.FromDict(arguments),
(arguments, expected))
def testAllArguments(self):
quest = run_test.RunTest.FromDict({
'dimensions':
'{"key": "value"}',
'extra_test_args':
'["--custom-arg", "custom value"]',
})
extra_args = ['--custom-arg', 'custom value'
] + run_test._DEFAULT_EXTRA_ARGS
expected = run_test.RunTest({'key': 'value'}, extra_args)
self.assertEqual(quest, expected)
def testExecutionExpired(self, swarming_task_result, swarming_tasks_new):
# If the Swarming task expires, the bots are overloaded or the dimensions
# don't correspond to any bot. Raise an error that's fatal to the Job.
swarming_tasks_new.return_value = {'task_id': 'task id'}
swarming_task_result.return_value = {'state': 'EXPIRED'}
quest = run_test.RunTest([{'key': 'value'}], ['arg'])
execution = quest.Start('change_1', 'input isolate hash')
execution.Poll()
with self.assertRaises(run_test.SwarmingExpiredError):
execution.Poll()
def testStartupBenchmarkRepeatCount(self):
arguments = {
'target': 'telemetry_perf_tests',
'dimensions': '{"key": "value"}',
'benchmark': 'start_with_url.warm.startup_pages',
'browser': 'release',
}
expected = run_test.RunTest({'key': 'value'},
_STARTUP_BENCHMARK_ARGUMENTS)
self.assertEqual(run_test.RunTest.FromDict(arguments), expected)
def testStringExtraTestArgs(self):
arguments = {
'target': 'telemetry_perf_tests',
'dimensions': '{"key": "value"}',
'benchmark': 'speedometer',
'browser': 'release',
'story': 'http://www.fifa.com/',
'extra_test_args': '--custom-arg "custom value"',
}
expected = run_test.RunTest({'key': 'value'}, _ALL_TELEMETRY_ARGUMENTS)
self.assertEqual(run_test.RunTest.FromDict(arguments), expected)
def testUnknownConfig(self, swarming_task_result, swarming_tasks_new):
quest = run_test.RunTest('configuration', 'test_suite', 'test')
execution = quest.Start('input isolate hash')
execution.Poll()
swarming_task_result.assert_not_called()
swarming_tasks_new.assert_not_called()
self.assertTrue(execution.completed)
self.assertTrue(execution.failed)
self.assertEqual(len(execution.result_values), 1)
self.assertIsInstance(execution.result_values[0],
run_test.UnknownConfigError)
def testSwarmingError(self, swarming_task_result, swarming_tasks_new):
swarming_task_result.return_value = {'state': 'BOT_DIED'}
swarming_tasks_new.return_value = {'task_id': 'task id'}
quest = run_test.RunTest('server', DIMENSIONS, ['arg'])
execution = quest.Start(None, 'isolate server', 'input isolate hash')
execution.Poll()
execution.Poll()
self.assertTrue(execution.completed)
self.assertTrue(execution.failed)
last_exception_line = execution.exception.splitlines()[-1]
self.assertTrue(last_exception_line.startswith('SwarmingTaskError'))
def testExecutionExpired(self, swarming_task_result, swarming_tasks_new):
# If the Swarming task expires, the bots are overloaded or the dimensions
# don't correspond to any bot. Raise an error that's fatal to the Job.
swarming_tasks_new.return_value = {'task_id': 'task id'}
swarming_task_result.return_value = {'state': 'EXPIRED'}
quest = run_test.RunTest('server', DIMENSIONS, ['arg'],
_BASE_SWARMING_TAGS, None, None)
execution = quest.Start('change_1', 'isolate server',
'input isolate hash')
execution.Poll()
with self.assertRaises(errors.SwarmingExpired):
execution.Poll()
def testSuccess(self, swarming_task_result, swarming_tasks_new):
# Goes through a full run of two Executions.
# Call RunTest.Start() to create an Execution.
quest = run_test.RunTest('chromium-rel-mac11-pro', 'test_suite',
'test', 1)
execution = quest.Start('input isolate hash')
swarming_task_result.assert_not_called()
swarming_tasks_new.assert_not_called()
# Call the first Poll() to start the swarming task.
swarming_tasks_new.return_value = {'task_id': 'task id'}
execution.Poll()
swarming_task_result.assert_not_called()
swarming_tasks_new.assert_called_once()
self.assertNewTaskHasDimensions(swarming_tasks_new)
self.assertFalse(execution.completed)
self.assertFalse(execution.failed)
# Call subsequent Poll()s to check the task status.
swarming_task_result.return_value = {'state': 'PENDING'}
execution.Poll()
self.assertFalse(execution.completed)
self.assertFalse(execution.failed)
swarming_task_result.return_value = {
'bot_id': 'bot id',
'exit_code': 0,
'failure': False,
'outputs_ref': {
'isolated': 'output isolate hash'
},
'state': 'COMPLETED',
}
execution.Poll()
self.assertTrue(execution.completed)
self.assertFalse(execution.failed)
self.assertEqual(execution.result_values, (0, ))
self.assertEqual(execution.result_arguments,
{'isolate_hash': 'output isolate hash'})
# Start a second Execution to check bot_id handling. We get a bot_id from
# Swarming from the first Execution and reuse it in subsequent Executions.
execution = quest.Start('input isolate hash')
execution.Poll()
self.assertNewTaskHasBotId(swarming_tasks_new)
def testSwarmingError(self, swarming_task_result, swarming_tasks_new):
swarming_task_result.return_value = {'state': 'BOT_DIED'}
swarming_tasks_new.return_value = {'task_id': 'task id'}
quest = run_test.RunTest('Mac Pro 10.11 Perf', 'test_suite', 'test')
execution = quest.Start('input isolate hash')
execution.Poll()
execution.Poll()
self.assertTrue(execution.completed)
self.assertTrue(execution.failed)
self.assertEqual(len(execution.result_values), 1)
self.assertIsInstance(execution.result_values[0],
run_test.SwarmingTaskError)
def testSwarmingError(self, swarming_task_result, swarming_tasks_new):
swarming_task_result.return_value = {'state': 'BOT_DIED'}
swarming_tasks_new.return_value = {'task_id': 'task id'}
quest = run_test.RunTest(_SWARMING_DIMENSIONS, _SWARMING_EXTRA_ARGS)
execution = quest.Start('input isolate hash')
execution.Poll()
execution.Poll()
self.assertTrue(execution.completed)
self.assertTrue(execution.failed)
self.assertEqual(len(execution.result_values), 1)
self.assertIsInstance(execution.result_values[0], basestring)
last_exception_line = execution.result_values[0].splitlines()[-1]
self.assertTrue(last_exception_line.startswith('SwarmingTaskError'))
def testTestError(self, swarming_task_result, swarming_tasks_new):
swarming_task_result.return_value = {
'bot_id': 'bot id',
'exit_code': 1,
'failure': True,
'state': 'COMPLETED',
}
swarming_tasks_new.return_value = {'task_id': 'task id'}
quest = run_test.RunTest(_SWARMING_DIMENSIONS, _SWARMING_EXTRA_ARGS)
execution = quest.Start(None, 'isolate_hash')
execution.Poll()
execution.Poll()
self.assertTrue(execution.completed)
self.assertTrue(execution.failed)
last_exception_line = execution.exception.splitlines()[-1]
self.assertTrue(last_exception_line.startswith('SwarmingTestError'))
def testTestError(self, swarming_task_result, swarming_tasks_new):
swarming_task_result.return_value = {
'bot_id': 'bot id',
'exit_code': 1,
'failure': True,
'state': 'COMPLETED',
}
swarming_tasks_new.return_value = {'task_id': 'task id'}
quest = run_test.RunTest('Mac Pro 10.11 Perf', 'test_suite', 'test')
execution = quest.Start('isolate_hash')
execution.Poll()
execution.Poll()
self.assertTrue(execution.completed)
self.assertTrue(execution.failed)
self.assertEqual(len(execution.result_values), 1)
self.assertIsInstance(execution.result_values[0],
run_test.SwarmingTestError)
def testTestError(self, swarming_task_stdout,
swarming_task_result, swarming_tasks_new):
swarming_task_stdout.return_value = {'output': ''}
swarming_task_result.return_value = {
'bot_id': 'bot id',
'exit_code': 1,
'failure': True,
'state': 'COMPLETED',
}
swarming_tasks_new.return_value = {'task_id': 'task id'}
quest = run_test.RunTest('server', DIMENSIONS, ['arg'], _BASE_SWARMING_TAGS)
execution = quest.Start(None, 'isolate server', 'isolate_hash')
execution.Poll()
execution.Poll()
self.assertTrue(execution.completed)
self.assertTrue(execution.failed)
last_exception_line = execution.exception['traceback'].splitlines()[-1]
self.assertTrue(last_exception_line.startswith('SwarmingTaskFailed'))
def testStringDimensions(self):
arguments = dict(_BASE_ARGUMENTS)
arguments['dimensions'] = json.dumps(DIMENSIONS)
quest = run_test.RunTest.FromDict(arguments)
expected = run_test.RunTest('server', DIMENSIONS, [])
self.assertEqual(quest, expected)
