def testAfterTestRunHandler(self, mock_get_request, mock_get_test_context,
mock_add_task):
# configure mock TFC request
request_id = self.mock_test_run.request_id
mock_request = api_messages.RequestMessage(id=request_id)
mock_request.commands.append(api_messages.CommandMessage(id='bar'))
mock_get_request.return_value = mock_request
# configure mock TFC test context
mock_test_context = api_messages.TestContext(
command_line='command_line',
env_vars=[api_messages.KeyValuePair(key='key', value='value')],
test_resources=[
api_messages.TestResource(
url='url',
name='name',
path='path',
decompress=True,
decompress_dir='dir',
params=api_messages.TestResourceParameters(
decompress_files=['file'])),
api_messages.TestResource(url='url2',
name='name2',
path='path2')
])
mock_get_test_context.return_value = mock_test_context
expected_test_context = ndb_models.TestContextObj(
command_line='command_line',
env_vars=[ndb_models.NameValuePair(name='key', value='value')],
test_resources=[
ndb_models.TestResourceObj(
url='url',
name='name',
decompress=True,
decompress_dir='dir',
params=ndb_models.TestResourceParameters(
decompress_files=['file'])),
ndb_models.TestResourceObj(url='url2', name='name2')
])
tfc_event_handler._AfterTestRunHandler(self.mock_test_run)
# test run updated, run hooks invoked, output uploaded, and metrics tracked
mock_get_request.assert_called_with(request_id)
mock_get_test_context.assert_called_with(request_id, 'bar')
self.assertEqual(expected_test_context,
self.mock_test_run.next_test_context)
mock_add_task.assert_has_calls([
mock.call(test_run_hook.ExecuteHooks,
self.mock_test_run.key.id(),
ndb_models.TestRunPhase.AFTER_RUN,
_transactional=True),
mock.call(tfc_event_handler._TrackTestRun,
self.mock_test_run.key.id(),
_transactional=True),
])
def testKickTestRun_withPrevTestContext(
self, mock_download_resources, mock_new_request):
test_run = self._CreateMockTestRun(
command='command',
retry_command_line='retry_command_line',
runner_sharding_args='--shard-count ${TF_SHARD_COUNT}',
shard_count=6)
test_run.prev_test_context = ndb_models.TestContextObj(
command_line='prev_command_line',
env_vars=[],
test_resources=[ndb_models.TestResourceObj(name='bar', url='zzz')])
test_run.put()
test_run_id = test_run.key.id()
test_run = ndb_models.TestRun.get_by_id(test_run_id)
mock_download_resources.return_value = {
r.url: 'cache_url' for r in test_run.test_resources
}
mock_request = api_messages.RequestMessage(id='request_id')
mock_new_request.return_value = mock_request
test_kicker.KickTestRun(test_run_id)
mock_download_resources.assert_called_once_with(
[r.url for r in test_run.test_resources], test_run=test_run)
mock_new_request.assert_called()
msg = mock_new_request.call_args[0][0]
self._CheckNewRequestMessage(
msg=msg,
test_run=test_run,
output_url='file:///data/app_default_bucket/test_runs/{}/output'.format(
test_run_id),
test_resource_urls={
'foo':
'cache_url',
'bar':
'cache_url',
'mtt.json':
'http://localhost:8000/_ah/api/mtt/v1/test_runs/{}/metadata'
.format(test_run_id)
},
command_lines=['command --shard-count 6 --invocation-data mtt=1'],
retry_command_line=(
'retry_command_line --shard-count 6 --invocation-data mtt=1'),
shard_count=1)
# prev_test_context's command_line should be replaced.
self.assertEqual(
'retry_command_line --shard-count 6 --invocation-data mtt=1',
msg.prev_test_context.command_line)
test_run = ndb_models.TestRun.get_by_id(test_run_id)
self.assertEqual(mock_request.id, test_run.request_id)
self.assertEqual(ndb_models.TestRunState.QUEUED, test_run.state)
def testKickTestRun_onPremiseMode(self, mock_download_resources,
mock_new_request):
test_run = self._CreateMockTestRun()
# test_run = ndb_models.TestRun.get_by_id(test_run_id)
test_run.prev_test_context = ndb_models.TestContextObj(
command_line='prev_command_line',
env_vars=[],
test_resources=[
ndb_models.TestResourceObj(name='bar', url='file:///root/path')
])
test_run.put()
test_run_id = test_run.key.id()
test_run = ndb_models.TestRun.get_by_id(test_run_id)
mock_download_resources.return_value = {
r.url: 'file:///data/cache_url' for r in test_run.test_resources
}
mock_request = api_messages.RequestMessage(id='request_id')
mock_new_request.return_value = mock_request
test_kicker.KickTestRun(test_run_id)
mock_download_resources.assert_called_once_with(
[r.url for r in test_run.test_resources], test_run=test_run)
mock_new_request.assert_called()
msg = mock_new_request.call_args[0][0]
self._CheckNewRequestMessage(
msg=msg,
test_run=test_run,
output_url='http://test.hostname.com:8006/file/app_default_bucket/test_runs/{}/output'
.format(test_run_id),
test_resource_urls={
'foo':
'http://test.hostname.com:8006/file/cache_url',
'bar':
'http://test.hostname.com:8006/file/cache_url',
'mtt.json':
'http://test.hostname.com:8000/_ah/api/mtt/v1/test_runs/{}/metadata'
.format(test_run_id)
},
command_lines=['command --invocation-data mtt=1'])
self.assertEqual([
api_messages.TestResource(
name='bar', url='http://test.hostname.com:8006/file/root/path')
], msg.prev_test_context.test_resources)
def testKickTestRun_runnerSharding(
self, mock_download_resources, mock_new_request):
test_run_id = self._CreateMockTestRun(
command='command',
runner_sharding_args='--shard-count ${TF_SHARD_COUNT}',
run_target='run_target;run_target',
shard_count=2,
sharding_mode=ndb_models.ShardingMode.RUNNER).key.id()
test_run = ndb_models.TestRun.get_by_id(test_run_id)
mock_download_resources.return_value = {
r.url: 'cache_url' for r in test_run.test_resources
}
mock_request = api_messages.RequestMessage(id='request_id')
mock_new_request.return_value = mock_request
test_kicker.KickTestRun(test_run_id)
mock_download_resources.assert_called_once_with(
[r.url for r in test_run.test_resources], test_run=test_run)
mock_new_request.assert_called()
msg = mock_new_request.call_args[0][0]
self._CheckNewRequestMessage(
msg=msg,
test_run=test_run,
output_url='file:///data/app_default_bucket/test_runs/{}/output'.format(
test_run_id),
test_resource_urls={
'foo':
'cache_url',
'bar':
'cache_url',
'mtt.json':
'http://localhost:8000/_ah/api/mtt/v1/test_runs/{}/metadata'
.format(test_run_id)
},
command_lines=['command --shard-count 2 --invocation-data mtt=1'],
run_target='run_target;run_target',
shard_count=1)
test_run = ndb_models.TestRun.get_by_id(test_run_id)
self.assertEqual(mock_request.id, test_run.request_id)
self.assertEqual(ndb_models.TestRunState.QUEUED, test_run.state)
def testTrackTestRun(self, mock_time, mock_get_request, mock_log):
# mock current time
now = datetime.datetime.now()
mock_time.return_value = now
# configure mock TFC request with 1 attempt and 2 devices
request_id = self.mock_test_run.request_id
mock_request = api_messages.RequestMessage(id=request_id)
mock_request.command_attempts.append(
api_messages.CommandAttemptMessage(
device_serials=['1', '2'],
start_time=now,
end_time=now,
state=common.CommandState.COMPLETED))
mock_get_request.return_value = mock_request
# update mock test run
self.mock_test_run.test_package_info = ndb_models.TestPackageInfo(
name='name', version='version')
self.mock_test_run.state = ndb_models.TestRunState.COMPLETED
self.mock_test_run.create_time = now - datetime.timedelta(seconds=123)
self.mock_test_run.failed_test_run_count = 1
self.mock_test_run.total_test_count = 2
self.mock_test_run.failed_test_count = 3
self.mock_test_run.put()
# check that right data was logged
tfc_event_handler._TrackTestRun(self.mock_test_run.key.id())
mock_log.assert_called_with(analytics.TEST_RUN_CATEGORY,
analytics.END_ACTION,
test_name='name',
test_version='version',
state='COMPLETED',
is_rerun=False,
duration_seconds=123,
device_count=2,
attempt_count=1,
failed_module_count=1,
test_count=2,
failed_test_count=3)
def testKickTestRun(
self, mock_download_resources, mock_new_request, mock_track_test_run):
test_run_id = self._CreateMockTestRun().key.id()
test_run = ndb_models.TestRun.get_by_id(test_run_id)
test_run.test_run_config.use_parallel_setup = True
test_run.put()
mock_download_resources.return_value = {
r.url: 'cache_url' for r in test_run.test_resources
}
mock_request = api_messages.RequestMessage(id='request_id')
mock_new_request.return_value = mock_request
test_kicker.KickTestRun(test_run_id)
mock_download_resources.assert_called_once_with(
[r.url for r in test_run.test_resources], test_run=test_run)
mock_new_request.assert_called()
msg = mock_new_request.call_args[0][0]
self._CheckNewRequestMessage(
msg=msg,
test_run=test_run,
output_url='file:///data/app_default_bucket/test_runs/{}/output'.format(
test_run_id),
test_resource_urls={
'foo':
'cache_url',
'bar':
'cache_url',
'mtt.json':
'http://localhost:8000/_ah/api/mtt/v1/test_runs/{}/metadata'
.format(test_run_id)
},
command_lines=['command --invocation-data mtt=1'])
test_run = ndb_models.TestRun.get_by_id(test_run_id)
self.assertEqual(mock_request.id, test_run.request_id)
self.assertEqual(ndb_models.TestRunState.QUEUED, test_run.state)
# metrics tracked
mock_track_test_run.assert_called_with(test_run)
def testKickTestRun_withTestRunActions(
self, mock_download_resources, mock_execute_hooks, mock_new_request):
# Create test run action with two TF result reporters
test_run_action = ndb_models.TestRunAction(
name='Foo',
hook_class_name='foo',
tradefed_result_reporters=[
ndb_models.TradefedConfigObject(
class_name='com.android.foo',
option_values=[ndb_models.NameMultiValuePair(
name='test-run-id', values=['${MTT_TEST_RUN_ID}'])]
),
ndb_models.TradefedConfigObject(class_name='com.android.bar'),
])
test_run_action.put()
# Create test run with test run action
test_run = self._CreateMockTestRun(command='command')
test_run.test_run_actions = [test_run_action]
test_run.put()
test_run_id = test_run.key.id()
test_run = ndb_models.TestRun.get_by_id(test_run_id)
# Mock responses
mock_download_resources.return_value = {
r.url: 'cache_url' for r in test_run.test_resources
}
mock_request = api_messages.RequestMessage(id='request_id')
mock_new_request.return_value = mock_request
test_kicker.KickTestRun(test_run_id)
# Resources downloaded, hooks executed, and new TFC request created
mock_download_resources.assert_called_once_with(
[r.url for r in test_run.test_resources], test_run=test_run)
mock_execute_hooks.assert_called_with(
test_run_id, ndb_models.TestRunPhase.BEFORE_RUN)
mock_new_request.assert_called()
msg = mock_new_request.call_args[0][0]
self._CheckNewRequestMessage(
msg=msg,
test_run=test_run,
output_url='file:///data/app_default_bucket/test_runs/{}/output'.format(
test_run_id),
test_resource_urls={
'foo':
'cache_url',
'bar':
'cache_url',
'mtt.json':
'http://localhost:8000/_ah/api/mtt/v1/test_runs/{}/metadata'
.format(test_run_id)
},
command_lines=['command --invocation-data mtt=1'])
# TFC request has two TF result reporters with right class names and options
tradefed_config_objects = msg.test_environment.tradefed_config_objects
self.assertEqual(
[tradefed_config_objects.pop(0)], test_kicker.DEFAULT_TF_CONFIG_OBJECTS)
self.assertLen(tradefed_config_objects, 2)
self.assertEqual(api_messages.TradefedConfigObjectType.RESULT_REPORTER,
tradefed_config_objects[1].type)
self.assertEqual('com.android.foo', tradefed_config_objects[0].class_name)
self.assertEqual('test-run-id',
tradefed_config_objects[0].option_values[0].key)
self.assertEqual([str(test_run_id)],
tradefed_config_objects[0].option_values[0].values)
self.assertEqual(api_messages.TradefedConfigObjectType.RESULT_REPORTER,
tradefed_config_objects[1].type)
self.assertEqual('com.android.bar', tradefed_config_objects[1].class_name)
self.assertEmpty(tradefed_config_objects[1].option_values)
# Test run now queued
test_run = ndb_models.TestRun.get_by_id(test_run_id)
self.assertEqual(mock_request.id, test_run.request_id)
self.assertEqual(ndb_models.TestRunState.QUEUED, test_run.state)
def testKickTestRun_moduleSharding(
self,
mock_download_resources,
mock_open_file,
mock_get_test_module_infos,
mock_new_request):
test_run = self._CreateMockTestRun(
command='command',
run_target='run_target',
shard_count=6,
sharding_mode=ndb_models.ShardingMode.MODULE,
module_config_pattern='path/to/.*\\.config',
module_execution_args='-m ${MODULE_NAME}',
extra_test_resources=[
ndb_models.TestResourceObj(
name='test_package',
url='test_package_url',
test_resource_type=ndb_models.TestResourceType.TEST_PACKAGE)
])
test_run_id = test_run.key.id()
test_run = ndb_models.TestRun.get_by_id(test_run_id)
mock_download_resources.return_value = {
r.url: '%s_cache_url' % r.name for r in test_run.test_resources
}
mock_get_test_module_infos.return_value = [
file_util.TestModuleInfo(name='CtsFooTestCases'),
file_util.TestModuleInfo(name='CtsDeqpTestCases'),
file_util.TestModuleInfo(name='CtsBarTestCases'),
]
mock_request = api_messages.RequestMessage(id='request_id')
mock_new_request.return_value = mock_request
test_kicker.KickTestRun(test_run_id)
mock_download_resources.assert_called_once_with(
[r.url for r in test_run.test_resources], test_run=test_run)
mock_open_file.assert_called_with('test_package_cache_url')
mock_get_test_module_infos.assert_called_once_with(
mock_open_file.return_value, 'path/to/.*\\.config')
mock_new_request.assert_called()
msg = mock_new_request.call_args[0][0]
self._CheckNewRequestMessage(
msg=msg,
test_run=test_run,
output_url='file:///data/app_default_bucket/test_runs/{}/output'.format(
test_run_id),
test_resource_urls={
'foo': 'foo_cache_url',
'bar': 'bar_cache_url',
'test_package': 'test_package_cache_url',
'mtt.json':
'http://localhost:8000/_ah/api/mtt/v1/test_runs/{}/metadata'
.format(test_run_id)
},
command_lines=[
'command -m CtsDeqpTestCases --invocation-data mtt=1',
'command -m CtsBarTestCases --invocation-data mtt=1',
'command -m CtsFooTestCases --invocation-data mtt=1',
],
run_target='run_target',
shard_count=1,
max_concurrent_tasks=6)
test_run = ndb_models.TestRun.get_by_id(test_run_id)
self.assertEqual(mock_request.id, test_run.request_id)
self.assertEqual(ndb_models.TestRunState.QUEUED, test_run.state)
def testTrackTestInvocation_manualRetry(self, mock_time, mock_log):
# mock current time
now = datetime.datetime.now()
mock_time.return_value = now
# configure mock TFC request with 1 attempt
request_id = self.mock_test_run.request_id
mock_request = api_messages.RequestMessage(id=request_id,
command_line='command')
mock_request.command_attempts.append(
api_messages.CommandAttemptMessage(
state=common.CommandState.COMPLETED,
device_serials=['2'],
start_time=now - datetime.timedelta(seconds=200),
end_time=now - datetime.timedelta(seconds=100),
total_test_count=3,
failed_test_run_count=2,
failed_test_count=1))
# create previous test runs
first_test_run = ndb_models.TestRun(
test_run_config=ndb_models.TestRunConfig(
test_key=self.mock_test.key, run_target='run_target'),
test=self.mock_test,
request_id='request_id1',
state=ndb_models.TestRunState.UNKNOWN,
create_time=now - datetime.timedelta(seconds=5000),
total_test_count=100,
failed_test_run_count=90,
failed_test_count=80,
prev_test_context=ndb_models.TestContextObj())
first_test_run.put()
second_test_run = ndb_models.TestRun(
test_run_config=ndb_models.TestRunConfig(
test_key=self.mock_test.key, run_target='run_target'),
test=self.mock_test,
request_id='request_id2',
state=ndb_models.TestRunState.UNKNOWN,
create_time=now - datetime.timedelta(seconds=2000),
total_test_count=10,
failed_test_run_count=9,
failed_test_count=8,
prev_test_run_key=first_test_run.key)
second_test_run.put()
# update mock test run
self.mock_test_run.test_package_info = ndb_models.TestPackageInfo(
name='name', version='version')
self.mock_test_run.state = ndb_models.TestRunState.COMPLETED
self.mock_test_run.create_time = now - datetime.timedelta(seconds=1000)
self.mock_test_run.failed_test_count = 1
self.mock_test_run.prev_test_run_key = second_test_run.key
# check that right data was logged
tfc_event_handler._TrackTestInvocations(mock_request,
self.mock_test_run)
mock_log.assert_called_with(
analytics.INVOCATION_CATEGORY,
analytics.END_ACTION,
command='command',
test_run_command='mock test run command',
test_run_retry_command='mock test run retry command',
test_id=tfc_event_handler.LOCAL_ID_TAG,
test_name='name',
test_version='version',
state='COMPLETED',
failed_test_count_threshold=None,
duration_seconds=100,
elapsed_time_seconds=4900,
device_count=1,
test_count=3,
failed_module_count=2,
failed_test_count=1,
prev_total_test_count=10,
prev_failed_module_count=9,
prev_failed_test_count=8,
missing_previous_run=True,
is_sequence_run=False)
def testTrackTestInvocation(self, mock_time, mock_log):
# mock current time
now = datetime.datetime.now()
mock_time.return_value = now
# configure mock TFC request with 2 attempts
request_id = self.mock_test_run.request_id
mock_request = api_messages.RequestMessage(id=request_id,
command_line='command')
mock_request.command_attempts.append(
api_messages.CommandAttemptMessage(
state=common.CommandState.COMPLETED,
device_serials=['1', '2'],
start_time=now - datetime.timedelta(seconds=400),
end_time=now - datetime.timedelta(seconds=300),
total_test_count=60,
failed_test_run_count=50,
failed_test_count=40))
mock_request.command_attempts.append(
api_messages.CommandAttemptMessage(
state=common.CommandState.COMPLETED,
device_serials=['2'],
start_time=now - datetime.timedelta(seconds=200),
end_time=now - datetime.timedelta(seconds=100),
total_test_count=3,
failed_test_run_count=2,
failed_test_count=1))
# set test id
self.mock_test.key = ndb.Key(ndb_models.Test, 'some.test.id')
self.mock_test.put()
self.mock_test_run.test_run_config.test_key = self.mock_test.key
# update mock test run
self.mock_test_run.test_package_info = ndb_models.TestPackageInfo(
name='name', version='version')
self.mock_test_run.state = ndb_models.TestRunState.COMPLETED
self.mock_test_run.create_time = now - datetime.timedelta(seconds=1000)
self.mock_test_run.failed_test_count = 1
self.mock_test_run.sequence_id = 'abc-def-ghi'
# check that right data was logged
tfc_event_handler._TrackTestInvocations(mock_request,
self.mock_test_run)
mock_log.assert_called_with(
analytics.INVOCATION_CATEGORY,
analytics.END_ACTION,
command='command',
test_run_command='mock test run command',
test_run_retry_command='mock test run retry command',
test_id='some.test.id',
test_name='name',
test_version='version',
state='COMPLETED',
failed_test_count_threshold='10',
duration_seconds=100,
elapsed_time_seconds=900,
device_count=1,
test_count=3,
failed_module_count=2,
failed_test_count=1,
prev_total_test_count=60,
prev_failed_module_count=50,
prev_failed_test_count=40,
missing_previous_run=False,
is_sequence_run=True)