def testNew_withRunTarget(self, mock_run_test):
test = self._createMockTest()
request = {
'labels': ['label'],
'test_run_config': {
'test_id': str(test.key.id()),
'cluster': 'cluster',
'device_specs': ['foo', 'bar'],
'run_count': 10,
'shard_count': 100,
'max_retry_on_test_failures': 1000,
'test_resource_objs': [
{
'name': 'bar',
'url': 'bar_url'
},
{
'name': 'zzz',
'url': 'zzz_url'
},
],
},
}
test_run = ndb_models.TestRun(
test=test,
labels=['label'],
test_run_config=ndb_models.TestRunConfig(
test_key=test.key,
cluster='cluster',
device_specs=['foo', 'bar'],
run_count=10,
shard_count=100,
max_retry_on_test_failures=1000))
test_run.put()
mock_run_test.return_value = test_run
res = self.app.post_json('/_ah/api/mtt/v1/test_runs', request)
mock_run_test.assert_called_with(
labels=['label'],
test_run_config=ndb_models.TestRunConfig(
test_key=test.key,
cluster='cluster',
device_specs=['foo', 'bar'],
run_count=10,
shard_count=100,
max_retry_on_test_failures=1000,
test_resource_objs=[
ndb_models.TestResourceObj(name='bar', url='bar_url'),
ndb_models.TestResourceObj(name='zzz', url='zzz_url'),
]),
rerun_context=None,
rerun_configs=[]
)
self.assertEqual('200 OK', res.status)
test_run_msg = protojson.decode_message(messages.TestRun, res.body)
self.assertEqual(
messages.Convert(test_run, messages.TestRun), test_run_msg)
def testKickTestPlan(self, create_test_run):
"""Tests that a test plan can be kicked off."""
test = ndb_models.Test(id='test_id')
test_device_action = ndb_models.DeviceAction(id='test_device_action')
test_run_action = ndb_models.TestRunAction(id='test_run_action')
# Create a test plan with multiple resources and actions
test_plan = ndb_models.TestPlan(
name='test_plan',
labels=['label'],
cron_exp='0 0 * * *',
test_run_configs=[
ndb_models.TestRunConfig(
test_key=test.key,
cluster='cluster',
run_target='run_target',
before_device_action_keys=[test_device_action.key],
test_run_action_refs=[
ndb_models.TestRunActionRef(action_key=test_run_action.key),
],
test_resource_objs=[
ndb_models.TestResourceObj(name='res_1', url='url_1')]),
])
test_plan.put()
# Test run will be created successfully
test_run = ndb_models.TestRun(id='test_run_id')
create_test_run.return_value = test_run
executed = test_plan_kicker.KickTestPlan(test_plan.key.id())
self.assertTrue(executed)
# Test run is created with the right test and test plan components
create_test_run.assert_called_with(
labels=['label'],
test_plan_key=test_plan.key,
test_run_config=ndb_models.TestRunConfig(
test_key=test.key,
cluster='cluster',
run_target='run_target',
before_device_action_keys=[test_device_action.key],
test_run_action_refs=[
ndb_models.TestRunActionRef(action_key=test_run_action.key),
],
test_resource_objs=[
ndb_models.TestResourceObj(name='res_1', url='url_1')
]),
)
# Test run key is stored in the test plan status
status = ndb_models.TestPlanStatus.query(ancestor=test_plan.key).get()
self.assertEqual(status.last_run_keys, [test_run.key])
def _CreateMockTestRunConfig(self, test, run_target='run_target'):
"""Create a mock ndb_models.TestRunConfig object."""
return ndb_models.TestRunConfig(
test_key=test.key,
run_target=run_target,
cluster='cluster',
)
def _CreateMockTestRun(
self,
test_name='test',
command='command',
retry_command_line=None,
runner_sharding_args=None,
run_target='run_target',
shard_count=1,
sharding_mode=ndb_models.ShardingMode.RUNNER,
edited_command=None,
module_config_pattern=None,
module_execution_args=None,
extra_test_resources=None):
test = ndb_models.Test(
name=test_name,
command=command,
retry_command_line=retry_command_line,
runner_sharding_args=runner_sharding_args,
module_config_pattern=module_config_pattern,
module_execution_args=module_execution_args)
test.put()
test_run_config = ndb_models.TestRunConfig(
test_key=test.key, cluster='cluster', run_target=run_target,
shard_count=shard_count, sharding_mode=sharding_mode,
command=edited_command)
test_resources = [
ndb_models.TestResourceObj(name='foo', url='http://foo_origin_url'),
ndb_models.TestResourceObj(name='bar', url='https://bar_origin_url'),
] + (extra_test_resources or [])
test_run = ndb_models.TestRun(
id=str(uuid.uuid4()),
test=test, labels=['label'], test_run_config=test_run_config,
test_resources=test_resources, state=ndb_models.TestRunState.PENDING)
test_run.put()
return test_run
def _CreateMockTestRun(self, config, state=None, create_time=None):
"""Creates a test run.
Args:
config: a test run config
state: a test run state.
create_time: a test run create time.
Returns:
a ndb_models.TestRun object.
"""
test = ndb_models.Test(name='test', command='command')
test.put()
test_run_config = ndb_models.TestRunConfig(test_key=test.key,
cluster='cluster',
run_target='run_target')
test_resources = [
ndb_models.TestResourceObj(name='foo',
url='http://foo_origin_url'),
ndb_models.TestResourceObj(name='bar',
url='https://bar_origin_url'),
]
test_run = ndb_models.TestRun(labels=['label'],
test=test,
test_run_config=test_run_config,
test_resources=test_resources,
state=state
or ndb_models.TestRunState.PENDING,
create_time=create_time)
test_run.key = ndb.Key(ndb_models.TestRun, str(uuid.uuid4()))
test_run.put()
return test_run
def testKickTestPlan_withError(self, create_test_run, set_test_run_state):
"""Tests that errors are handled when kicking off a test plan."""
test = ndb_models.Test(id='test_id')
test_run_config = ndb_models.TestRunConfig(
test_key=test.key, cluster='cluster', run_target='run_target')
test_plan = ndb_models.TestPlan(
name='test_plan', test_run_configs=[test_run_config, test_run_config])
test_plan.put()
# First test run created successfully, but second fails even with retries
test_run = ndb_models.TestRun(id='test_run_id')
create_test_run.side_effect = (
[test_run] +
test_plan_kicker.MAX_RETRY_COUNT * [RuntimeError('test_run_error')])
# Retries a few times before canceling test run and raising exception
with self.assertRaises(RuntimeError):
test_plan_kicker.KickTestPlan(test_plan.key.id())
self.assertEqual(create_test_run.call_count,
1 + test_plan_kicker.MAX_RETRY_COUNT)
set_test_run_state.assert_called_once_with('test_run_id',
ndb_models.TestRunState.CANCELED)
# Stores the canceled test run key and the error message
status = ndb_models.TestPlanStatus.query(ancestor=test_plan.key).get()
self.assertEqual(status.last_run_keys, [test_run.key])
self.assertEqual(status.last_run_error, 'test_run_error')
def _TestRunConfigMessageConverter(msg):
return ndb_models.TestRunConfig(
test_key=ConvertToKeyOrNone(ndb_models.Test, msg.test_id),
cluster=msg.cluster,
command=msg.command,
retry_command=msg.retry_command,
device_specs=msg.device_specs,
run_target=msg.run_target,
run_count=msg.run_count,
shard_count=msg.shard_count,
sharding_mode=msg.sharding_mode,
extra_args=msg.extra_args, # TODO: Deprecated
retry_extra_args=msg.retry_extra_args, # TODO: Deprecated
max_retry_on_test_failures=msg.max_retry_on_test_failures,
queue_timeout_seconds=msg.queue_timeout_seconds,
invocation_timeout_seconds=msg.invocation_timeout_seconds,
output_idle_timeout_seconds=msg.output_idle_timeout_seconds,
before_device_action_keys=[
ConvertToKeyOrNone(ndb_models.DeviceAction, device_action_id)
for device_action_id in msg.before_device_action_ids
],
test_run_action_refs=ConvertList(
msg.test_run_action_refs, ndb_models.TestRunActionRef),
test_resource_objs=ConvertList(
msg.test_resource_objs, ndb_models.TestResourceObj),
use_parallel_setup=msg.use_parallel_setup)
def _CreateMockTestRunConfig(self, test, device_action_keys=None):
"""Creates a mock ndb_models.TestRunConfig object."""
config = ndb_models.TestRunConfig(
test_key=test.key,
before_device_action_keys=device_action_keys,
cluster='cluster',
run_target='run_target')
return config
def testRun(self, mock_test_plan_runner):
"""Tests test_plans.run API."""
test_run_config = ndb_models.TestRunConfig(
cluster='cluster',
test_key=messages.ConvertToKey(ndb_models.Test, 'test'),
run_target='target')
test_plan = _CreateMockTestPlan('foo',
test_run_configs=[test_run_config])
self.app.post('/_ah/api/mtt/v1/test_plans/%s/run' % test_plan.key.id())
mock_test_plan_runner.assert_called_with(test_plan.key.id())
def testExecuteHook_withContextVariables(self, mock_init, mock_log,
mock_execute):
"""Tests that a hook can be constructed and executed."""
test = ndb_models.Test(name='test', command='command')
test.put()
test_run = ndb_models.TestRun(
test=test,
test_run_config=ndb_models.TestRunConfig(test_key=test.key),
test_resources=[
ndb_models.TestResourceObj(
name='device_image',
url='mtt:///android_ci/branch/target/build_id/image.zip',
test_resource_type=ndb_models.TestResourceType.DEVICE_IMAGE
)
])
test_run.put()
mock_init.return_value = None
hook_context = mock.MagicMock()
hook_context.test_run = test_run
credentials = authorized_user.Credentials(None)
action = ndb_models.TestRunAction(
name='Test',
hook_class_name='simple',
options=[
ndb_models.NameValuePair(name='ham', value='eggs'),
ndb_models.NameValuePair(name='test_run_id',
value='${MTT_TEST_RUN_ID}'),
ndb_models.NameValuePair(name='device_image_url',
value='${MTT_DEVICE_IMAGE_URL}'),
ndb_models.NameValuePair(name='device_image_branch',
value='${MTT_DEVICE_IMAGE_BRANCH}'),
ndb_models.NameValuePair(name='device_image_target',
value='${MTT_DEVICE_IMAGE_TARGET}'),
ndb_models.NameValuePair(name='device_image_build_id',
value='${MTT_DEVICE_IMAGE_BUILD_ID}'),
],
credentials=credentials,
)
test_run_hook._ExecuteHook(action, hook_context)
mock_init.assert_called_with(
_credentials=credentials,
ham='eggs',
test_run_id=str(test_run.key.id()),
device_image_url=
'mtt:///android_ci/branch/target/build_id/image.zip',
device_image_branch='branch',
device_image_target='target',
device_image_build_id='build_id')
mock_log.assert_called()
mock_execute.assert_called_with(hook_context)
def testCreateTestRun(self, mock_find_resources):
test = ndb_models.Test(
name='test', command='command', test_resource_defs=[
ndb_models.TestResourceDef(
name='foo', default_download_url='default_download_url'),
ndb_models.TestResourceDef(
name='bar', default_download_url='default_download_url',
decompress=True, decompress_dir='dir'),
])
test.put()
overwritten_obj = ndb_models.TestResourceObj(
name='foo', url='origin_url', cache_url='cache_url')
test_run_config = ndb_models.TestRunConfig(
test_key=test.key,
cluster='cluster',
device_specs=['device_serial:serial'],
test_resource_objs=[overwritten_obj],
)
mock_find_resources.return_value = [overwritten_obj]
test_run = test_kicker.CreateTestRun(['label'], test_run_config)
test_run = ndb_models.TestRun.get_by_id(test_run.key.id())
self.assertModelEqual(test, test_run.test)
self.assertEqual(['label'], test_run.labels)
self.assertModelEqual(test_run_config, test_run.test_run_config)
self.assertModelSetEqual([
ndb_models.TestResourceObj(
name='bar',
url='default_download_url',
decompress=True,
decompress_dir='dir',
mount_zip=False,
params=ndb_models.TestResourceParameters()),
ndb_models.TestResourceObj(
name='foo',
url='origin_url',
cache_url='cache_url',
decompress=False,
decompress_dir='',
mount_zip=False,
params=ndb_models.TestResourceParameters()),
], test_run.test_resources)
self.assertEqual(ndb_models.TestRunState.PENDING, test_run.state)
tasks = self.mock_task_scheduler.GetTasks(
queue_names=[test_kicker.TEST_KICKER_QUEUE])
self.assertLen(tasks, 1)
task = tasks[0]
data = json.loads(task.payload)
self.assertEqual(test_run.key.id(), data['test_run_id'])
self.assertIsNone(test_run.sequence_id)
def setUp(self):
super(TfcEventHandlerTest, self).setUp()
self.mock_test = ndb_models.Test(name='test',
command='command',
result_file='result_file')
self.mock_test.put()
self.mock_test_plan = ndb_models.TestPlan(name='plan')
self.mock_test_plan.put()
self.mock_test_run = ndb_models.TestRun(
test_plan_key=self.mock_test_plan.key,
test_run_config=ndb_models.TestRunConfig(
test_key=self.mock_test.key,
run_target='run_target',
command='mock test run command',
retry_command='mock test run retry command'),
test=self.mock_test,
request_id='request_id',
state=ndb_models.TestRunState.UNKNOWN)
self.mock_test_run.put()
def testKickTestPlan_withRetry(self, create_test_run):
"""Tests that a test plan kick operations can be retried."""
test = ndb_models.Test(id='test_id')
test_run_config = ndb_models.TestRunConfig(
test_key=test.key, cluster='cluster', run_target='run_target')
test_plan = ndb_models.TestPlan(
name='test_plan', test_run_configs=[test_run_config])
test_plan.put()
# First test run creation fails, but second attempt succeeds
test_run = ndb_models.TestRun(id='test_run_id')
create_test_run.side_effect = [RuntimeError('test_run_error'), test_run]
executed = test_plan_kicker.KickTestPlan(test_plan.key.id())
self.assertTrue(executed)
# Create test run operation retried until successful
self.assertEqual(create_test_run.call_count, 2)
status = ndb_models.TestPlanStatus.query(ancestor=test_plan.key).get()
# Test run key stored and error message is empty
self.assertEqual(status.last_run_keys, [test_run.key])
self.assertIsNone(status.last_run_error)
def testCreateTestRun_withTestRunActions(self):
"""Tests that test run can be created with test run actions."""
test = ndb_models.Test(name='test', command='command')
test.put()
# Create a placeholder action with two default options
action = ndb_models.TestRunAction(
name='Foo',
hook_class_name='foo',
options=[
ndb_models.NameValuePair(name='key1', value='default'),
ndb_models.NameValuePair(name='key2', value='default'),
])
action.put()
# Create action ref with overridden and added options
test_run_config = ndb_models.TestRunConfig(
test_key=test.key,
test_run_action_refs=[
ndb_models.TestRunActionRef(
action_key=action.key,
options=[
ndb_models.NameValuePair(name='key2', value='updated'),
ndb_models.NameValuePair(name='key3', value='added'),
],
)
])
# Test run created with the right test run action
test_run = test_kicker.CreateTestRun([], test_run_config)
self.assertModelEqual(test_run_config, test_run.test_run_config)
self.assertModelListEqual(test_run.test_run_actions, [
ndb_models.TestRunAction(
name='Foo',
hook_class_name='foo',
options=[
ndb_models.NameValuePair(name='key1', value='default'),
ndb_models.NameValuePair(name='key2', value='updated'),
ndb_models.NameValuePair(name='key3', value='added'),
])
])
def _createMockTestRuns(
self,
test=None,
labels=None,
cluster='cluster',
device_specs=None,
run_count=1,
shard_count=1,
test_devices=None,
test_package_info=None,
test_resources=None,
state=None,
output_path=None,
count=1,
prev_test_run_key=None):
"""Create a mock ndb_models.TestRun object."""
device_specs = device_specs or ['device_spec']
test_runs = []
for _ in range(count):
test_run = ndb_models.TestRun(
id=str(uuid.uuid4()),
prev_test_run_key=prev_test_run_key,
test=test,
labels=labels or [],
test_run_config=ndb_models.TestRunConfig(
test_key=test.key if test is not None else None,
cluster=cluster,
device_specs=device_specs,
run_count=run_count,
shard_count=shard_count),
test_devices=test_devices or [],
test_package_info=test_package_info,
test_resources=test_resources or [],
state=state,
output_path=output_path)
test_run.put()
test_runs.append(test_run)
return test_runs
def testTrackTestInvocation_manualRetry(self, mock_time, mock_log):
# mock current time
now = datetime.datetime.now()
mock_time.return_value = now
# configure mock TFC request with 1 attempt
request_id = self.mock_test_run.request_id
mock_request = api_messages.RequestMessage(id=request_id,
command_line='command')
mock_request.command_attempts.append(
api_messages.CommandAttemptMessage(
state=common.CommandState.COMPLETED,
device_serials=['2'],
start_time=now - datetime.timedelta(seconds=200),
end_time=now - datetime.timedelta(seconds=100),
total_test_count=3,
failed_test_run_count=2,
failed_test_count=1))
# create previous test runs
first_test_run = ndb_models.TestRun(
test_run_config=ndb_models.TestRunConfig(
test_key=self.mock_test.key, run_target='run_target'),
test=self.mock_test,
request_id='request_id1',
state=ndb_models.TestRunState.UNKNOWN,
create_time=now - datetime.timedelta(seconds=5000),
total_test_count=100,
failed_test_run_count=90,
failed_test_count=80,
prev_test_context=ndb_models.TestContextObj())
first_test_run.put()
second_test_run = ndb_models.TestRun(
test_run_config=ndb_models.TestRunConfig(
test_key=self.mock_test.key, run_target='run_target'),
test=self.mock_test,
request_id='request_id2',
state=ndb_models.TestRunState.UNKNOWN,
create_time=now - datetime.timedelta(seconds=2000),
total_test_count=10,
failed_test_run_count=9,
failed_test_count=8,
prev_test_run_key=first_test_run.key)
second_test_run.put()
# update mock test run
self.mock_test_run.test_package_info = ndb_models.TestPackageInfo(
name='name', version='version')
self.mock_test_run.state = ndb_models.TestRunState.COMPLETED
self.mock_test_run.create_time = now - datetime.timedelta(seconds=1000)
self.mock_test_run.failed_test_count = 1
self.mock_test_run.prev_test_run_key = second_test_run.key
# check that right data was logged
tfc_event_handler._TrackTestInvocations(mock_request,
self.mock_test_run)
mock_log.assert_called_with(
analytics.INVOCATION_CATEGORY,
analytics.END_ACTION,
command='command',
test_run_command='mock test run command',
test_run_retry_command='mock test run retry command',
test_id=tfc_event_handler.LOCAL_ID_TAG,
test_name='name',
test_version='version',
state='COMPLETED',
failed_test_count_threshold=None,
duration_seconds=100,
elapsed_time_seconds=4900,
device_count=1,
test_count=3,
failed_module_count=2,
failed_test_count=1,
prev_total_test_count=10,
prev_failed_module_count=9,
prev_failed_test_count=8,
missing_previous_run=True,
is_sequence_run=False)
def testCreateTestRun_withNodeConfig(self, mock_find_resources):
test = ndb_models.Test(
name='test', command='command', test_resource_defs=[
ndb_models.TestResourceDef(
name='abc', default_download_url='default_download_url'),
ndb_models.TestResourceDef(
name='def', default_download_url='default_download_url'),
ndb_models.TestResourceDef(
name='xyz', default_download_url='default_download_url'),
])
test.put()
node_config = ndb_models.GetNodeConfig()
node_config.env_vars.append(
ndb_models.NameValuePair(name='foo', value='bar'))
node_config.test_resource_default_download_urls = [
ndb_models.NameValuePair(name='def', value='default_download_url2'),
ndb_models.NameValuePair(name='xyz', value='default_download_url2'),
]
node_config.put()
overwritten_obj = ndb_models.TestResourceObj(
name='xyz', url='origin_url', cache_url='cache_url')
test_run_config = ndb_models.TestRunConfig(
test_key=test.key,
cluster='cluster',
device_specs=['device_serial:serial'],
test_resource_objs=[overwritten_obj])
mock_find_resources.return_value = [overwritten_obj]
test_run = test_kicker.CreateTestRun(
['label'], test_run_config)
test_run = ndb_models.TestRun.get_by_id(test_run.key.id())
self.assertEqual(test.command, test_run.test.command)
self.assertModelListEqual(node_config.env_vars, test_run.test.env_vars)
self.assertEqual(['label'], test_run.labels)
self.assertModelEqual(test_run_config, test_run.test_run_config)
self.assertModelSetEqual([
ndb_models.TestResourceObj(
name='abc',
url='default_download_url',
decompress=False,
decompress_dir='',
mount_zip=False,
params=ndb_models.TestResourceParameters()),
ndb_models.TestResourceObj(
name='def',
url='default_download_url2',
decompress=False,
decompress_dir='',
mount_zip=False,
params=ndb_models.TestResourceParameters()),
ndb_models.TestResourceObj(
name='xyz',
url='origin_url',
cache_url='cache_url',
decompress=False,
decompress_dir='',
mount_zip=False,
params=ndb_models.TestResourceParameters()),
], test_run.test_resources)
self.assertEqual(ndb_models.TestRunState.PENDING, test_run.state)
tasks = self.mock_task_scheduler.GetTasks(
queue_names=[test_kicker.TEST_KICKER_QUEUE])
self.assertEqual(1, len(tasks))
task = tasks[0]
data = json.loads(task.payload)
self.assertEqual(test_run.key.id(), data['test_run_id'])