def testNew_withRunTarget(self, mock_run_test): test = self._createMockTest() request = { 'labels': ['label'], 'test_run_config': { 'test_id': str(test.key.id()), 'cluster': 'cluster', 'device_specs': ['foo', 'bar'], 'run_count': 10, 'shard_count': 100, 'max_retry_on_test_failures': 1000, 'test_resource_objs': [ { 'name': 'bar', 'url': 'bar_url' }, { 'name': 'zzz', 'url': 'zzz_url' }, ], }, } test_run = ndb_models.TestRun( test=test, labels=['label'], test_run_config=ndb_models.TestRunConfig( test_key=test.key, cluster='cluster', device_specs=['foo', 'bar'], run_count=10, shard_count=100, max_retry_on_test_failures=1000)) test_run.put() mock_run_test.return_value = test_run res = self.app.post_json('/_ah/api/mtt/v1/test_runs', request) mock_run_test.assert_called_with( labels=['label'], test_run_config=ndb_models.TestRunConfig( test_key=test.key, cluster='cluster', device_specs=['foo', 'bar'], run_count=10, shard_count=100, max_retry_on_test_failures=1000, test_resource_objs=[ ndb_models.TestResourceObj(name='bar', url='bar_url'), ndb_models.TestResourceObj(name='zzz', url='zzz_url'), ]), rerun_context=None, rerun_configs=[] ) self.assertEqual('200 OK', res.status) test_run_msg = protojson.decode_message(messages.TestRun, res.body) self.assertEqual( messages.Convert(test_run, messages.TestRun), test_run_msg)
def testKickTestPlan(self, create_test_run): """Tests that a test plan can be kicked off.""" test = ndb_models.Test(id='test_id') test_device_action = ndb_models.DeviceAction(id='test_device_action') test_run_action = ndb_models.TestRunAction(id='test_run_action') # Create a test plan with multiple resources and actions test_plan = ndb_models.TestPlan( name='test_plan', labels=['label'], cron_exp='0 0 * * *', test_run_configs=[ ndb_models.TestRunConfig( test_key=test.key, cluster='cluster', run_target='run_target', before_device_action_keys=[test_device_action.key], test_run_action_refs=[ ndb_models.TestRunActionRef(action_key=test_run_action.key), ], test_resource_objs=[ ndb_models.TestResourceObj(name='res_1', url='url_1')]), ]) test_plan.put() # Test run will be created successfully test_run = ndb_models.TestRun(id='test_run_id') create_test_run.return_value = test_run executed = test_plan_kicker.KickTestPlan(test_plan.key.id()) self.assertTrue(executed) # Test run is created with the right test and test plan components create_test_run.assert_called_with( labels=['label'], test_plan_key=test_plan.key, test_run_config=ndb_models.TestRunConfig( test_key=test.key, cluster='cluster', run_target='run_target', before_device_action_keys=[test_device_action.key], test_run_action_refs=[ ndb_models.TestRunActionRef(action_key=test_run_action.key), ], test_resource_objs=[ ndb_models.TestResourceObj(name='res_1', url='url_1') ]), ) # Test run key is stored in the test plan status status = ndb_models.TestPlanStatus.query(ancestor=test_plan.key).get() self.assertEqual(status.last_run_keys, [test_run.key])
def _CreateMockTestRunConfig(self, test, run_target='run_target'): """Create a mock ndb_models.TestRunConfig object.""" return ndb_models.TestRunConfig( test_key=test.key, run_target=run_target, cluster='cluster', )
def _CreateMockTestRun( self, test_name='test', command='command', retry_command_line=None, runner_sharding_args=None, run_target='run_target', shard_count=1, sharding_mode=ndb_models.ShardingMode.RUNNER, edited_command=None, module_config_pattern=None, module_execution_args=None, extra_test_resources=None): test = ndb_models.Test( name=test_name, command=command, retry_command_line=retry_command_line, runner_sharding_args=runner_sharding_args, module_config_pattern=module_config_pattern, module_execution_args=module_execution_args) test.put() test_run_config = ndb_models.TestRunConfig( test_key=test.key, cluster='cluster', run_target=run_target, shard_count=shard_count, sharding_mode=sharding_mode, command=edited_command) test_resources = [ ndb_models.TestResourceObj(name='foo', url='http://foo_origin_url'), ndb_models.TestResourceObj(name='bar', url='https://bar_origin_url'), ] + (extra_test_resources or []) test_run = ndb_models.TestRun( id=str(uuid.uuid4()), test=test, labels=['label'], test_run_config=test_run_config, test_resources=test_resources, state=ndb_models.TestRunState.PENDING) test_run.put() return test_run
def _CreateMockTestRun(self, config, state=None, create_time=None): """Creates a test run. Args: config: a test run config state: a test run state. create_time: a test run create time. Returns: a ndb_models.TestRun object. """ test = ndb_models.Test(name='test', command='command') test.put() test_run_config = ndb_models.TestRunConfig(test_key=test.key, cluster='cluster', run_target='run_target') test_resources = [ ndb_models.TestResourceObj(name='foo', url='http://foo_origin_url'), ndb_models.TestResourceObj(name='bar', url='https://bar_origin_url'), ] test_run = ndb_models.TestRun(labels=['label'], test=test, test_run_config=test_run_config, test_resources=test_resources, state=state or ndb_models.TestRunState.PENDING, create_time=create_time) test_run.key = ndb.Key(ndb_models.TestRun, str(uuid.uuid4())) test_run.put() return test_run
def testKickTestPlan_withError(self, create_test_run, set_test_run_state): """Tests that errors are handled when kicking off a test plan.""" test = ndb_models.Test(id='test_id') test_run_config = ndb_models.TestRunConfig( test_key=test.key, cluster='cluster', run_target='run_target') test_plan = ndb_models.TestPlan( name='test_plan', test_run_configs=[test_run_config, test_run_config]) test_plan.put() # First test run created successfully, but second fails even with retries test_run = ndb_models.TestRun(id='test_run_id') create_test_run.side_effect = ( [test_run] + test_plan_kicker.MAX_RETRY_COUNT * [RuntimeError('test_run_error')]) # Retries a few times before canceling test run and raising exception with self.assertRaises(RuntimeError): test_plan_kicker.KickTestPlan(test_plan.key.id()) self.assertEqual(create_test_run.call_count, 1 + test_plan_kicker.MAX_RETRY_COUNT) set_test_run_state.assert_called_once_with('test_run_id', ndb_models.TestRunState.CANCELED) # Stores the canceled test run key and the error message status = ndb_models.TestPlanStatus.query(ancestor=test_plan.key).get() self.assertEqual(status.last_run_keys, [test_run.key]) self.assertEqual(status.last_run_error, 'test_run_error')
def _TestRunConfigMessageConverter(msg): return ndb_models.TestRunConfig( test_key=ConvertToKeyOrNone(ndb_models.Test, msg.test_id), cluster=msg.cluster, command=msg.command, retry_command=msg.retry_command, device_specs=msg.device_specs, run_target=msg.run_target, run_count=msg.run_count, shard_count=msg.shard_count, sharding_mode=msg.sharding_mode, extra_args=msg.extra_args, # TODO: Deprecated retry_extra_args=msg.retry_extra_args, # TODO: Deprecated max_retry_on_test_failures=msg.max_retry_on_test_failures, queue_timeout_seconds=msg.queue_timeout_seconds, invocation_timeout_seconds=msg.invocation_timeout_seconds, output_idle_timeout_seconds=msg.output_idle_timeout_seconds, before_device_action_keys=[ ConvertToKeyOrNone(ndb_models.DeviceAction, device_action_id) for device_action_id in msg.before_device_action_ids ], test_run_action_refs=ConvertList( msg.test_run_action_refs, ndb_models.TestRunActionRef), test_resource_objs=ConvertList( msg.test_resource_objs, ndb_models.TestResourceObj), use_parallel_setup=msg.use_parallel_setup)
def _CreateMockTestRunConfig(self, test, device_action_keys=None): """Creates a mock ndb_models.TestRunConfig object.""" config = ndb_models.TestRunConfig( test_key=test.key, before_device_action_keys=device_action_keys, cluster='cluster', run_target='run_target') return config
def testRun(self, mock_test_plan_runner): """Tests test_plans.run API.""" test_run_config = ndb_models.TestRunConfig( cluster='cluster', test_key=messages.ConvertToKey(ndb_models.Test, 'test'), run_target='target') test_plan = _CreateMockTestPlan('foo', test_run_configs=[test_run_config]) self.app.post('/_ah/api/mtt/v1/test_plans/%s/run' % test_plan.key.id()) mock_test_plan_runner.assert_called_with(test_plan.key.id())
def testExecuteHook_withContextVariables(self, mock_init, mock_log, mock_execute): """Tests that a hook can be constructed and executed.""" test = ndb_models.Test(name='test', command='command') test.put() test_run = ndb_models.TestRun( test=test, test_run_config=ndb_models.TestRunConfig(test_key=test.key), test_resources=[ ndb_models.TestResourceObj( name='device_image', url='mtt:///android_ci/branch/target/build_id/image.zip', test_resource_type=ndb_models.TestResourceType.DEVICE_IMAGE ) ]) test_run.put() mock_init.return_value = None hook_context = mock.MagicMock() hook_context.test_run = test_run credentials = authorized_user.Credentials(None) action = ndb_models.TestRunAction( name='Test', hook_class_name='simple', options=[ ndb_models.NameValuePair(name='ham', value='eggs'), ndb_models.NameValuePair(name='test_run_id', value='${MTT_TEST_RUN_ID}'), ndb_models.NameValuePair(name='device_image_url', value='${MTT_DEVICE_IMAGE_URL}'), ndb_models.NameValuePair(name='device_image_branch', value='${MTT_DEVICE_IMAGE_BRANCH}'), ndb_models.NameValuePair(name='device_image_target', value='${MTT_DEVICE_IMAGE_TARGET}'), ndb_models.NameValuePair(name='device_image_build_id', value='${MTT_DEVICE_IMAGE_BUILD_ID}'), ], credentials=credentials, ) test_run_hook._ExecuteHook(action, hook_context) mock_init.assert_called_with( _credentials=credentials, ham='eggs', test_run_id=str(test_run.key.id()), device_image_url= 'mtt:///android_ci/branch/target/build_id/image.zip', device_image_branch='branch', device_image_target='target', device_image_build_id='build_id') mock_log.assert_called() mock_execute.assert_called_with(hook_context)
def testCreateTestRun(self, mock_find_resources): test = ndb_models.Test( name='test', command='command', test_resource_defs=[ ndb_models.TestResourceDef( name='foo', default_download_url='default_download_url'), ndb_models.TestResourceDef( name='bar', default_download_url='default_download_url', decompress=True, decompress_dir='dir'), ]) test.put() overwritten_obj = ndb_models.TestResourceObj( name='foo', url='origin_url', cache_url='cache_url') test_run_config = ndb_models.TestRunConfig( test_key=test.key, cluster='cluster', device_specs=['device_serial:serial'], test_resource_objs=[overwritten_obj], ) mock_find_resources.return_value = [overwritten_obj] test_run = test_kicker.CreateTestRun(['label'], test_run_config) test_run = ndb_models.TestRun.get_by_id(test_run.key.id()) self.assertModelEqual(test, test_run.test) self.assertEqual(['label'], test_run.labels) self.assertModelEqual(test_run_config, test_run.test_run_config) self.assertModelSetEqual([ ndb_models.TestResourceObj( name='bar', url='default_download_url', decompress=True, decompress_dir='dir', mount_zip=False, params=ndb_models.TestResourceParameters()), ndb_models.TestResourceObj( name='foo', url='origin_url', cache_url='cache_url', decompress=False, decompress_dir='', mount_zip=False, params=ndb_models.TestResourceParameters()), ], test_run.test_resources) self.assertEqual(ndb_models.TestRunState.PENDING, test_run.state) tasks = self.mock_task_scheduler.GetTasks( queue_names=[test_kicker.TEST_KICKER_QUEUE]) self.assertLen(tasks, 1) task = tasks[0] data = json.loads(task.payload) self.assertEqual(test_run.key.id(), data['test_run_id']) self.assertIsNone(test_run.sequence_id)
def setUp(self): super(TfcEventHandlerTest, self).setUp() self.mock_test = ndb_models.Test(name='test', command='command', result_file='result_file') self.mock_test.put() self.mock_test_plan = ndb_models.TestPlan(name='plan') self.mock_test_plan.put() self.mock_test_run = ndb_models.TestRun( test_plan_key=self.mock_test_plan.key, test_run_config=ndb_models.TestRunConfig( test_key=self.mock_test.key, run_target='run_target', command='mock test run command', retry_command='mock test run retry command'), test=self.mock_test, request_id='request_id', state=ndb_models.TestRunState.UNKNOWN) self.mock_test_run.put()
def testKickTestPlan_withRetry(self, create_test_run): """Tests that a test plan kick operations can be retried.""" test = ndb_models.Test(id='test_id') test_run_config = ndb_models.TestRunConfig( test_key=test.key, cluster='cluster', run_target='run_target') test_plan = ndb_models.TestPlan( name='test_plan', test_run_configs=[test_run_config]) test_plan.put() # First test run creation fails, but second attempt succeeds test_run = ndb_models.TestRun(id='test_run_id') create_test_run.side_effect = [RuntimeError('test_run_error'), test_run] executed = test_plan_kicker.KickTestPlan(test_plan.key.id()) self.assertTrue(executed) # Create test run operation retried until successful self.assertEqual(create_test_run.call_count, 2) status = ndb_models.TestPlanStatus.query(ancestor=test_plan.key).get() # Test run key stored and error message is empty self.assertEqual(status.last_run_keys, [test_run.key]) self.assertIsNone(status.last_run_error)
def testCreateTestRun_withTestRunActions(self): """Tests that test run can be created with test run actions.""" test = ndb_models.Test(name='test', command='command') test.put() # Create a placeholder action with two default options action = ndb_models.TestRunAction( name='Foo', hook_class_name='foo', options=[ ndb_models.NameValuePair(name='key1', value='default'), ndb_models.NameValuePair(name='key2', value='default'), ]) action.put() # Create action ref with overridden and added options test_run_config = ndb_models.TestRunConfig( test_key=test.key, test_run_action_refs=[ ndb_models.TestRunActionRef( action_key=action.key, options=[ ndb_models.NameValuePair(name='key2', value='updated'), ndb_models.NameValuePair(name='key3', value='added'), ], ) ]) # Test run created with the right test run action test_run = test_kicker.CreateTestRun([], test_run_config) self.assertModelEqual(test_run_config, test_run.test_run_config) self.assertModelListEqual(test_run.test_run_actions, [ ndb_models.TestRunAction( name='Foo', hook_class_name='foo', options=[ ndb_models.NameValuePair(name='key1', value='default'), ndb_models.NameValuePair(name='key2', value='updated'), ndb_models.NameValuePair(name='key3', value='added'), ]) ])
def _createMockTestRuns( self, test=None, labels=None, cluster='cluster', device_specs=None, run_count=1, shard_count=1, test_devices=None, test_package_info=None, test_resources=None, state=None, output_path=None, count=1, prev_test_run_key=None): """Create a mock ndb_models.TestRun object.""" device_specs = device_specs or ['device_spec'] test_runs = [] for _ in range(count): test_run = ndb_models.TestRun( id=str(uuid.uuid4()), prev_test_run_key=prev_test_run_key, test=test, labels=labels or [], test_run_config=ndb_models.TestRunConfig( test_key=test.key if test is not None else None, cluster=cluster, device_specs=device_specs, run_count=run_count, shard_count=shard_count), test_devices=test_devices or [], test_package_info=test_package_info, test_resources=test_resources or [], state=state, output_path=output_path) test_run.put() test_runs.append(test_run) return test_runs
def testTrackTestInvocation_manualRetry(self, mock_time, mock_log): # mock current time now = datetime.datetime.now() mock_time.return_value = now # configure mock TFC request with 1 attempt request_id = self.mock_test_run.request_id mock_request = api_messages.RequestMessage(id=request_id, command_line='command') mock_request.command_attempts.append( api_messages.CommandAttemptMessage( state=common.CommandState.COMPLETED, device_serials=['2'], start_time=now - datetime.timedelta(seconds=200), end_time=now - datetime.timedelta(seconds=100), total_test_count=3, failed_test_run_count=2, failed_test_count=1)) # create previous test runs first_test_run = ndb_models.TestRun( test_run_config=ndb_models.TestRunConfig( test_key=self.mock_test.key, run_target='run_target'), test=self.mock_test, request_id='request_id1', state=ndb_models.TestRunState.UNKNOWN, create_time=now - datetime.timedelta(seconds=5000), total_test_count=100, failed_test_run_count=90, failed_test_count=80, prev_test_context=ndb_models.TestContextObj()) first_test_run.put() second_test_run = ndb_models.TestRun( test_run_config=ndb_models.TestRunConfig( test_key=self.mock_test.key, run_target='run_target'), test=self.mock_test, request_id='request_id2', state=ndb_models.TestRunState.UNKNOWN, create_time=now - datetime.timedelta(seconds=2000), total_test_count=10, failed_test_run_count=9, failed_test_count=8, prev_test_run_key=first_test_run.key) second_test_run.put() # update mock test run self.mock_test_run.test_package_info = ndb_models.TestPackageInfo( name='name', version='version') self.mock_test_run.state = ndb_models.TestRunState.COMPLETED self.mock_test_run.create_time = now - datetime.timedelta(seconds=1000) self.mock_test_run.failed_test_count = 1 self.mock_test_run.prev_test_run_key = second_test_run.key # check that right data was logged tfc_event_handler._TrackTestInvocations(mock_request, self.mock_test_run) mock_log.assert_called_with( analytics.INVOCATION_CATEGORY, analytics.END_ACTION, command='command', test_run_command='mock test run command', test_run_retry_command='mock test run retry command', test_id=tfc_event_handler.LOCAL_ID_TAG, test_name='name', test_version='version', state='COMPLETED', failed_test_count_threshold=None, duration_seconds=100, elapsed_time_seconds=4900, device_count=1, test_count=3, failed_module_count=2, failed_test_count=1, prev_total_test_count=10, prev_failed_module_count=9, prev_failed_test_count=8, missing_previous_run=True, is_sequence_run=False)
def testCreateTestRun_withNodeConfig(self, mock_find_resources): test = ndb_models.Test( name='test', command='command', test_resource_defs=[ ndb_models.TestResourceDef( name='abc', default_download_url='default_download_url'), ndb_models.TestResourceDef( name='def', default_download_url='default_download_url'), ndb_models.TestResourceDef( name='xyz', default_download_url='default_download_url'), ]) test.put() node_config = ndb_models.GetNodeConfig() node_config.env_vars.append( ndb_models.NameValuePair(name='foo', value='bar')) node_config.test_resource_default_download_urls = [ ndb_models.NameValuePair(name='def', value='default_download_url2'), ndb_models.NameValuePair(name='xyz', value='default_download_url2'), ] node_config.put() overwritten_obj = ndb_models.TestResourceObj( name='xyz', url='origin_url', cache_url='cache_url') test_run_config = ndb_models.TestRunConfig( test_key=test.key, cluster='cluster', device_specs=['device_serial:serial'], test_resource_objs=[overwritten_obj]) mock_find_resources.return_value = [overwritten_obj] test_run = test_kicker.CreateTestRun( ['label'], test_run_config) test_run = ndb_models.TestRun.get_by_id(test_run.key.id()) self.assertEqual(test.command, test_run.test.command) self.assertModelListEqual(node_config.env_vars, test_run.test.env_vars) self.assertEqual(['label'], test_run.labels) self.assertModelEqual(test_run_config, test_run.test_run_config) self.assertModelSetEqual([ ndb_models.TestResourceObj( name='abc', url='default_download_url', decompress=False, decompress_dir='', mount_zip=False, params=ndb_models.TestResourceParameters()), ndb_models.TestResourceObj( name='def', url='default_download_url2', decompress=False, decompress_dir='', mount_zip=False, params=ndb_models.TestResourceParameters()), ndb_models.TestResourceObj( name='xyz', url='origin_url', cache_url='cache_url', decompress=False, decompress_dir='', mount_zip=False, params=ndb_models.TestResourceParameters()), ], test_run.test_resources) self.assertEqual(ndb_models.TestRunState.PENDING, test_run.state) tasks = self.mock_task_scheduler.GetTasks( queue_names=[test_kicker.TEST_KICKER_QUEUE]) self.assertEqual(1, len(tasks)) task = tasks[0] data = json.loads(task.payload) self.assertEqual(test_run.key.id(), data['test_run_id'])