def test_without_buckets(self): self.mock_cannot(user.Action.SEARCH_BUILDS, 'chromium/ci') build1 = self.put_build(builder=dict(bucket='try')) build2 = self.put_build(builder=dict(bucket='ci')) builds, _ = self.search() self.assertEqual(builds, [build1]) builds, _ = self.search(tags=[self.INDEXED_TAG]) self.assertEqual(builds, [build1]) # All buckets are available. user.get_accessible_buckets_async.return_value = future(None) user.can_async.side_effect = None builds, _ = self.search() self.assertEqual(builds, [build2, build1]) builds, _ = self.search(tags=[self.INDEXED_TAG]) self.assertEqual(builds, [build2, build1]) # No buckets are available. user.get_accessible_buckets_async.return_value = future(set()) self.mock_cannot(user.Action.SEARCH_BUILDS) builds, _ = self.search() self.assertEqual(builds, []) builds, _ = self.search(tags=[self.INDEXED_TAG]) self.assertEqual(builds, [])
def setUp(self): super(BaseTest, self).setUp() user.clear_request_cache() self.patch('tq.enqueue_async', autospec=True, return_value=future(None)) self.now = NOW self.patch( 'components.utils.utcnow', autospec=True, side_effect=lambda: self.now ) self.settings = service_config_pb2.SettingsCfg( swarming=dict( milo_hostname='milo.example.com', bbagent_package=dict( package_name='infra/tools/bbagent', version='luci-runner-version', version_canary='luci-runner-version-canary', builders=service_config_pb2.BuilderPredicate( regex=['chromium/try/linux'], ), ), kitchen_package=dict( package_name='infra/tools/kitchen', version='kitchen-version', version_canary='kitchen-version-canary', ), user_packages=[ dict( package_name='infra/tools/git', version='git-version', version_canary='git-version-canary', ), dict( package_name='infra/cpython/python', version='py-version', subdir='python', ), dict( package_name='infra/excluded', version='excluded-version', builders=service_config_pb2.BuilderPredicate( regex_exclude=['.*'], ), ), ], ), known_public_gerrit_hosts=['chromium-review.googlesource.com'], ) self.patch( 'config.get_settings_async', autospec=True, return_value=future(self.settings) )
def setUp(self): super(ExpireBuildTests, self).setUp() self.now = datetime.datetime(2015, 1, 1) self.patch('components.utils.utcnow', side_effect=lambda: self.now) self.patch('tq.enqueue_async', autospec=True, return_value=future(None))
def setUp(self): super(SyncBuildTest, self).setUp() self.patch('components.net.json_request_async', autospec=True) self.patch('components.auth.delegate_async', return_value=future('blah')) self.build_token = 'beeff00d' self.patch( 'tokens.generate_build_token', autospec=True, return_value='deadbeef' ) self.task_def = {'is_task_def': True, 'task_slices': [{ 'properties': {}, }]} self.patch( 'swarming.compute_task_def', autospec=True, return_value=self.task_def ) self.patch( 'google.appengine.api.app_identity.get_default_version_hostname', return_value='cr-buildbucket.appspot.com' ) self.build_bundle = test_util.build_bundle( id=1, created_by='user:[email protected]' ) self.build_bundle.build.swarming_task_key = None with self.build_bundle.infra.mutate() as infra: infra.swarming.task_id = '' self.build_bundle.put()
def test_put_with_generic_buildset(self, add_async): tags = [ dict(key='buildset', value='x'), dict(key='t', value='0'), ] build = test_util.build(id=1, tags=tags) add_async.return_value = future(build) req = { 'client_operation_id': '42', 'bucket': 'luci.chromium.try', 'tags': ['buildset:x', 't:0'], 'parameters_json': json.dumps({api_common.BUILDER_PARAMETER: 'linux'}), } resp = self.call_api('put', req).json_body add_async.assert_called_once_with( creation.BuildRequest( schedule_build_request=rpc_pb2.ScheduleBuildRequest( builder=dict( project='chromium', bucket='try', builder='linux', ), tags=tags, request_id='42', properties=dict(), ), parameters={}, ) ) self.assertEqual(resp['build']['id'], '1') self.assertIn('buildset:x', resp['build']['tags']) self.assertIn('t:0', resp['build']['tags'])
def setUp(self): super(CreateTaskTest, self).setUp() self.patch('components.net.json_request_async', autospec=True) self.patch('components.auth.delegate_async', return_value=future('blah')) self.build_token = 'beeff00d' self.patch('tokens.generate_build_token', autospec=True, return_value='deadbeef') self.task_def = { 'is_task_def': True, 'task_slices': [{ 'properties': {}, }] } self.patch('swarming.prepare_task_def', autospec=True, return_value=self.task_def) self.build = test_util.build(id=1, created_by='user:[email protected]') self.build.swarming_task_key = None with self.build.mutate_infra() as infra: infra.swarming.task_id = '' self.build.put()
def test_success(self): expected_task_def = self.task_def.copy() expected_secrets = launcher_pb2.BuildSecrets( build_token=self.build_token) expected_task_def[u'task_slices'][0][u'properties'][ u'secret_bytes'] = (base64.b64encode( expected_secrets.SerializeToString())) net.json_request_async.return_value = future({'task_id': 'x'}) swarming._create_swarming_task(1) actual_task_def = net.json_request_async.call_args[1]['payload'] self.assertEqual(actual_task_def, expected_task_def) self.assertEqual( net.json_request_async.call_args[0][0], 'https://swarming.example.com/_ah/api/swarming/v1/tasks/new') # Test delegation token params. self.assertEqual(auth.delegate_async.mock_calls, [ mock.call( services=[u'https://swarming.example.com'], audience=[auth.Identity('user', 'test@localhost')], impersonate=auth.Identity('user', '*****@*****.**'), tags=['buildbucket:bucket:chromium/try'], ) ])
def test_filter_by_project_admin(self): user.get_accessible_buckets_async.return_value = future(None) build = self.put_build(builder=dict(project='chromium')) self.put_build(builder=dict(project='v8')) builds, _ = self.search(project='chromium') self.assertEqual(builds, [build])
def test_put_batch_with_exception(self, add_many_async): add_many_async.return_value = future([(None, Exception())]) req = { 'builds': [{ 'bucket': 'luci.chromium.try' }], } self.call_api('put_batch', req, status=500)
def test_get_build_url_milo(self, get_settings_async): get_settings_async.return_value = test_util.future( service_config_pb2.SettingsCfg(swarming=dict( milo_hostname='milo.example.com')), ) build = test_util.build(id=1) build.url = None self.assertEqual(api_common.get_build_url(build), 'https://milo.example.com/b/1')
def test_put_with_gerrit_change(self, add_async): buildset = 'patch/gerrit/gerrit.example.com/1234/5' buildset_tag = 'buildset:' + buildset props = {'patch_project': 'repo'} expected_sbr = rpc_pb2.ScheduleBuildRequest( builder=dict( project='chromium', bucket='try', builder='linux', ), gerrit_changes=[ dict( host='gerrit.example.com', project='repo', change=1234, patchset=5, ) ], tags=[dict(key='t', value='0')], request_id='42', properties=bbutil.dict_to_struct(props), ) expected_request = creation.BuildRequest( schedule_build_request=expected_sbr, parameters={}, ) build = test_util.build( id=1, input=dict( gerrit_changes=expected_sbr.gerrit_changes, properties=expected_sbr.properties, ), tags=expected_sbr.tags, ) build.tags.append(buildset_tag) build.tags.sort() add_async.return_value = future(build) req = { 'client_operation_id': '42', 'bucket': 'luci.chromium.try', 'tags': [buildset_tag, 't:0'], 'parameters_json': json.dumps({ api_common.BUILDER_PARAMETER: 'linux', api_common.PROPERTIES_PARAMETER: props, }), } resp = self.call_api('put', req).json_body add_async.assert_called_once_with(expected_request) self.assertEqual(resp['build']['id'], '1') self.assertIn(buildset_tag, resp['build']['tags']) self.assertIn('t:0', resp['build']['tags'])
def test_sync_build_async_no_task(self, load_task_result_async): load_task_result_async.return_value = future(None) build = test_util.build() build.put() swarming.CronUpdateBuilds().update_build_async(build).get_result() build = build.key.get() self.assertEqual(build.proto.status, common_pb2.INFRA_FAILURE) self.assertTrue(build.proto.summary_markdown)
def setUp(self): super(UserTest, self).setUp() self.current_identity = auth.Identity.from_bytes('user:[email protected]') self.patch('components.auth.get_current_identity', autospec=True, side_effect=lambda: self.current_identity) user.clear_request_cache() self.patch('components.auth.is_admin', autospec=True, return_value=False) bucket_a = Bucket(name='a', acls=[ Acl(role=Acl.WRITER, group='a-writers'), Acl(role=Acl.READER, group='a-readers'), ]) bucket_b = Bucket(name='b', acls=[ Acl(role=Acl.WRITER, group='b-writers'), Acl(role=Acl.READER, group='b-readers'), ]) bucket_c = Bucket(name='c', acls=[ Acl(role=Acl.READER, group='c-readers'), Acl(role=Acl.READER, identity='user:[email protected]'), Acl(role=Acl.WRITER, group='c-writers'), Acl(role=Acl.READER, identity='project:p1'), ]) all_buckets = [('p1', bucket_a), ('p2', bucket_b), ('p3', bucket_c)] self.patch('config.get_buckets_async', autospec=True, return_value=future(all_buckets)) bucket_map = { config.format_bucket_id(pid, b.name): b for pid, b in all_buckets } self.patch('config.get_bucket_async', autospec=True, side_effect=lambda bid: future( ('deadbeef', bucket_map.get(bid))))
def test_update_global_metrics(self, set_build_count_metric_async, set_build_latency): set_build_count_metric_async.return_value = future(None) set_build_latency.return_value = future(None) model.Builder(id='chromium:luci.chromium.try:release').put() model.Builder(id='chromium:luci.chromium.try:debug').put() model.Builder(id='chromium:try:debug').put() config.put_bucket( 'chromium', 'a' * 40, test_util.parse_bucket_cfg(''' name: "luci.chromium.try" swarming { builders {} } '''), ) metrics.update_global_metrics() set_build_latency.assert_any_call('chromium/try', 'luci.chromium.try', 'release', True) set_build_latency.assert_any_call('chromium/try', 'luci.chromium.try', 'release', False) set_build_latency.assert_any_call('chromium/try', 'luci.chromium.try', 'debug', True) set_build_latency.assert_any_call('chromium/try', 'luci.chromium.try', 'debug', False) set_build_count_metric_async.assert_any_call( 'chromium/try', 'luci.chromium.try', 'release', model.BuildStatus.SCHEDULED, False) set_build_count_metric_async.assert_any_call( 'chromium/try', 'luci.chromium.try', 'release', model.BuildStatus.SCHEDULED, True) set_build_count_metric_async.assert_any_call( 'chromium/try', 'luci.chromium.try', 'debug', model.BuildStatus.SCHEDULED, False) set_build_count_metric_async.assert_any_call( 'chromium/try', 'luci.chromium.try', 'debug', model.BuildStatus.SCHEDULED, True)
def test_update_global_metrics(self, set_build_status_metric, get_buckets_async): get_buckets_async.return_value = future( [project_config_pb2.Bucket(name='x')]) metrics.update_global_metrics() set_build_status_metric.assert_any_call(metrics.CURRENTLY_PENDING, 'x', model.BuildStatus.SCHEDULED) set_build_status_metric.assert_any_call(metrics.CURRENTLY_RUNNING, 'x', model.BuildStatus.STARTED)
def test_delegate_async(self, delegate_async): delegate_async.return_value = future('token') token = user.delegate_async('swarming.example.com', tag='buildbucket:bucket:x').get_result() self.assertEqual(token, 'token') delegate_async.assert_called_with( audience=[user.self_identity()], services=['https://swarming.example.com'], impersonate=auth.get_current_identity(), tags=['buildbucket:bucket:x'], )
def setUp(self): super(SearchTest, self).setUp() self.current_identity = auth.Identity('service', 'unittest') self.patch('components.auth.get_current_identity', side_effect=lambda: self.current_identity) self.patch('user.can_async', return_value=future(True)) self.now = datetime.datetime(2015, 1, 1) self.patch('components.utils.utcnow', side_effect=lambda: self.now) self.chromium_try = project_config_pb2.Bucket(name='try') self.patch('config.get_bucket_async', return_value=future({'deadbeef': self.chromium_try})) self.patch( 'user.get_accessible_buckets_async', autospec=True, return_value=future({'chromium/try'}), ) self.patch('search.TagIndex.random_shard_index', return_value=0)
def test_sync_build_async(self, load_task_result_async): load_task_result_async.return_value = future({ 'state': 'RUNNING', }) build = test_util.build() build.put() swarming.CronUpdateBuilds().update_build_async(build).get_result() build = build.key.get() self.assertEqual(build.proto.status, common_pb2.STARTED) self.assertFalse(build.proto.HasField('end_time')) load_task_result_async.return_value = future({ 'state': 'COMPLETED', }) swarming.CronUpdateBuilds().update_build_async(build).get_result() build = build.key.get() self.assertEqual(build.proto.status, common_pb2.SUCCESS)
def test_create_task(self): expected_task_def = self.task_def.copy() expected_secrets = launcher_pb2.BuildSecrets(build_token=self.build_token) expected_task_def[u'task_slices'][0][u'properties'][u'secret_bytes'] = ( base64.b64encode(expected_secrets.SerializeToString()) ) net.json_request_async.return_value = future({'task_id': 'x'}) swarming._sync_build_and_swarming(1, 0) actual_task_def = net.json_request_async.call_args[1]['payload'] self.assertEqual(actual_task_def, expected_task_def) self.assertEqual( net.json_request_async.call_args[0][0], 'https://swarming.example.com/_ah/api/swarming/v1/tasks/new' ) # Test delegation token params. self.assertEqual( auth.delegate_async.mock_calls, [ mock.call( services=[u'https://swarming.example.com'], audience=[auth.Identity('user', 'test@localhost')], impersonate=auth.Identity('user', '*****@*****.**'), tags=['buildbucket:bucket:chromium/try'], ) ] ) # Assert that we've persisted information about the new task. bundle = model.BuildBundle.get(1, infra=True) self.assertIsNotNone(bundle) self.assertTrue(bundle.build.swarming_task_key) self.assertTrue(bundle.infra.parse().swarming.task_id) expected_continuation_payload = { 'id': 1, 'generation': 1, } expected_continuation = { 'name': 'sync-task-1-1', 'url': '/internal/task/swarming/sync-build/1', 'payload': json.dumps(expected_continuation_payload, sort_keys=True), 'retry_options': { 'task_age_limit': model.BUILD_TIMEOUT.total_seconds() }, 'countdown': 60, } tq.enqueue_async.assert_called_with( swarming.SYNC_QUEUE_NAME, [expected_continuation], transactional=False )
def test_update_build_async(self, load_task_result_async): load_task_result_async.return_value = future({ 'state': 'RUNNING', }) build = self.build swarming.CronUpdateBuilds().update_build_async(build).get_result() build = build.key.get() self.assertEqual(build.status, model.BuildStatus.STARTED) self.assertIsNotNone(build.lease_key) self.assertIsNone(build.complete_time) load_task_result_async.return_value = future({ 'state': 'COMPLETED', }) swarming.CronUpdateBuilds().update_build_async(build).get_result() build = build.key.get() self.assertEqual(build.status, model.BuildStatus.COMPLETED) self.assertEqual(build.result, model.BuildResult.SUCCESS) self.assertIsNone(build.lease_key) self.assertIsNotNone(build.complete_time)
def test_update_build_async_no_task(self, load_task_result_async): load_task_result_async.return_value = future(None) build = self.build swarming.CronUpdateBuilds().update_build_async(build).get_result() build = build.key.get() self.assertEqual(build.status, model.BuildStatus.COMPLETED) self.assertEqual(build.result, model.BuildResult.FAILURE) self.assertEqual(build.failure_reason, model.FailureReason.INFRA_FAILURE) self.assertIsNotNone(build.result_details) self.assertIsNone(build.lease_key) self.assertIsNotNone(build.complete_time)
def get_self_config_async(path, *_args, **_kwargs): if path not in ( 'swarming_task_template.json', 'swarming_task_template_canary.json'): # pragma: no cover self.fail() if path == 'swarming_task_template.json': template = self.task_template else: template = self.task_template_canary return future( ('template_rev', json.dumps(template) if template is not None else None))
def setUp(self): super(AclTest, self).setUp() self.current_identity = auth.Identity.from_bytes('user:[email protected]') self.patch( 'components.auth.get_current_identity', autospec=True, return_value=self.current_identity) self.patch('components.auth.is_admin', autospec=True, return_value=False) bucket_a = Bucket( name='a', acls=[ Acl(role=Acl.WRITER, group='a-writers'), Acl(role=Acl.READER, group='a-readers'), ]) bucket_b = Bucket( name='b', acls=[ Acl(role=Acl.WRITER, group='b-writers'), Acl(role=Acl.READER, group='b-readers'), ]) bucket_c = Bucket( name='c', acls=[ Acl(role=Acl.READER, group='c-readers'), Acl(role=Acl.READER, identity='user:[email protected]'), Acl(role=Acl.WRITER, group='c-writers'), ]) all_buckets = [bucket_a, bucket_b, bucket_c] self.patch( 'config.get_buckets_async', autospec=True, return_value=future(all_buckets)) bucket_map = {b.name: b for b in all_buckets} self.patch( 'config.get_bucket_async', autospec=True, side_effect=lambda name: future(('chromium', bucket_map.get(name))))
def test_put_with_commit(self, add_async): buildset = ( 'commit/gitiles/gitiles.example.com/chromium/src/+/' 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa' ) buildset_tag = 'buildset:' + buildset gitiles_ref_tag = 'gitiles_ref:refs/heads/master' gitiles_commit = common_pb2.GitilesCommit( host='gitiles.example.com', project='chromium/src', ref='refs/heads/master', id='aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa', ) build = test_util.build( id=1, input=dict(gitiles_commit=gitiles_commit), tags=[dict(key='t', value='0')], ) build.tags.append(buildset_tag) build.tags.append(gitiles_ref_tag) build.tags.sort() add_async.return_value = future(build) req = { 'client_operation_id': '42', 'bucket': 'luci.chromium.try', 'tags': [buildset_tag, gitiles_ref_tag, 't:0'], 'parameters_json': json.dumps({api_common.BUILDER_PARAMETER: 'linux'}), } resp = self.call_api('put', req).json_body add_async.assert_called_once_with( creation.BuildRequest( schedule_build_request=rpc_pb2.ScheduleBuildRequest( builder=dict( project='chromium', bucket='try', builder='linux', ), gitiles_commit=gitiles_commit, tags=[dict(key='t', value='0')], request_id='42', properties=dict(), ), parameters={}, ) ) self.assertEqual(resp['build']['id'], '1') self.assertIn(buildset_tag, resp['build']['tags']) self.assertIn(gitiles_ref_tag, resp['build']['tags']) self.assertIn('t:0', resp['build']['tags'])
def callback_test(self, build): with mock.patch('notifications.enqueue_notifications_async', autospec=True): notifications.enqueue_notifications_async.return_value = future( None) build.pubsub_callback = model.PubSubCallback( topic='projects/example/topics/buildbucket', user_data='hello', auth_token='secret', ) build.put() yield build = build.key.get() notifications.enqueue_notifications_async.assert_called_with(build)
def test_get_accessible_buckets_async_admin(self, is_admin): is_admin.return_value = True config.get_buckets_async.return_value = future([ ('p1', Bucket( name='available_bucket1', acls=[ Acl(role=Acl.READER, group='xxx'), Acl(role=Acl.WRITER, group='yyy') ], )), ]) self.assertIsNone(user.get_accessible_buckets_async().get_result())
def test_enqueue_bq_export_async(self, enqueue_async): enqueue_async.return_value = test_util.future(None) build = test_util.build(id=1, status=common_pb2.SUCCESS) ndb.transactional( # pylint: disable=no-value-for-parameter lambda: bq.enqueue_bq_export_async(build).get_result())() task_def = { 'method': 'PULL', 'payload': { 'id': 1 }, } enqueue_async.assert_any_call('bq-export', [task_def])
def test_put_with_leasing(self, add_async): expiration = utils.utcnow() + datetime.timedelta(hours=1) build = test_util.build(id=1) build.lease_expiration_date = expiration add_async.return_value = future(build) req = { 'bucket': 'luci.chromium.try', 'lease_expiration_ts': str(utils.datetime_to_timestamp(expiration)), } resp = self.call_api('put', req).json_body build_req = add_async.call_args[0][0] self.assertEqual(build_req.lease_expiration_date, expiration) self.assertEqual( resp['build']['lease_expiration_ts'], req['lease_expiration_ts'] )
def test_get_accessible_buckets_async(self, is_group_member): is_group_member.side_effect = lambda g, _=None: g in ('xxx', 'yyy') config.get_buckets_async.return_value = future({ 'p1/available_bucket1': Bucket( name='available_bucket1', acls=[ Acl(role=Acl.READER, group='xxx'), Acl(role=Acl.WRITER, group='yyy') ], ), 'p2/available_bucket2': Bucket( name='available_bucket2', acls=[ Acl(role=Acl.READER, group='xxx'), Acl(role=Acl.WRITER, group='zzz') ], ), 'p3/available_bucket3': Bucket( name='available_bucket3', acls=[ Acl(role=Acl.READER, identity='user:[email protected]'), ], ), 'p4/not_available_bucket': Bucket( name='not_available_bucket', acls=[Acl(role=Acl.WRITER, group='zzz')], ), }) # call twice for per-request caching of futures. user.get_accessible_buckets_async() availble_buckets = user.get_accessible_buckets_async().get_result() expected = { 'p1/available_bucket1', 'p2/available_bucket2', 'p3/available_bucket3', } self.assertEqual(availble_buckets, expected) # call again for memcache coverage. user.clear_request_cache() availble_buckets = user.get_accessible_buckets_async().get_result() self.assertEqual(availble_buckets, expected)
def test_cancel_with_details(self, cancel): build = test_util.build(id=1) cancel.return_value = future(build) props = {'a': 'b'} model.BuildOutputProperties( key=model.BuildOutputProperties.key_for(build.key), properties=bbutil.dict_to_struct(props).SerializeToString(), ).put() result_details = {'properties': props} req = {'id': '1', 'result_details_json': json.dumps(result_details)} res = self.call_api('cancel', req).json_body cancel.assert_called_once_with(1, result_details=result_details) self.assertEqual(res['build']['result_details_json'], req['result_details_json'])