def test_update_tag_indexes_async(self): builds = [ test_util.build(id=1, builder=dict(project='chromium'), tags=[ dict(key='buildset', value='common'), dict(key='buildset', value='unique'), ]), test_util.build(id=2, builder=dict(project='v8'), tags=[ dict(key='buildset', value='common'), ]), ] ndb.Future.wait_all(search.update_tag_indexes_async(builds)) common = search.TagIndex.get_by_id('buildset:common') self.assertEqual( {e.build_id for e in common.entries}, {1, 2}, ) unique = search.TagIndex.get_by_id('buildset:unique') self.assertEqual( {e.build_id for e in unique.entries}, {1}, )
def test_get_and_search(self, search_async, get_async): search_async.return_value = future( ([test_util.build(id=1), test_util.build(id=2)], '')) get_async.return_value = future(test_util.build(id=3)) req = rpc_pb2.BatchRequest(requests=[ dict(search_builds=dict(predicate=dict(builder=dict( project='chromium', bucket='try', builder='linux-rel', ), ), ), ), dict(get_build=dict(id=3)), ], ) res = self.call(self.api.Batch, req) search_async.assert_called_once_with( search.Query( bucket_ids=['chromium/try'], builder='linux-rel', status=common_pb2.STATUS_UNSPECIFIED, include_experimental=False, tags=[], start_cursor='', ), ) get_async.assert_called_once_with(3) self.assertEqual(len(res.responses), 2) self.assertEqual(len(res.responses[0].search_builds.builds), 2) self.assertEqual(res.responses[0].search_builds.builds[0].id, 1L) self.assertEqual(res.responses[0].search_builds.builds[1].id, 2L) self.assertEqual(res.responses[1].get_build.id, 3L)
def test_basic(self, search_async): builds = [test_util.build(id=54), test_util.build(id=55)] search_async.return_value = future((builds, 'next page token')) req = rpc_pb2.SearchBuildsRequest( predicate=dict(builder=dict(project='chromium', bucket='try', builder='linux-try'), ), page_size=10, page_token='page token', ) res = self.call(self.api.SearchBuilds, req) search_async.assert_called_once_with( search.Query( bucket_ids=['chromium/try'], builder='linux-try', include_experimental=False, tags=[], status=common_pb2.STATUS_UNSPECIFIED, max_builds=10, start_cursor='page token', )) self.assertEqual(len(res.builds), 2) self.assertEqual(res.builds[0].id, 54) self.assertEqual(res.builds[1].id, 55) self.assertEqual(res.next_page_token, 'next page token')
def test_delete_many_scheduled_builds(self): scheduled_build = test_util.build(id=1, status=common_pb2.SCHEDULED) completed_build = test_util.build(id=2, status=common_pb2.SUCCESS) scheduled_build.put() completed_build.put() self.assertIsNotNone(scheduled_build.key.get()) self.assertIsNotNone(completed_build.key.get()) service._task_delete_many_builds(scheduled_build.bucket_id, model.BuildStatus.SCHEDULED) self.assertIsNone(scheduled_build.key.get()) self.assertIsNotNone(completed_build.key.get())
def test_cancel(self, cancel_task_async): test_util.build(id=1).put() build = service.cancel_async(1, summary_markdown='nope').get_result() self.assertEqual(build.proto.status, common_pb2.CANCELED) self.assertEqual(build.proto.end_time.ToDatetime(), utils.utcnow()) self.assertEqual(build.proto.summary_markdown, 'nope') self.assertEqual(build.proto.canceled_by, self.current_identity.to_bytes()) cancel_task_async.assert_called_with('swarming.example.com', 'deadbeef') self.assertEqual(build.status_changed_time, utils.utcnow())
def test_ended_build(self): test_util.build(id=123, status=common_pb2.SUCCESS).put() req, ctx = self._mk_update_req(build_pb2.Build(id=123)) self.call( self.api.UpdateBuild, req, ctx=ctx, expected_code=prpc.StatusCode.FAILED_PRECONDITION, expected_details='Cannot update an ended build', )
def mkbuild(bucket='try', builder='release', status=common_pb2.SCHEDULED, experimental=False): test_util.build( builder=dict(project='chromium', bucket=bucket, builder=builder), status=status, input=dict(experimental=experimental), ).put()
def test_delete_many_started_builds(self): scheduled_build = test_util.build(id=1, status=common_pb2.SCHEDULED) started_build = test_util.build(id=2, status=common_pb2.STARTED) completed_build = test_util.build(id=3, status=common_pb2.SUCCESS) ndb.put_multi([scheduled_build, started_build, completed_build]) service._task_delete_many_builds(scheduled_build.bucket_id, model.BuildStatus.STARTED) self.assertIsNotNone(scheduled_build.key.get()) self.assertIsNone(started_build.key.get()) self.assertIsNotNone(completed_build.key.get())
def test_update_steps_of_scheduled_build(self): test_util.build(id=123, status=common_pb2.SCHEDULED).put() build_proto = build_pb2.Build(id=123) req, ctx = self._mk_update_req(build_proto, paths=['build.steps']) self.call( self.api.UpdateBuild, req, ctx=ctx, expected_code=prpc.StatusCode.INVALID_ARGUMENT, )
def test_delete_many_builds_created_by(self): build1 = test_util.build(id=1, created_by='user:[email protected]') build2 = test_util.build(id=2, created_by='user:[email protected]') ndb.put_multi([build1, build2]) service._task_delete_many_builds( build1.bucket_id, model.BuildStatus.SCHEDULED, created_by=build2.created_by, ) self.assertIsNone(build2.key.get()) self.assertIsNotNone(build1.key.get())
def test_missing_token(self): test_util.build(id=123).put() build = build_pb2.Build( id=123, status=common_pb2.STARTED, ) req, ctx = self._mk_update_req(build, token=None) self.call( self.api.UpdateBuild, req, ctx=ctx, expected_code=prpc.StatusCode.UNAUTHENTICATED, expected_details='missing token in build update request', )
def test_schedule_build_requests(self, add_many_async): add_many_async.return_value = future([ (test_util.build(id=42), None), (test_util.build(id=43), None), (None, errors.InvalidInputError('bad')), (None, Exception('unexpected')), (None, auth.AuthorizationError('bad')), ]) user.can_async.side_effect = ( lambda bucket_id, _: future('forbidden' not in bucket_id)) linux_builder = dict(project='chromium', bucket='try', builder='linux') win_builder = dict(project='chromium', bucket='try', builder='windows') req = rpc_pb2.BatchRequest( requests=[ dict(schedule_build=dict(builder=linux_builder)), dict(schedule_build=dict(builder=linux_builder, fields=dict(paths=['tags']))), dict(schedule_build=dict(builder=linux_builder, fields=dict(paths=['wrong-field']))), dict(schedule_build=dict(builder=win_builder)), dict(schedule_build=dict(builder=win_builder)), dict(schedule_build=dict(builder=win_builder)), dict(schedule_build=dict(builder=dict(project='chromium', bucket='forbidden', builder='nope'), )), dict( schedule_build=dict(), # invalid request ), ], ) res = self.call(self.api.Batch, req) codes = [r.error.code for r in res.responses] self.assertEqual(codes, [ prpc.StatusCode.OK.value, prpc.StatusCode.OK.value, prpc.StatusCode.INVALID_ARGUMENT.value, prpc.StatusCode.INVALID_ARGUMENT.value, prpc.StatusCode.INTERNAL.value, prpc.StatusCode.PERMISSION_DENIED.value, prpc.StatusCode.PERMISSION_DENIED.value, prpc.StatusCode.INVALID_ARGUMENT.value, ]) self.assertEqual(res.responses[0].schedule_build.id, 42) self.assertFalse(len(res.responses[0].schedule_build.tags)) self.assertTrue(len(res.responses[1].schedule_build.tags))
def test_build_to_dict_non_luci(self): build = test_util.build(builder=dict(bucket='master.chromium')) build.is_luci = False actual = api_common.build_to_dict(build, None) self.assertEqual(actual['project'], 'chromium') self.assertEqual(actual['bucket'], 'master.chromium')
def test_fields_for(self): build = test_util.build( builder=dict(project='chromium', bucket='try', builder='linux'), status=common_pb2.FAILURE, tags=[ dict(key='user_agent', value='cq'), dict(key='something', value='else'), ], canary=common_pb2.YES, ) expected = { 'bucket': 'luci.chromium.try', 'builder': 'linux', 'canary': True, 'user_agent': 'cq', 'status': 'COMPLETED', 'result': 'FAILURE', 'failure_reason': 'BUILD_FAILURE', 'cancelation_reason': '', } self.assertEqual(set(expected), set(metrics._BUILD_FIELDS)) actual = metrics._fields_for(build, expected.keys()) self.assertEqual(expected, actual) with self.assertRaises(ValueError): metrics._fields_for(build, ['wrong field'])
def test_cron_export_builds_to_bq_insert_errors(self, delete_tasks): builds = [ test_util.build(id=i + 1, status=common_pb2.SUCCESS) for i in xrange(3) ] ndb.put_multi(builds) tasks = [ taskqueue.Task(method='PULL', payload=json.dumps({'id': b.key.id()})) for b in builds ] self.queue.add(tasks) net.json_request.return_value = { 'insertErrors': [{ 'index': 1, 'errors': [{ 'reason': 'bad', 'message': ':(' }], }] } bq._process_pull_task_batch(self.queue.name, 'builds') self.assertTrue(net.json_request.called) # assert second task is not deleted deleted = delete_tasks.call_args[0][1] self.assertEqual( [t.payload for t in deleted], [tasks[0].payload, tasks[2].payload], )
def test_trimming_exclude(self, get_async): get_async.return_value = future( test_util.build(input=dict( properties=bbutil.dict_to_struct({'a': 'b'}))), ) req = rpc_pb2.GetBuildRequest(id=1) res = self.call(self.api.GetBuild, req) self.assertFalse(res.input.HasField('properties'))
def test_failed(self, on_build_completed, on_build_completing_async): steps = model.BuildSteps.make( build_pb2.Build( id=123, steps=[dict(name='step', status=common_pb2.SCHEDULED)], )) steps.put() on_build_completing_async.return_value = future(None) build = test_util.build(id=123) build.put() req, ctx = self._mk_update_req( build_pb2.Build( id=123, status=common_pb2.FAILURE, summary_markdown='bad', ), paths=['build.status', 'build.summary_markdown'], ) self.call(self.api.UpdateBuild, req, ctx=ctx) build = build.key.get() self.assertEqual(build.proto.status, common_pb2.FAILURE) self.assertEqual(build.proto.summary_markdown, 'bad') self.assertEqual(build.proto.end_time.ToDatetime(), self.now) on_build_completing_async.assert_called_once_with(build) on_build_completed.assert_called_once_with(build) steps = steps.key.get() step_container = build_pb2.Build() steps.read_steps(step_container) self.assertEqual(step_container.steps[0].status, common_pb2.CANCELED)
def test_process(self): builds = [ test_util.build( id=i, tags=[ dict(key='t', value='%d' % (i % 3)), dict(key='a', value='b'), ], ) for i in xrange(50, 60) ] backfill_tag_index._process_builds(builds, 't', 5) backfill_tag_index._enqueue_flush_entries.assert_called_with( 't', { '0': [ ['chromium/try', 51], ['chromium/try', 54], ], '1': [['chromium/try', 52]], '2': [ ['chromium/try', 50], ['chromium/try', 53], ], })
def test_infra(self): build = test_util.build(infra=dict(swarming=dict( hostname='swarming.example.com'))) actual = self.to_proto(build, load_infra=True) self.assertEqual(actual.infra.swarming.hostname, 'swarming.example.com')
def test_lease_unsuccessful(self, lease): lease.return_value = (False, test_util.build(id=1)) req = { 'id': '1', 'lease_expiration_ts': self.future_ts, } self.expect_error('lease', req, 'CANNOT_LEASE_BUILD')
def setUp(self): super(CreateTaskTest, self).setUp() self.patch('components.net.json_request_async', autospec=True) self.patch('components.auth.delegate_async', return_value=future('blah')) self.build_token = 'beeff00d' self.patch('tokens.generate_build_token', autospec=True, return_value='deadbeef') self.task_def = { 'is_task_def': True, 'task_slices': [{ 'properties': {}, }] } self.patch('swarming.prepare_task_def', autospec=True, return_value=self.task_def) self.build = test_util.build(id=1, created_by='user:[email protected]') self.build.swarming_task_key = None with self.build.mutate_infra() as infra: infra.swarming.task_id = '' self.build.put()
def test_put_with_generic_buildset(self, add_async): tags = [ dict(key='buildset', value='x'), dict(key='t', value='0'), ] build = test_util.build(id=1, tags=tags) add_async.return_value = future(build) req = { 'client_operation_id': '42', 'bucket': 'luci.chromium.try', 'tags': ['buildset:x', 't:0'], 'parameters_json': json.dumps({api_common.BUILDER_PARAMETER: 'linux'}), } resp = self.call_api('put', req).json_body add_async.assert_called_once_with( creation.BuildRequest( schedule_build_request=rpc_pb2.ScheduleBuildRequest( builder=dict( project='chromium', bucket='try', builder='linux', ), tags=tags, request_id='42', properties=dict(), ), parameters={}, ) ) self.assertEqual(resp['build']['id'], '1') self.assertIn('buildset:x', resp['build']['tags']) self.assertIn('t:0', resp['build']['tags'])
def test_cancel(self, cancel_async): cancel_async.return_value = future( test_util.build(id=54, status=common_pb2.CANCELED), ) req = rpc_pb2.CancelBuildRequest(id=54, summary_markdown='unnecesary') res = self.call(self.api.CancelBuild, req) self.assertEqual(res.id, 54) self.assertEqual(res.status, common_pb2.CANCELED) cancel_async.assert_called_once_with(54, summary_markdown='unnecesary')
def test_expired_build_to_message(self): yesterday = utils.utcnow() - datetime.timedelta(days=1) yesterday_timestamp = utils.datetime_to_timestamp(yesterday) build = test_util.build() build.lease_key = 1 build.lease_expiration_date = yesterday msg = api_common.build_to_message(build, None) self.assertEqual(msg.lease_expiration_ts, yesterday_timestamp)
def test_trimming_include(self, get_async): get_async.return_value = future( test_util.build(input=dict( properties=bbutil.dict_to_struct({'a': 'b'}))), ) req = rpc_pb2.GetBuildRequest(id=1, fields=dict(paths=['input.properties'])) res = self.call(self.api.GetBuild, req) self.assertEqual(res.input.properties.items(), [('a', 'b')])
def test_invalid_token(self): test_util.build(id=123).put() self.validate_build_token.side_effect = auth.InvalidTokenError build = build_pb2.Build( id=123, status=common_pb2.STARTED, ) req, ctx = self._mk_update_req(build) self.call( self.api.UpdateBuild, req, ctx=ctx, expected_code=prpc.StatusCode.UNAUTHENTICATED, )
def test_retry_forbidden(self): config.put_bucket( 'chromium', 'a' * 40, test_util.parse_bucket_cfg(''' name: "readonly" acls { role: READER identity: "anonymous:anonymous" } '''), ) test_util.build(id=1, builder=dict(project='chromium', bucket='readonly')).put() self.call_api('retry', {'id': '1'}, status=403)
def test_sync_build_async_no_task(self, load_task_result_async): load_task_result_async.return_value = future(None) build = test_util.build() build.put() swarming.CronUpdateBuilds().update_build_async(build).get_result() build = build.key.get() self.assertEqual(build.proto.status, common_pb2.INFRA_FAILURE) self.assertTrue(build.proto.summary_markdown)
def test_post(self, load_task_result_async): build = test_util.build(id=1) build.put() self.mock_request({ 'build_id': 1L, 'created_ts': 1448841600000000, 'swarming_hostname': 'swarming.example.com', })
def test_put_with_gerrit_change(self, add_async): buildset = 'patch/gerrit/gerrit.example.com/1234/5' buildset_tag = 'buildset:' + buildset props = {'patch_project': 'repo'} expected_sbr = rpc_pb2.ScheduleBuildRequest( builder=dict( project='chromium', bucket='try', builder='linux', ), gerrit_changes=[ dict( host='gerrit.example.com', project='repo', change=1234, patchset=5, ) ], tags=[dict(key='t', value='0')], request_id='42', properties=bbutil.dict_to_struct(props), ) expected_request = creation.BuildRequest( schedule_build_request=expected_sbr, parameters={}, ) build = test_util.build( id=1, input=dict( gerrit_changes=expected_sbr.gerrit_changes, properties=expected_sbr.properties, ), tags=expected_sbr.tags, ) build.tags.append(buildset_tag) build.tags.sort() add_async.return_value = future(build) req = { 'client_operation_id': '42', 'bucket': 'luci.chromium.try', 'tags': [buildset_tag, 't:0'], 'parameters_json': json.dumps({ api_common.BUILDER_PARAMETER: 'linux', api_common.PROPERTIES_PARAMETER: props, }), } resp = self.call_api('put', req).json_body add_async.assert_called_once_with(expected_request) self.assertEqual(resp['build']['id'], '1') self.assertIn(buildset_tag, resp['build']['tags']) self.assertIn('t:0', resp['build']['tags'])