def test_many_with_request_id(self):
req1 = self.build_request(
dict(
tags=[dict(key='buildset', value='a')],
request_id='0',
), )
req2 = self.build_request(dict(tags=[dict(key='buildset', value='a')]))
creation.add_async(req1).get_result()
creation.add_many_async([req1, req2]).get_result()
# Build for req1 must be added only once.
idx = search.TagIndex.get_by_id('buildset:a')
self.assertEqual(len(idx.entries), 2)
self.assertEqual(idx.entries[0].bucket_id, 'chromium/try')
def test_non_existing_builder(self):
builder_id = build_pb2.BuilderID(
project='chromium',
bucket='try',
builder='non-existing',
)
req = self.build_request(dict(builder=builder_id))
(_, ex), = creation.add_many_async([req]).get_result()
self.assertIsInstance(ex, errors.BuilderNotFoundError)
def test_update_builders(self):
recently = self.now - datetime.timedelta(minutes=1)
while_ago = self.now - datetime.timedelta(minutes=61)
ndb.put_multi([
model.Builder(id='chromium:try:linux', last_scheduled=recently),
model.Builder(id='chromium:try:mac', last_scheduled=while_ago),
])
creation.add_many_async([
self.build_request(dict(builder=dict(builder='linux'))),
self.build_request(dict(builder=dict(builder='mac'))),
self.build_request(dict(builder=dict(builder='win'))),
]).get_result()
builders = model.Builder.query().fetch()
self.assertEqual(len(builders), 3)
self.assertEqual(builders[0].key.id(), 'chromium:try:linux')
self.assertEqual(builders[0].last_scheduled, recently)
self.assertEqual(builders[1].key.id(), 'chromium:try:mac')
self.assertEqual(builders[1].last_scheduled, self.now)
self.assertEqual(builders[2].key.id(), 'chromium:try:win')
self.assertEqual(builders[2].last_scheduled, self.now)
def test_create_sync_task(self):
expected_ex1 = errors.InvalidInputError()
def create_sync_task(build, *_args, **_kwargs):
if 'buildset:a' in build.tags:
raise expected_ex1
self.create_sync_task.side_effect = create_sync_task
((b1, ex1), (b2, ex2)) = creation.add_many_async([
self.build_request(dict(tags=[dict(key='buildset', value='a')])),
self.build_request(dict(tags=[dict(key='buildset', value='b')])),
]).get_result()
self.assertEqual(ex1, expected_ex1)
self.assertIsNone(b1)
self.assertIsNone(ex2)
self.assertIsNotNone(b2)
def put_batch(self, request):
"""Creates builds."""
# Convert buckets to v2.
buckets = sorted({b.bucket for b in request.builds})
bucket_ids = dict(zip(buckets, convert_bucket_list(buckets)))
for b in request.builds:
b.bucket = bucket_ids[b.bucket]
# Check permissions.
check_scheduling_permissions(bucket_ids.itervalues())
# Prepare response.
res = self.PutBatchResponseMessage()
res.results = [
res.OneResult(client_operation_id=b.client_operation_id)
for b in request.builds
]
# Try to convert each PutRequest to BuildRequest.
build_reqs = [] # [(index, creation.BuildRequest])
for i, b in enumerate(request.builds):
try:
build_reqs.append((i, put_request_message_to_build_request(b)))
except errors.Error as ex:
res.results[i].error = exception_to_error_message(ex)
# Try to create builds.
results = creation.add_many_async([br for i, br in build_reqs
]).get_result()
# Convert results to messages.
for (i, _), (build, ex) in zip(build_reqs, results):
one_res = res.results[i]
if build:
one_res.build = build_to_message(build, include_lease_key=True)
elif isinstance(ex, errors.Error):
one_res.error = exception_to_error_message(ex)
else:
raise ex
return res
def test_many(self):
results = creation.add_many_async([
self.build_request(dict(tags=[dict(key='buildset', value='a')])),
self.build_request(dict(tags=[dict(key='buildset', value='a')])),
]).get_result()
self.assertEqual(len(results), 2)
self.assertIsNotNone(results[0][0])
self.assertIsNone(results[0][1])
self.assertIsNotNone(results[1][0])
self.assertIsNone(results[1][1])
self.assertEqual(
results,
sorted(results, key=lambda (b, _): b.key.id(), reverse=True))
results.reverse()
index = search.TagIndex.get_by_id('buildset:a')
self.assertIsNotNone(index)
self.assertEqual(len(index.entries), 2)
self.assertEqual(index.entries[0].build_id, results[1][0].key.id())
self.assertEqual(index.entries[0].bucket_id, results[1][0].bucket_id)
self.assertEqual(index.entries[1].build_id, results[0][0].key.id())
self.assertEqual(index.entries[1].bucket_id, results[0][0].bucket_id)
def schedule_build_multi(batch):
"""Schedules multiple builds.
Args:
batch: list of _ReqRes where
request is rpc_pb2.ScheduleBuildRequest and
response is rpc_pb2.BatchResponse.Response.
Response objects will be mutated.
"""
# Validate requests.
valid_items = []
for rr in batch:
try:
validation.validate_schedule_build_request(rr.request)
except validation.Error as ex:
rr.response.error.code = prpc.StatusCode.INVALID_ARGUMENT.value
rr.response.error.message = ex.message
continue
# Parse the field mask.
# Normally it is done by rpc_impl_async.
mask = None
if rr.request.HasField('fields'):
try:
mask = protoutil.Mask.from_field_mask(
rr.request.fields, build_pb2.Build.DESCRIPTOR)
except ValueError as ex:
rr.response.error.code = prpc.StatusCode.INVALID_ARGUMENT.value
rr.response.error.message = 'invalid fields: %s' % ex.message
continue
valid_items.append(_ScheduleItem(rr.request, rr.response, mask))
# Check permissions.
def get_bucket_id(req):
return config.format_bucket_id(req.builder.project, req.builder.bucket)
bucket_ids = {get_bucket_id(x.request) for x in valid_items}
can_add = dict(utils.async_apply(bucket_ids, user.can_add_build_async))
identity_str = auth.get_current_identity().to_bytes()
to_schedule = []
for x in valid_items:
bid = get_bucket_id(x.request)
if can_add[bid]:
to_schedule.append(x)
else:
x.response.error.code = prpc.StatusCode.PERMISSION_DENIED.value
x.response.error.message = (
'%s cannot schedule builds in bucket %s' % (identity_str, bid))
# Schedule builds.
if not to_schedule: # pragma: no cover
return
build_requests = [
creation.BuildRequest(schedule_build_request=x.request)
for x in to_schedule
]
results = creation.add_many_async(build_requests).get_result()
futs = []
for x, (build, ex) in zip(to_schedule, results):
res = x.response
err = res.error
if isinstance(ex, errors.Error):
err.code = ex.code.value
err.message = ex.message
elif isinstance(ex, auth.AuthorizationError):
err.code = prpc.StatusCode.PERMISSION_DENIED.value
err.message = ex.message
elif ex:
err.code = prpc.StatusCode.INTERNAL.value
err.message = ex.message
else:
futs.append(build_to_proto_async(build, res.schedule_build,
x.mask))
for f in futs:
f.get_result()