def get_buildset_for(project, issue_id, patchset_id):
# See the convention
# https://chromium.googlesource.com/infra/infra/+/master/appengine/cr-buildbucket/doc/index.md#buildset-tag
hostname = common.get_preferred_domain(project, default_to_appid=False)
if not hostname:
logging.error(
'Preferred domain name for this app is not set. '
'See PREFERRED_DOMAIN_NAMES in settings.py: %r', hostname)
raise ndb.Return([])
return BUILDSET_TAG_FORMAT.format(
hostname=hostname,
issue=issue_id,
patch=patchset_id,
)
def get_buildset_for(project, issue_id, patchset_id):
# See the convention
# https://chromium.googlesource.com/infra/infra/+/master/appengine/cr-buildbucket/doc/index.md#buildset-tag
hostname = common.get_preferred_domain(project, default_to_appid=False)
if not hostname:
logging.error(
'Preferred domain name for this app is not set. '
'See PREFERRED_DOMAIN_NAMES in settings.py: %r', hostname)
raise ndb.Return([])
return BUILDSET_TAG_FORMAT.format(
hostname=hostname,
issue=issue_id,
patch=patchset_id,
)
def get_builds_for_patchset_async(project, issue_id, patchset_id):
"""Queries BuildBucket for builds associated with the patchset.
Requests for max 500 builds and does not check "next_cursor". Currently if
more than 100 builds are requested, only 100 are returned. Presumably there
will be no patchsets with >100 builds.
Returns:
A list of buildbucket build dicts.
"""
# See tag conventions http://cr-buildbucket.appspot.com/#docs/conventions .
hostname = common.get_preferred_domain(project, default_to_appid=False)
if not hostname:
logging.error(
'Preferred domain name for this app is not set. '
'See PREFERRED_DOMAIN_NAMES in settings.py: %r', hostname)
raise ndb.Return([])
buildset_tag = BUILDSET_TAG_FORMAT.format(
hostname=hostname,
issue=issue_id,
patch=patchset_id,
)
params = {
'max_builds': 500,
'tag': 'buildset:%s' % buildset_tag,
}
url = '%s/search' % BUILDBUCKET_API_ROOT
logging.info(
'Fetching builds for patchset %s/%s. Buildset: %s',
issue_id, patchset_id, buildset_tag)
try:
resp = yield net.json_request_async(
url, params=params,
scopes='https://www.googleapis.com/auth/userinfo.email')
except net.NotFoundError as ex:
logging.error(
'Buildbucket returned 404 unexpectedly. Body: %s', ex.response)
raise
if 'error' in resp:
bb_error = resp.get('error', {})
raise BuildBucketError(
'BuildBucket responded with error (reason %s): %s' % (
bb_error.get('reason', 'no-reason'),
bb_error.get('message', 'no-message')))
raise ndb.Return(resp.get('builds') or [])
def get_builds_for_patchset_async(project, issue_id, patchset_id):
"""Queries BuildBucket for builds associated with the patchset.
Requests for max 500 builds and does not check "next_cursor". Currently if
more than 100 builds are requested, only 100 are returned. Presumably there
will be no patchsets with >100 builds.
Returns:
A list of buildbucket build dicts.
"""
# See tag conventions http://cr-buildbucket.appspot.com/#docs/conventions .
hostname = common.get_preferred_domain(project, default_to_appid=False)
if not hostname:
logging.error(
'Preferred domain name for this app is not set. '
'See PREFERRED_DOMAIN_NAMES in settings.py: %r', hostname)
raise ndb.Return([])
buildset_tag = BUILDSET_TAG_FORMAT.format(
hostname=hostname,
issue=issue_id,
patch=patchset_id,
)
params = {
'max_builds': 500,
'tag': 'buildset:%s' % buildset_tag,
}
url = '%s/search' % BUILDBUCKET_API_ROOT
logging.info('Fetching builds for patchset %s/%s. Buildset: %s', issue_id,
patchset_id, buildset_tag)
try:
resp = yield net.json_request_async(
url,
params=params,
scopes='https://www.googleapis.com/auth/userinfo.email')
except net.NotFoundError as ex:
logging.error('Buildbucket returned 404 unexpectedly. Body: %s',
ex.response)
raise
if 'error' in resp:
bb_error = resp.get('error', {})
raise BuildBucketError(
'BuildBucket responded with error (reason %s): %s' % (bb_error.get(
'reason', 'no-reason'), bb_error.get('message', 'no-message')))
raise ndb.Return(resp.get('builds') or [])
def schedule(issue, patchset_id, builds):
"""Schedules builds on buildbucket.
|builds| is a list of (master, builder) tuples, where master must not have
'.master' prefix.
"""
account = models.Account.current_user_account
assert account, 'User is not logged in; cannot schedule builds.'
if not builds:
return
self_hostname = common.get_preferred_domain(issue.project)
req = {'builds':[]}
opid = uuid.uuid4()
for i, (master, builder) in enumerate(builds):
# Build definitions are similar to what CQ produces:
# https://chrome-internal.googlesource.com/infra/infra_internal/+/c3092da98975c7a3e083093f21f0f4130c66a51c/commit_queue/buildbucket_util.py#171
req['builds'].append({
'bucket': 'master.%s' % master,
'parameters_json': json.dumps({
'builder_name': builder,
'changes': [{
'author': {'email': issue.owner.email()},
'url': 'https://%s/%s/%s/' % (
self_hostname, issue.key.id(), patchset_id)
}],
'properties': {
'issue': issue.key.id(),
'master': master,
'patch_project': issue.project,
'patch_storage': 'rietveld',
'patchset': patchset_id,
'project': issue.project,
'rietveld': self_hostname,
},
}),
'tags': [
'builder:%s' % builder,
'buildset:%s' % get_buildset_for(
issue.project, issue.key.id(), patchset_id),
'master:%s' % master,
'user_agent:rietveld',
],
'client_operation_id': '%s:%s' % (opid, i),
})
logging.debug(
'Scheduling %d builds on behalf of %s', len(req['builds']), account.email)
res = rpc_async('PUT', 'builds/batch', payload=req).get_result()
for r in res['results']:
error = r.get('error')
if error:
logging.error('Build scheduling failed. Response: %r', res)
raise BuildBucketError('Could not schedule build(s): %r' % error)
actual_builds = [r['build'] for r in res['results']]
logging.info(
'Scheduled buildbucket builds: %r',
', '.join([str(b['id']) for b in actual_builds]))
return actual_builds
def schedule(issue, patchset_id, builds):
"""Schedules builds on buildbucket.
|builds| is a list of dicts with keys:
bucket: required, target buildbucket bucket name. May have "master." prefix.
builder: required.
revision
properties
"""
account = models.Account.current_user_account
assert account, 'User is not logged in; cannot schedule builds.'
if not builds:
return
self_hostname = common.get_preferred_domain(issue.project)
req = {'builds': []}
opid = uuid.uuid4()
for i, build in enumerate(builds):
assert 'bucket' in build, build
assert 'builder' in build, build
master_prefix = 'master.'
master_name = None # Buildbot master name without "master." prefix.
if build['bucket'].startswith(master_prefix):
master_name = build['bucket'][len(master_prefix):]
# Build definitions are similar to what CQ produces:
# https://chrome-internal.googlesource.com/infra/infra_internal/+/c3092da98975c7a3e083093f21f0f4130c66a51c/commit_queue/buildbucket_util.py#171
change = {
'author': {
'email': issue.owner.email()
},
'url':
'https://%s/%s/%s/' % (self_hostname, issue.key.id(), patchset_id)
}
if build.get('revision'):
change['revision'] = build.get('revision')
properties = build.get('properties') or {}
properties.update({
'issue': issue.key.id(),
'patch_project': issue.project,
'patch_storage': 'rietveld',
'patchset': patchset_id,
'rietveld': 'https://%s' % self_hostname,
})
if master_name:
properties['master'] = master_name
if 'presubmit' in build['builder'].lower():
properties['dry_run'] = 'true'
tags = [
'builder:%s' % build['builder'],
'buildset:%s' %
get_buildset_for(issue.project, issue.key.id(), patchset_id),
'user_agent:rietveld',
]
if master_name:
tags.append('master:%s' % master_name)
req['builds'].append({
'bucket':
build['bucket'],
'parameters_json':
json.dumps({
'builder_name': build['builder'],
'changes': [change],
'properties': properties,
}),
'tags':
tags,
'client_operation_id':
'%s:%s' % (opid, i),
})
logging.debug('Scheduling %d builds on behalf of %s', len(req['builds']),
account.email)
res = rpc_async('PUT', 'builds/batch', payload=req).get_result()
for r in res['results']:
error = r.get('error')
if error:
logging.error('Build scheduling failed. Response: %r', res)
raise BuildBucketError('Could not schedule build(s): %r' % error)
actual_builds = [r['build'] for r in res['results']]
logging.info('Scheduled buildbucket builds: %r',
', '.join([str(b['id']) for b in actual_builds]))
return actual_builds