def _sync_build_and_swarming(build_id, generation):
"""Synchronizes build and Swarming.
If the swarming task does not exist yet, creates it.
Otherwise updates the build state to match swarming task state.
Enqueues a new sync push task if the build did not end.
"""
bundle = model.BuildBundle.get(build_id, infra=True, input_properties=True)
if not bundle: # pragma: no cover
logging.warning('build not found')
return
build = bundle.build
if build.is_ended:
logging.info('build ended')
return
build.proto.infra.ParseFromString(bundle.infra.infra)
build.proto.input.properties.ParseFromString(
bundle.input_properties.properties)
build.proto.infra.buildbucket.hostname = (
app_identity.get_default_version_hostname())
sw = build.proto.infra.swarming
if not sw.task_id:
_create_swarming_task(build)
else:
result = _load_task_result(sw.hostname, sw.task_id)
if not result:
logging.error('Task %s/%s referenced by build %s is not found',
sw.hostname, sw.task_id, build.key.id())
_sync_build_with_task_result(build_id, result)
# Enqueue a continuation task.
next_gen = generation + 1
payload = {
'id': build.key.id(),
'generation': next_gen,
}
deadline = build.create_time + model.BUILD_TIMEOUT
age_limit = deadline - utils.utcnow()
continuation = {
'name': 'sync-task-%d-%d' % (build_id, next_gen),
'url': '/internal/task/swarming/sync-build/%s' % build.key.id(),
'payload': json.dumps(payload, sort_keys=True),
'retry_options': {
'task_age_limit': age_limit.total_seconds()
},
'countdown': 60, # Run the continuation task in 1m.
}
try:
tq.enqueue_async(SYNC_QUEUE_NAME, [continuation],
transactional=False).get_result()
except (taskqueue.TaskAlreadyExistsError,
taskqueue.TombstonedTaskError): # pragma: no cover
# Previous attempt for this generation of the task might have already
# created the next generation task, and in case of TombstonedTaskError this
# task may be already executing or even finished. This is OK.
pass
def txn_async():
if (yield b.key.get_async()): # pragma: no cover
raise errors.Error('build number collision')
futs = [b.put_async()]
if sync_task:
futs.append(
tq.enqueue_async(swarming.SYNC_QUEUE_NAME, [sync_task]))
yield futs
def enqueue_bq_export_async(build):
"""Enqueues a pull task to export a completed build to BigQuery."""
assert ndb.in_transaction()
assert build
assert build.is_ended
task_def = {
'method': 'PULL',
'payload': {
'id': build.key.id()
},
}
return tq.enqueue_async('bq-export', [task_def])
def enqueue_notifications_async(build):
assert ndb.in_transaction()
assert build
def mktask(mode):
return dict(
url='/internal/task/buildbucket/notify/%d' % build.key.id(),
payload=dict(id=build.key.id(), mode=mode),
retry_options=dict(task_age_limit=model.BUILD_TIMEOUT.total_seconds()),
)
tasks = [mktask('global')]
if build.pubsub_callback: # pragma: no branch
tasks.append(mktask('callback'))
return tq.enqueue_async('backend-default', tasks)
def enqueue_bq_export_async(build):
"""Enqueues a pull task to export a completed build to BigQuery."""
assert ndb.in_transaction()
assert build
assert build.is_ended
return tq.enqueue_async(
'bq-export-experimental' if build.experimental else 'bq-export-prod',
[{
'method': 'PULL',
'payload': {
'id': build.key.id()
},
}],
)