def _sync_build_and_swarming(build_id, generation): """Synchronizes build and Swarming. If the swarming task does not exist yet, creates it. Otherwise updates the build state to match swarming task state. Enqueues a new sync push task if the build did not end. """ bundle = model.BuildBundle.get(build_id, infra=True, input_properties=True) if not bundle: # pragma: no cover logging.warning('build not found') return build = bundle.build if build.is_ended: logging.info('build ended') return build.proto.infra.ParseFromString(bundle.infra.infra) build.proto.input.properties.ParseFromString( bundle.input_properties.properties) build.proto.infra.buildbucket.hostname = ( app_identity.get_default_version_hostname()) sw = build.proto.infra.swarming if not sw.task_id: _create_swarming_task(build) else: result = _load_task_result(sw.hostname, sw.task_id) if not result: logging.error('Task %s/%s referenced by build %s is not found', sw.hostname, sw.task_id, build.key.id()) _sync_build_with_task_result(build_id, result) # Enqueue a continuation task. next_gen = generation + 1 payload = { 'id': build.key.id(), 'generation': next_gen, } deadline = build.create_time + model.BUILD_TIMEOUT age_limit = deadline - utils.utcnow() continuation = { 'name': 'sync-task-%d-%d' % (build_id, next_gen), 'url': '/internal/task/swarming/sync-build/%s' % build.key.id(), 'payload': json.dumps(payload, sort_keys=True), 'retry_options': { 'task_age_limit': age_limit.total_seconds() }, 'countdown': 60, # Run the continuation task in 1m. } try: tq.enqueue_async(SYNC_QUEUE_NAME, [continuation], transactional=False).get_result() except (taskqueue.TaskAlreadyExistsError, taskqueue.TombstonedTaskError): # pragma: no cover # Previous attempt for this generation of the task might have already # created the next generation task, and in case of TombstonedTaskError this # task may be already executing or even finished. This is OK. pass
def txn_async(): if (yield b.key.get_async()): # pragma: no cover raise errors.Error('build number collision') futs = [b.put_async()] if sync_task: futs.append( tq.enqueue_async(swarming.SYNC_QUEUE_NAME, [sync_task])) yield futs
def enqueue_bq_export_async(build): """Enqueues a pull task to export a completed build to BigQuery.""" assert ndb.in_transaction() assert build assert build.is_ended task_def = { 'method': 'PULL', 'payload': { 'id': build.key.id() }, } return tq.enqueue_async('bq-export', [task_def])
def enqueue_notifications_async(build): assert ndb.in_transaction() assert build def mktask(mode): return dict( url='/internal/task/buildbucket/notify/%d' % build.key.id(), payload=dict(id=build.key.id(), mode=mode), retry_options=dict(task_age_limit=model.BUILD_TIMEOUT.total_seconds()), ) tasks = [mktask('global')] if build.pubsub_callback: # pragma: no branch tasks.append(mktask('callback')) return tq.enqueue_async('backend-default', tasks)
def enqueue_bq_export_async(build): """Enqueues a pull task to export a completed build to BigQuery.""" assert ndb.in_transaction() assert build assert build.is_ended return tq.enqueue_async( 'bq-export-experimental' if build.experimental else 'bq-export-prod', [{ 'method': 'PULL', 'payload': { 'id': build.key.id() }, }], )