def test_get_buckets_async(self): config.Bucket( id='master.tryserver.chromium.linux', project_id='chromium', revision='deadbeef', config_content=MASTER_TRYSERVER_CHROMIUM_LINUX_CONFIG_TEXT).put() config.Bucket( id='master.tryserver.chromium.win', project_id='chromium', revision='deadbeef', config_content=MASTER_TRYSERVER_CHROMIUM_WIN_CONFIG_TEXT).put() actual = config.get_buckets_async().get_result() expected = [ project_config_pb2.Bucket( name='master.tryserver.chromium.linux', acls=[ project_config_pb2.Acl( role=project_config_pb2.Acl.READER, group='all'), project_config_pb2.Acl( role=project_config_pb2.Acl.SCHEDULER, group='tryjob-access'), ]), project_config_pb2.Bucket( name='master.tryserver.chromium.win', acls=[ project_config_pb2.Acl( role=project_config_pb2.Acl.READER, group='all'), project_config_pb2.Acl( role=project_config_pb2.Acl.SCHEDULER, group='tryjob-access'), ]), ] self.assertEqual(actual, expected)
def get_available_buckets(): """Returns buckets available to the current identity. Results are memcached for 10 minutes per identity. Returns: Set of bucket names or None if all buckets are available. """ if auth.is_admin(): return None identity = auth.get_current_identity().to_bytes() cache_key = 'available_buckets/%s' % identity available_buckets = memcache.get(cache_key) if available_buckets is not None: return available_buckets logging.info('Computing a list of available buckets for %s' % identity) group_buckets_map = collections.defaultdict(set) available_buckets = set() all_buckets = config.get_buckets_async().get_result() for bucket in all_buckets: for rule in bucket.acls: if rule.identity == identity: available_buckets.add(bucket.name) if rule.group: group_buckets_map[rule.group].add(bucket.name) for group, buckets in group_buckets_map.iteritems(): if available_buckets.issuperset(buckets): continue if auth.is_group_member(group): available_buckets.update(buckets) # Cache for 10 min memcache.set(cache_key, available_buckets, 10 * 60) return available_buckets
def get_available_buckets(): """Returns buckets available to the current identity. Results are memcached for 10 minutes per identity. Returns: Set of bucket names or None if all buckets are available. """ if auth.is_admin(): return None identity = auth.get_current_identity().to_bytes() cache_key = 'available_buckets/%s' % identity available_buckets = memcache.get(cache_key) if available_buckets is not None: return available_buckets logging.info( 'Computing a list of available buckets for %s' % identity) group_buckets_map = collections.defaultdict(set) available_buckets = set() all_buckets = config.get_buckets_async().get_result() for bucket in all_buckets: for rule in bucket.acls: if rule.identity == identity: available_buckets.add(bucket.name) if rule.group: group_buckets_map[rule.group].add(bucket.name) for group, buckets in group_buckets_map.iteritems(): if available_buckets.issuperset(buckets): continue if auth.is_group_member(group): available_buckets.update(buckets) # Cache for 10 min memcache.set(cache_key, available_buckets, 10 * 60) return available_buckets
def impl(): if auth.is_admin(): raise ndb.Return(None) identity = auth.get_current_identity().to_bytes() cache_key = 'accessible_buckets_v2/%s' % identity ctx = ndb.get_context() available_buckets = yield ctx.memcache_get(cache_key) if available_buckets is not None: raise ndb.Return(available_buckets) logging.info('Computing a list of available buckets for %s' % identity) group_buckets_map = collections.defaultdict(set) available_buckets = set() all_buckets = yield config.get_buckets_async() for bucket_id, cfg in all_buckets.iteritems(): for rule in cfg.acls: if rule.identity == identity: available_buckets.add(bucket_id) elif rule.group: # pragma: no branch group_buckets_map[rule.group].add(bucket_id) for group, buckets in group_buckets_map.iteritems(): if available_buckets.issuperset(buckets): continue if auth.is_group_member(group): available_buckets.update(buckets) # Cache for 10 min yield ctx.memcache_set(cache_key, available_buckets, 10 * 60) raise ndb.Return(available_buckets)
def test_get_buckets_async_with_bucket_ids(self): config.put_bucket('chromium', 'deadbeef', LUCI_CHROMIUM_TRY) config.put_bucket('chromium', 'deadbeef', MASTER_TRYSERVER_CHROMIUM_WIN) bid = 'chromium/try' actual = config.get_buckets_async([bid]).get_result() expected = {'chromium/try': short_bucket_cfg(LUCI_CHROMIUM_TRY)} self.assertEqual(actual, expected)
def test_get_buckets_async(self): config.Bucket( id='master.tryserver.chromium.linux', project_id='chromium', revision='deadbeef', config_content=MASTER_TRYSERVER_CHROMIUM_LINUX_CONFIG_TEXT).put() config.Bucket( id='master.tryserver.chromium.win', project_id='chromium', revision='deadbeef', config_content=MASTER_TRYSERVER_CHROMIUM_WIN_CONFIG_TEXT).put() actual = config.get_buckets_async().get_result() expected = [ project_config_pb2.Bucket( name='master.tryserver.chromium.linux', acls=[ project_config_pb2.Acl(role=project_config_pb2.Acl.READER, group='all'), project_config_pb2.Acl( role=project_config_pb2.Acl.SCHEDULER, group='tryjob-access'), ]), project_config_pb2.Bucket( name='master.tryserver.chromium.win', acls=[ project_config_pb2.Acl(role=project_config_pb2.Acl.READER, group='all'), project_config_pb2.Acl( role=project_config_pb2.Acl.SCHEDULER, group='tryjob-access'), ]), ] self.assertEqual(actual, expected)
def get_builders(self, _request): """Returns defined swarmbucket builders. Can be used by code review tool to discover builders. """ buckets = config.get_buckets_async().get_result() available = acl.get_available_buckets() if available is not None: # pragma: no branch available = set(available) buckets = [b for b in buckets if b.name in available] res = GetBuildersResponseMessage() for bucket in buckets: if not bucket.swarming.builders: continue res.buckets.append( BucketMessage( name=bucket.name, builders=[ BuilderMessage( name=builder.name, category=builder.category, ) for builder in bucket.swarming.builders ], )) return res
def test_get_buckets_async(self): config.put_bucket('chromium', 'deadbeef', MASTER_TRYSERVER_CHROMIUM_LINUX) config.put_bucket('chromium', 'deadbeef', LUCI_CHROMIUM_TRY) config.put_bucket('dart', 'deadbeef', LUCI_DART_TRY) actual = config.get_buckets_async().get_result() expected = { 'chromium/master.tryserver.chromium.linux': MASTER_TRYSERVER_CHROMIUM_LINUX, 'chromium/try': short_bucket_cfg(LUCI_CHROMIUM_TRY), 'dart/try': short_bucket_cfg(LUCI_DART_TRY), } self.assertEqual(actual, expected)
def update_global_metrics(): """Updates the metrics in GLOBAL_METRICS.""" start = utils.utcnow() builder_ids = set() # {(bucket_id, builder)} for key in model.Builder.query().iter(keys_only=True): project_id, bucket, builder = key.id().split(':', 2) bucket_id = ( # TODO(crbug.com/851036): remove parse_luci_bucket call # once we don't have Builder entities with legacy bucket names. api_common.parse_luci_bucket(bucket) or config.format_bucket_id(project_id, bucket)) builder_ids.add((bucket_id, builder)) all_luci_bucket_ids = { bid for bid, config in config.get_buckets_async().get_result().iteritems() if config and config.swarming.builders } # Collect a list of counting/latency queries. count_query_queue = [] latency_query_queue = [] # TODO(crbug.com/851036): join with the loop above and remove builder_ids set. for bucket_id, builder in builder_ids: legacy_bucket_name = api_common.legacy_bucket_name( bucket_id, bucket_id in all_luci_bucket_ids) latency_query_queue.extend([ (bucket_id, legacy_bucket_name, builder, True), (bucket_id, legacy_bucket_name, builder, False), ]) for status in (model.BuildStatus.SCHEDULED, model.BuildStatus.STARTED): for experimental in (False, True): count_query_queue.append((bucket_id, legacy_bucket_name, builder, status, experimental)) # Process counting/latency queries with _CONCURRENT_QUERY_LIMIT workers. @ndb.tasklet def worker(): while count_query_queue: item = count_query_queue.pop() yield set_build_count_metric_async(*item) while latency_query_queue: item = latency_query_queue.pop() yield set_build_latency(*item) for w in [worker() for _ in xrange(_CONCURRENT_QUERY_LIMIT)]: w.check_success() logging.info('global metric computation took %s', utils.utcnow() - start)
def send_all_metrics(): buf = metrics.Buffer() futures = [] for b in config.get_buckets_async().get_result(): futures.extend([ send_build_status_metric( buf, b.name, METRIC_PENDING_BUILDS, model.BuildStatus.SCHEDULED), send_build_status_metric( buf, b.name, METRIC_RUNNING_BUILDS, model.BuildStatus.STARTED), ]) ndb.Future.wait_all(futures) buf.flush() for f in futures: f.check_success()
def send_all_metrics(): buf = metrics.Buffer() futures = [] for b in config.get_buckets_async().get_result(): futures.extend([ send_build_status_metric(buf, b.name, METRIC_PENDING_BUILDS, model.BuildStatus.SCHEDULED), send_build_status_metric(buf, b.name, METRIC_RUNNING_BUILDS, model.BuildStatus.STARTED), ]) ndb.Future.wait_all(futures) buf.flush() for f in futures: f.check_success()
def get_builders(self, request): """Returns defined swarmbucket builders. Returns legacy bucket names, e.g. "luci.chromium.try", not "chromium/try". Can be used to discover builders. """ if len(request.bucket) > 100: raise endpoints.BadRequestException( 'Number of buckets cannot be greater than 100') if request.bucket: # Buckets were specified explicitly. bucket_ids = map(api_common.parse_luci_bucket, request.bucket) bucket_ids = [bid for bid in bucket_ids if bid] # Filter out inaccessible ones. bids_can = utils.async_apply(bucket_ids, user.can_access_bucket_async) bucket_ids = [bid for bid, can in bids_can if can] else: # Buckets were not specified explicitly. # Use the available ones. bucket_ids = user.get_accessible_buckets_async().get_result() # bucket_ids is None => all buckets are available. res = GetBuildersResponseMessage() buckets = config.get_buckets_async(bucket_ids).get_result() for bucket_id, cfg in buckets.iteritems(): if not cfg or not cfg.swarming.builders: continue def to_dims(b): return flatten_swarmingcfg.format_dimensions( swarmingcfg.read_dimensions(b)) res.buckets.append( BucketMessage( name=api_common.format_luci_bucket(bucket_id), builders=[ BuilderMessage(name=builder.name, category=builder.category, properties_json=json.dumps( flatten_swarmingcfg.read_properties( builder.recipe)), swarming_hostname=builder.swarming_host, swarming_dimensions=to_dims(builder)) for builder in cfg.swarming.builders ], swarming_hostname=cfg.swarming.hostname, )) return res
def update_global_metrics(): """Updates the metrics in GLOBAL_METRICS.""" futures = [] for b in config.get_buckets_async().get_result(): futures.extend([ set_build_status_metric(CURRENTLY_PENDING, b.name, model.BuildStatus.SCHEDULED), set_build_status_metric(CURRENTLY_RUNNING, b.name, model.BuildStatus.STARTED), set_build_latency(LEASE_LATENCY_SEC, b.name, True), set_build_latency(SCHEDULING_LATENCY_SEC, b.name, False), ]) for f in futures: f.check_success()
def test_get_buckets_async_with_bucket_ids_not_found(self): bid = 'chromium/try' actual = config.get_buckets_async([bid]).get_result() self.assertEqual(actual, {bid: None})
def add_many_async(build_requests): """Adds many builds in a batch. Does not check permissions. Assumes build_requests is valid. Returns: A list of (new_build, exception) tuples in the same order. Exactly one item of a tuple will be non-None. The exception can be errors.InvalidInputError. Raises: Any exception that datastore operations can raise. """ # When changing this code, make corresponding changes to # swarmbucket_api.SwarmbucketApi.get_task_def. now = utils.utcnow() identity = auth.get_current_identity() logging.info('%s is creating %d builds', identity.to_bytes(), len(build_requests)) settings = yield config.get_settings_async() # Fetch and index configs. bucket_ids = {br.bucket_id for br in build_requests} bucket_cfgs = yield config.get_buckets_async(bucket_ids) builder_cfgs = {} # {bucket_id: {builder_name: cfg}} for bucket_id, bucket_cfg in bucket_cfgs.iteritems(): builder_cfgs[bucket_id] = { b.name: b for b in bucket_cfg.swarming.builders } # Prepare NewBuild objects. new_builds = [] for r in build_requests: builder = r.schedule_build_request.builder.builder bucket_builder_cfgs = builder_cfgs[r.bucket_id] builder_cfg = bucket_builder_cfgs.get(builder) # Apply builder config overrides, if any. # Exists for backward compatibility, runs only in V1 code path. if builder_cfg and r.override_builder_cfg: # pragma: no cover builder_cfg = copy.deepcopy(builder_cfg) r.override_builder_cfg(builder_cfg) nb = NewBuild(r, builder_cfg) if bucket_builder_cfgs and not builder_cfg: nb.exception = errors.BuilderNotFoundError( 'builder "%s" not found in bucket "%s"' % (builder, r.bucket_id)) new_builds.append(nb) # Check memcache. yield [nb.check_cache_async() for nb in new_builds if not nb.final] # Create and put builds. to_create = [nb for nb in new_builds if not nb.final] if to_create: build_ids = model.create_build_ids(now, len(to_create)) builds = yield [ nb.request.create_build_async(build_id, settings, nb.builder_cfg, identity, now) for nb, build_id in zip(to_create, build_ids) ] for nb, build in zip(to_create, builds): nb.build = build yield _update_builders_async(to_create, now) yield _generate_build_numbers_async(to_create) yield search.update_tag_indexes_async([nb.build for nb in to_create]) yield [nb.put_and_cache_async() for nb in to_create] raise ndb.Return([nb.result() for nb in new_builds])