def test_get_buckets_async(self):
config.Bucket(
id='master.tryserver.chromium.linux',
project_id='chromium',
revision='deadbeef',
config_content=MASTER_TRYSERVER_CHROMIUM_LINUX_CONFIG_TEXT).put()
config.Bucket(
id='master.tryserver.chromium.win',
project_id='chromium',
revision='deadbeef',
config_content=MASTER_TRYSERVER_CHROMIUM_WIN_CONFIG_TEXT).put()
actual = config.get_buckets_async().get_result()
expected = [
project_config_pb2.Bucket(
name='master.tryserver.chromium.linux',
acls=[
project_config_pb2.Acl(
role=project_config_pb2.Acl.READER, group='all'),
project_config_pb2.Acl(
role=project_config_pb2.Acl.SCHEDULER, group='tryjob-access'),
]),
project_config_pb2.Bucket(
name='master.tryserver.chromium.win',
acls=[
project_config_pb2.Acl(
role=project_config_pb2.Acl.READER, group='all'),
project_config_pb2.Acl(
role=project_config_pb2.Acl.SCHEDULER, group='tryjob-access'),
]),
]
self.assertEqual(actual, expected)
def get_available_buckets():
"""Returns buckets available to the current identity.
Results are memcached for 10 minutes per identity.
Returns:
Set of bucket names or None if all buckets are available.
"""
if auth.is_admin():
return None
identity = auth.get_current_identity().to_bytes()
cache_key = 'available_buckets/%s' % identity
available_buckets = memcache.get(cache_key)
if available_buckets is not None:
return available_buckets
logging.info('Computing a list of available buckets for %s' % identity)
group_buckets_map = collections.defaultdict(set)
available_buckets = set()
all_buckets = config.get_buckets_async().get_result()
for bucket in all_buckets:
for rule in bucket.acls:
if rule.identity == identity:
available_buckets.add(bucket.name)
if rule.group:
group_buckets_map[rule.group].add(bucket.name)
for group, buckets in group_buckets_map.iteritems():
if available_buckets.issuperset(buckets):
continue
if auth.is_group_member(group):
available_buckets.update(buckets)
# Cache for 10 min
memcache.set(cache_key, available_buckets, 10 * 60)
return available_buckets
def get_available_buckets():
"""Returns buckets available to the current identity.
Results are memcached for 10 minutes per identity.
Returns:
Set of bucket names or None if all buckets are available.
"""
if auth.is_admin():
return None
identity = auth.get_current_identity().to_bytes()
cache_key = 'available_buckets/%s' % identity
available_buckets = memcache.get(cache_key)
if available_buckets is not None:
return available_buckets
logging.info(
'Computing a list of available buckets for %s' % identity)
group_buckets_map = collections.defaultdict(set)
available_buckets = set()
all_buckets = config.get_buckets_async().get_result()
for bucket in all_buckets:
for rule in bucket.acls:
if rule.identity == identity:
available_buckets.add(bucket.name)
if rule.group:
group_buckets_map[rule.group].add(bucket.name)
for group, buckets in group_buckets_map.iteritems():
if available_buckets.issuperset(buckets):
continue
if auth.is_group_member(group):
available_buckets.update(buckets)
# Cache for 10 min
memcache.set(cache_key, available_buckets, 10 * 60)
return available_buckets
def impl():
if auth.is_admin():
raise ndb.Return(None)
identity = auth.get_current_identity().to_bytes()
cache_key = 'accessible_buckets_v2/%s' % identity
ctx = ndb.get_context()
available_buckets = yield ctx.memcache_get(cache_key)
if available_buckets is not None:
raise ndb.Return(available_buckets)
logging.info('Computing a list of available buckets for %s' % identity)
group_buckets_map = collections.defaultdict(set)
available_buckets = set()
all_buckets = yield config.get_buckets_async()
for bucket_id, cfg in all_buckets.iteritems():
for rule in cfg.acls:
if rule.identity == identity:
available_buckets.add(bucket_id)
elif rule.group: # pragma: no branch
group_buckets_map[rule.group].add(bucket_id)
for group, buckets in group_buckets_map.iteritems():
if available_buckets.issuperset(buckets):
continue
if auth.is_group_member(group):
available_buckets.update(buckets)
# Cache for 10 min
yield ctx.memcache_set(cache_key, available_buckets, 10 * 60)
raise ndb.Return(available_buckets)
def test_get_buckets_async_with_bucket_ids(self):
config.put_bucket('chromium', 'deadbeef', LUCI_CHROMIUM_TRY)
config.put_bucket('chromium', 'deadbeef', MASTER_TRYSERVER_CHROMIUM_WIN)
bid = 'chromium/try'
actual = config.get_buckets_async([bid]).get_result()
expected = {'chromium/try': short_bucket_cfg(LUCI_CHROMIUM_TRY)}
self.assertEqual(actual, expected)
def test_get_buckets_async(self):
config.Bucket(
id='master.tryserver.chromium.linux',
project_id='chromium',
revision='deadbeef',
config_content=MASTER_TRYSERVER_CHROMIUM_LINUX_CONFIG_TEXT).put()
config.Bucket(
id='master.tryserver.chromium.win',
project_id='chromium',
revision='deadbeef',
config_content=MASTER_TRYSERVER_CHROMIUM_WIN_CONFIG_TEXT).put()
actual = config.get_buckets_async().get_result()
expected = [
project_config_pb2.Bucket(
name='master.tryserver.chromium.linux',
acls=[
project_config_pb2.Acl(role=project_config_pb2.Acl.READER,
group='all'),
project_config_pb2.Acl(
role=project_config_pb2.Acl.SCHEDULER,
group='tryjob-access'),
]),
project_config_pb2.Bucket(
name='master.tryserver.chromium.win',
acls=[
project_config_pb2.Acl(role=project_config_pb2.Acl.READER,
group='all'),
project_config_pb2.Acl(
role=project_config_pb2.Acl.SCHEDULER,
group='tryjob-access'),
]),
]
self.assertEqual(actual, expected)
def get_builders(self, _request):
"""Returns defined swarmbucket builders.
Can be used by code review tool to discover builders.
"""
buckets = config.get_buckets_async().get_result()
available = acl.get_available_buckets()
if available is not None: # pragma: no branch
available = set(available)
buckets = [b for b in buckets if b.name in available]
res = GetBuildersResponseMessage()
for bucket in buckets:
if not bucket.swarming.builders:
continue
res.buckets.append(
BucketMessage(
name=bucket.name,
builders=[
BuilderMessage(
name=builder.name,
category=builder.category,
) for builder in bucket.swarming.builders
],
))
return res
def test_get_buckets_async(self):
config.put_bucket('chromium', 'deadbeef',
MASTER_TRYSERVER_CHROMIUM_LINUX)
config.put_bucket('chromium', 'deadbeef', LUCI_CHROMIUM_TRY)
config.put_bucket('dart', 'deadbeef', LUCI_DART_TRY)
actual = config.get_buckets_async().get_result()
expected = {
'chromium/master.tryserver.chromium.linux':
MASTER_TRYSERVER_CHROMIUM_LINUX,
'chromium/try': short_bucket_cfg(LUCI_CHROMIUM_TRY),
'dart/try': short_bucket_cfg(LUCI_DART_TRY),
}
self.assertEqual(actual, expected)
def update_global_metrics():
"""Updates the metrics in GLOBAL_METRICS."""
start = utils.utcnow()
builder_ids = set() # {(bucket_id, builder)}
for key in model.Builder.query().iter(keys_only=True):
project_id, bucket, builder = key.id().split(':', 2)
bucket_id = (
# TODO(crbug.com/851036): remove parse_luci_bucket call
# once we don't have Builder entities with legacy bucket names.
api_common.parse_luci_bucket(bucket)
or config.format_bucket_id(project_id, bucket))
builder_ids.add((bucket_id, builder))
all_luci_bucket_ids = {
bid
for bid, config in config.get_buckets_async().get_result().iteritems()
if config and config.swarming.builders
}
# Collect a list of counting/latency queries.
count_query_queue = []
latency_query_queue = []
# TODO(crbug.com/851036): join with the loop above and remove builder_ids set.
for bucket_id, builder in builder_ids:
legacy_bucket_name = api_common.legacy_bucket_name(
bucket_id, bucket_id in all_luci_bucket_ids)
latency_query_queue.extend([
(bucket_id, legacy_bucket_name, builder, True),
(bucket_id, legacy_bucket_name, builder, False),
])
for status in (model.BuildStatus.SCHEDULED, model.BuildStatus.STARTED):
for experimental in (False, True):
count_query_queue.append((bucket_id, legacy_bucket_name,
builder, status, experimental))
# Process counting/latency queries with _CONCURRENT_QUERY_LIMIT workers.
@ndb.tasklet
def worker():
while count_query_queue:
item = count_query_queue.pop()
yield set_build_count_metric_async(*item)
while latency_query_queue:
item = latency_query_queue.pop()
yield set_build_latency(*item)
for w in [worker() for _ in xrange(_CONCURRENT_QUERY_LIMIT)]:
w.check_success()
logging.info('global metric computation took %s', utils.utcnow() - start)
def send_all_metrics():
buf = metrics.Buffer()
futures = []
for b in config.get_buckets_async().get_result():
futures.extend([
send_build_status_metric(
buf, b.name, METRIC_PENDING_BUILDS, model.BuildStatus.SCHEDULED),
send_build_status_metric(
buf, b.name, METRIC_RUNNING_BUILDS, model.BuildStatus.STARTED),
])
ndb.Future.wait_all(futures)
buf.flush()
for f in futures:
f.check_success()
def send_all_metrics():
buf = metrics.Buffer()
futures = []
for b in config.get_buckets_async().get_result():
futures.extend([
send_build_status_metric(buf, b.name, METRIC_PENDING_BUILDS,
model.BuildStatus.SCHEDULED),
send_build_status_metric(buf, b.name, METRIC_RUNNING_BUILDS,
model.BuildStatus.STARTED),
])
ndb.Future.wait_all(futures)
buf.flush()
for f in futures:
f.check_success()
def get_builders(self, request):
"""Returns defined swarmbucket builders.
Returns legacy bucket names, e.g. "luci.chromium.try", not "chromium/try".
Can be used to discover builders.
"""
if len(request.bucket) > 100:
raise endpoints.BadRequestException(
'Number of buckets cannot be greater than 100')
if request.bucket:
# Buckets were specified explicitly.
bucket_ids = map(api_common.parse_luci_bucket, request.bucket)
bucket_ids = [bid for bid in bucket_ids if bid]
# Filter out inaccessible ones.
bids_can = utils.async_apply(bucket_ids,
user.can_access_bucket_async)
bucket_ids = [bid for bid, can in bids_can if can]
else:
# Buckets were not specified explicitly.
# Use the available ones.
bucket_ids = user.get_accessible_buckets_async().get_result()
# bucket_ids is None => all buckets are available.
res = GetBuildersResponseMessage()
buckets = config.get_buckets_async(bucket_ids).get_result()
for bucket_id, cfg in buckets.iteritems():
if not cfg or not cfg.swarming.builders:
continue
def to_dims(b):
return flatten_swarmingcfg.format_dimensions(
swarmingcfg.read_dimensions(b))
res.buckets.append(
BucketMessage(
name=api_common.format_luci_bucket(bucket_id),
builders=[
BuilderMessage(name=builder.name,
category=builder.category,
properties_json=json.dumps(
flatten_swarmingcfg.read_properties(
builder.recipe)),
swarming_hostname=builder.swarming_host,
swarming_dimensions=to_dims(builder))
for builder in cfg.swarming.builders
],
swarming_hostname=cfg.swarming.hostname,
))
return res
def update_global_metrics():
"""Updates the metrics in GLOBAL_METRICS."""
futures = []
for b in config.get_buckets_async().get_result():
futures.extend([
set_build_status_metric(CURRENTLY_PENDING, b.name,
model.BuildStatus.SCHEDULED),
set_build_status_metric(CURRENTLY_RUNNING, b.name,
model.BuildStatus.STARTED),
set_build_latency(LEASE_LATENCY_SEC, b.name, True),
set_build_latency(SCHEDULING_LATENCY_SEC, b.name, False),
])
for f in futures:
f.check_success()
def test_get_buckets_async_with_bucket_ids_not_found(self):
bid = 'chromium/try'
actual = config.get_buckets_async([bid]).get_result()
self.assertEqual(actual, {bid: None})
def add_many_async(build_requests):
"""Adds many builds in a batch.
Does not check permissions.
Assumes build_requests is valid.
Returns:
A list of (new_build, exception) tuples in the same order.
Exactly one item of a tuple will be non-None.
The exception can be errors.InvalidInputError.
Raises:
Any exception that datastore operations can raise.
"""
# When changing this code, make corresponding changes to
# swarmbucket_api.SwarmbucketApi.get_task_def.
now = utils.utcnow()
identity = auth.get_current_identity()
logging.info('%s is creating %d builds', identity.to_bytes(),
len(build_requests))
settings = yield config.get_settings_async()
# Fetch and index configs.
bucket_ids = {br.bucket_id for br in build_requests}
bucket_cfgs = yield config.get_buckets_async(bucket_ids)
builder_cfgs = {} # {bucket_id: {builder_name: cfg}}
for bucket_id, bucket_cfg in bucket_cfgs.iteritems():
builder_cfgs[bucket_id] = {
b.name: b
for b in bucket_cfg.swarming.builders
}
# Prepare NewBuild objects.
new_builds = []
for r in build_requests:
builder = r.schedule_build_request.builder.builder
bucket_builder_cfgs = builder_cfgs[r.bucket_id]
builder_cfg = bucket_builder_cfgs.get(builder)
# Apply builder config overrides, if any.
# Exists for backward compatibility, runs only in V1 code path.
if builder_cfg and r.override_builder_cfg: # pragma: no cover
builder_cfg = copy.deepcopy(builder_cfg)
r.override_builder_cfg(builder_cfg)
nb = NewBuild(r, builder_cfg)
if bucket_builder_cfgs and not builder_cfg:
nb.exception = errors.BuilderNotFoundError(
'builder "%s" not found in bucket "%s"' %
(builder, r.bucket_id))
new_builds.append(nb)
# Check memcache.
yield [nb.check_cache_async() for nb in new_builds if not nb.final]
# Create and put builds.
to_create = [nb for nb in new_builds if not nb.final]
if to_create:
build_ids = model.create_build_ids(now, len(to_create))
builds = yield [
nb.request.create_build_async(build_id, settings, nb.builder_cfg,
identity, now)
for nb, build_id in zip(to_create, build_ids)
]
for nb, build in zip(to_create, builds):
nb.build = build
yield _update_builders_async(to_create, now)
yield _generate_build_numbers_async(to_create)
yield search.update_tag_indexes_async([nb.build for nb in to_create])
yield [nb.put_and_cache_async() for nb in to_create]
raise ndb.Return([nb.result() for nb in new_builds])
