def parse_luci_bucket(bucket):
"""Converts V1 LUCI bucket to a bucket ID string.
Returns '' if bucket is not a LUCI bucket.
"""
parts = bucket.split('.', 2)
if len(parts) == 3 and parts[0] == 'luci':
return config.format_bucket_id(parts[1], parts[2])
return ''
def schedule_build_async(req, res, _ctx, mask):
"""Schedules one build."""
validation.validate_schedule_build_request(req)
bucket_id = config.format_bucket_id(req.builder.project,
req.builder.bucket)
if not (yield user.can_add_build_async(bucket_id)):
raise current_identity_cannot('schedule builds to bucket %s',
bucket_id)
build_req = creation.BuildRequest(schedule_build_request=req)
build = yield creation.add_async(build_req)
yield build_to_proto_async(build, res, mask)
def update_global_metrics():
"""Updates the metrics in GLOBAL_METRICS."""
start = utils.utcnow()
builder_ids = set() # {(bucket_id, builder)}
for key in model.Builder.query().iter(keys_only=True):
project_id, bucket, builder = key.id().split(':', 2)
bucket_id = (
# TODO(crbug.com/851036): remove parse_luci_bucket call
# once we don't have Builder entities with legacy bucket names.
api_common.parse_luci_bucket(bucket)
or config.format_bucket_id(project_id, bucket))
builder_ids.add((bucket_id, builder))
all_luci_bucket_ids = {
bid
for bid, config in config.get_buckets_async().get_result().iteritems()
if config and config.swarming.builders
}
# Collect a list of counting/latency queries.
count_query_queue = []
latency_query_queue = []
# TODO(crbug.com/851036): join with the loop above and remove builder_ids set.
for bucket_id, builder in builder_ids:
legacy_bucket_name = api_common.legacy_bucket_name(
bucket_id, bucket_id in all_luci_bucket_ids)
latency_query_queue.extend([
(bucket_id, legacy_bucket_name, builder, True),
(bucket_id, legacy_bucket_name, builder, False),
])
for status in (model.BuildStatus.SCHEDULED, model.BuildStatus.STARTED):
for experimental in (False, True):
count_query_queue.append((bucket_id, legacy_bucket_name,
builder, status, experimental))
# Process counting/latency queries with _CONCURRENT_QUERY_LIMIT workers.
@ndb.tasklet
def worker():
while count_query_queue:
item = count_query_queue.pop()
yield set_build_count_metric_async(*item)
while latency_query_queue:
item = latency_query_queue.pop()
yield set_build_latency(*item)
for w in [worker() for _ in xrange(_CONCURRENT_QUERY_LIMIT)]:
w.check_success()
logging.info('global metric computation took %s', utils.utcnow() - start)
def get_task_def(self, request):
"""Returns a swarming task definition for a build request."""
try:
# Checks access too.
request.build_request.bucket = api.convert_bucket(
request.build_request.bucket)
build_request = api.put_request_message_to_build_request(
request.build_request)
# Find builder config.
builder_id = build_request.schedule_build_request.builder
builder_cfg = None
bucket_id = config.format_bucket_id(builder_id.project,
builder_id.bucket)
_, bucket_cfg = config.get_bucket_async(bucket_id).get_result()
assert bucket_cfg, 'if there is no bucket, access check would fail'
for cfg in bucket_cfg.swarming.builders: # pragma: no branch
if cfg.name == builder_id.builder:
builder_cfg = cfg
break
if not builder_cfg:
raise endpoints.NotFoundException(
'Builder %s/%s/%s not found' %
(builder_id.project, builder_id.bucket,
builder_id.builder))
settings = config.get_settings_async().get_result()
# Create a fake build and prepare a task definition.
identity = auth.get_current_identity()
build = build_request.create_build_async(
1, settings, builder_cfg, identity,
utils.utcnow()).get_result()
assert build.proto.HasField('infra')
build.proto.number = 1
settings = config.get_settings_async().get_result()
task_def = swarming.compute_task_def(build,
settings,
fake_build=True)
task_def_json = json.dumps(task_def)
return GetTaskDefinitionResponseMessage(
task_definition=task_def_json,
swarming_host=build.proto.infra.swarming.hostname,
)
except errors.InvalidInputError as ex:
raise endpoints.BadRequestException('invalid build request: %s' %
ex.message)
def setUp(self):
super(UserTest, self).setUp()
self.current_identity = auth.Identity.from_bytes('user:[email protected]')
self.patch('components.auth.get_current_identity',
autospec=True,
side_effect=lambda: self.current_identity)
user.clear_request_cache()
self.patch('components.auth.is_admin',
autospec=True,
return_value=False)
bucket_a = Bucket(name='a',
acls=[
Acl(role=Acl.WRITER, group='a-writers'),
Acl(role=Acl.READER, group='a-readers'),
])
bucket_b = Bucket(name='b',
acls=[
Acl(role=Acl.WRITER, group='b-writers'),
Acl(role=Acl.READER, group='b-readers'),
])
bucket_c = Bucket(name='c',
acls=[
Acl(role=Acl.READER, group='c-readers'),
Acl(role=Acl.READER,
identity='user:[email protected]'),
Acl(role=Acl.WRITER, group='c-writers'),
Acl(role=Acl.READER, identity='project:p1'),
])
all_buckets = [('p1', bucket_a), ('p2', bucket_b), ('p3', bucket_c)]
self.patch('config.get_buckets_async',
autospec=True,
return_value=future(all_buckets))
bucket_map = {
config.format_bucket_id(pid, b.name): b
for pid, b in all_buckets
}
self.patch('config.get_bucket_async',
autospec=True,
side_effect=lambda bid: future(
('deadbeef', bucket_map.get(bid))))
def get_bucket_id(req):
return config.format_bucket_id(req.builder.project, req.builder.bucket)
def bucket_id_string(builder_id):
return config.format_bucket_id(builder_id.project, builder_id.bucket)
def bucket_id(self):
builder_id = self.schedule_build_request.builder
return config.format_bucket_id(builder_id.project, builder_id.bucket)
class Build(ndb.Model):
"""Describes a build.
Build key:
Build keys are autogenerated, monotonically decreasing integers.
That is, when sorted by key, new builds are first.
Build has no parent.
Build id is a 64 bits integer represented as a string to the user.
- 1 highest order bit is set to 0 to keep value positive.
- 43 bits are 43 lower bits of bitwise-inverted time since
BEGINING_OF_THE_WORLD at 1ms resolution.
It is good for 2**43 / 365.3 / 24 / 60 / 60 / 1000 = 278 years
or 2010 + 278 = year 2288.
- 16 bits are set to a random value. Assuming an instance is internally
consistent with itself, it can ensure to not reuse the same 16 bits in two
consecutive requests and/or throttle itself to one request per
millisecond. Using random value reduces to 2**-15 the probability of
collision on exact same timestamp at 1ms resolution, so a maximum
theoretical rate of 65536000 requests/sec but an effective rate in the
range of ~64k qps without much transaction conflicts. We should be fine.
- 4 bits are 0. This is to represent the 'version' of the entity
schema.
The idea is taken from Swarming TaskRequest entity:
https://code.google.com/p/swarming/source/browse/appengine/swarming/server/task_request.py#329
"""
# ndb library sometimes silently ignores memcache errors
# => memcache is not synchronized with datastore
# => a build never finishes from the app code perspective
# => builder is stuck for days.
# We workaround this problem by setting a timeout.
_memcache_timeout = 600 # 10m
# Stores the build proto. The primary property of this entity.
# Majority of the other properties are either derivatives of this field or
# legacy.
#
# Does not include:
# output.properties: see BuildOutputProperties
# steps: see BuildSteps.
# tags: stored in tags attribute, because we have to index them anyway.
# input.properties: see BuildInputProperties.
# CAVEAT: field input.properties does exist during build creation, and
# moved into BuildInputProperties right before initial datastore.put.
# infra: see BuildInfra.
# CAVEAT: field infra does exist during build creation, and moved into
# BuildInfra right before initial datastore.put.
#
# Transition period: proto is either None or complete, i.e. created by
# creation.py or fix_builds.py.
proto = datastore_utils.ProtobufProperty(build_pb2.Build)
# A randomly generated key associated with the created swarming task.
# Embedded in a build token provided to a swarming task in secret bytes.
# Needed in case Buildbucket unintentionally creates multiple swarming tasks
# associated with the build.
# Populated by swarming.py on swarming task creation.
# A part of the message in build token (tokens.py) required for UpdateBuild
# api.
swarming_task_key = ndb.StringProperty(indexed=False)
# == proto-derived properties ================================================
#
# These properties are derived from "proto" properties.
# They are used to index builds.
status = ndb.ComputedProperty(lambda self: self.proto.status,
name='status_v2')
@property
def is_ended(self): # pragma: no cover
return is_terminal_status(self.proto.status)
incomplete = ndb.ComputedProperty(lambda self: not self.is_ended)
# ID of the LUCI project to which this build belongs.
project = ndb.ComputedProperty(lambda self: self.proto.builder.project)
# Indexed string "<project_id>/<bucket_name>".
# Example: "chromium/try".
# Prefix "luci.<project_id>." is stripped from bucket name.
bucket_id = ndb.ComputedProperty(lambda self: config.format_bucket_id(
self.proto.builder.project, self.proto.builder.bucket))
# Indexed string "<project_id>/<bucket_name>/<builder_name>".
# Example: "chromium/try/linux-rel".
# Prefix "luci.<project_id>." is stripped from bucket name.
builder_id = ndb.ComputedProperty(
lambda self: config.builder_id_string(self.proto.builder))
canary = ndb.ComputedProperty(lambda self: self.proto.canary)
# Value of proto.create_time.
# Making this property computed is not-entirely trivial because
# ComputedProperty saves it as int, as opposed to datetime.datetime.
# TODO(nodir): remove usages of create_time indices, rely on build id ordering
# instead.
create_time = ndb.DateTimeProperty()
# A list of colon-separated key-value pairs. Indexed.
# Used to populate tags in builds_to_protos_async, if requested.
tags = ndb.StringProperty(repeated=True)
# If True, the build won't affect monitoring and won't be surfaced in
# search results unless explicitly requested.
experimental = ndb.ComputedProperty(
lambda self: self.proto.input.experimental)
# Value of proto.created_by.
# Making this property computed is not-entirely trivial because
# ComputedProperty saves it as string, but IdentityProperty stores it
# as a blob property.
created_by = auth.IdentityProperty()
is_luci = ndb.BooleanProperty()
@property
def is_ended(self): # pragma: no cover
return self.proto.status not in (common_pb2.STATUS_UNSPECIFIED,
common_pb2.SCHEDULED,
common_pb2.STARTED)
# == Legacy properties =======================================================
status_legacy = msgprop.EnumProperty(BuildStatus,
default=BuildStatus.SCHEDULED,
name='status')
status_changed_time = ndb.DateTimeProperty(auto_now_add=True)
# immutable arbitrary build parameters.
parameters = datastore_utils.DeterministicJsonProperty(json_type=dict)
# PubSub message parameters for build status change notifications.
# TODO(nodir): replace with notification_pb2.NotificationConfig.
pubsub_callback = ndb.StructuredProperty(PubSubCallback, indexed=False)
# id of the original build that this build was derived from.
retry_of = ndb.IntegerProperty()
# a URL to a build-system-specific build, viewable by a human.
url = ndb.StringProperty(indexed=False)
# V1 status properties. Computed by _pre_put_hook.
result = msgprop.EnumProperty(BuildResult)
result_details = datastore_utils.DeterministicJsonProperty(json_type=dict)
cancelation_reason = msgprop.EnumProperty(CancelationReason)
failure_reason = msgprop.EnumProperty(FailureReason)
# Lease-time properties.
# TODO(nodir): move lease to a separate entity under Build.
# It would be more efficient.
# current lease expiration date.
# The moment the build is leased, |lease_expiration_date| is set to
# (utcnow + lease_duration).
lease_expiration_date = ndb.DateTimeProperty()
# None if build is not leased, otherwise a random value.
# Changes every time a build is leased. Can be used to verify that a client
# is the leaseholder.
lease_key = ndb.IntegerProperty(indexed=False)
# True if the build is currently leased. Otherwise False
is_leased = ndb.ComputedProperty(lambda self: self.lease_key is not None)
leasee = auth.IdentityProperty()
never_leased = ndb.BooleanProperty()
# ============================================================================
def _pre_put_hook(self):
"""Checks Build invariants before putting."""
super(Build, self)._pre_put_hook()
config.validate_project_id(self.proto.builder.project)
config.validate_bucket_name(self.proto.builder.bucket)
self.update_v1_status_fields()
self.proto.update_time.FromDatetime(utils.utcnow())
is_started = self.proto.status == common_pb2.STARTED
is_ended = self.is_ended
is_leased = self.lease_key is not None
assert not (is_ended and is_leased)
assert (self.lease_expiration_date is not None) == is_leased
assert (self.leasee is not None) == is_leased
tag_delm = buildtags.DELIMITER
assert not self.tags or all(tag_delm in t for t in self.tags)
assert self.proto.HasField('create_time')
assert self.proto.HasField('end_time') == is_ended
assert not is_started or self.proto.HasField('start_time')
def _ts_less(ts1, ts2):
return ts1.seconds and ts2.seconds and ts1.ToDatetime(
) < ts2.ToDatetime()
assert not _ts_less(self.proto.start_time, self.proto.create_time)
assert not _ts_less(self.proto.end_time, self.proto.create_time)
assert not _ts_less(self.proto.end_time, self.proto.start_time)
self.tags = sorted(set(self.tags))
def update_v1_status_fields(self):
"""Updates V1 status fields."""
self.status_legacy = None
self.result = None
self.failure_reason = None
self.cancelation_reason = None
status = self.proto.status
if status == common_pb2.SCHEDULED:
self.status_legacy = BuildStatus.SCHEDULED
elif status == common_pb2.STARTED:
self.status_legacy = BuildStatus.STARTED
elif status == common_pb2.SUCCESS:
self.status_legacy = BuildStatus.COMPLETED
self.result = BuildResult.SUCCESS
elif status == common_pb2.FAILURE:
self.status_legacy = BuildStatus.COMPLETED
self.result = BuildResult.FAILURE
self.failure_reason = FailureReason.BUILD_FAILURE
elif status == common_pb2.INFRA_FAILURE:
self.status_legacy = BuildStatus.COMPLETED
if self.proto.status_details.HasField('timeout'):
self.result = BuildResult.CANCELED
self.cancelation_reason = CancelationReason.TIMEOUT
else:
self.result = BuildResult.FAILURE
self.failure_reason = FailureReason.INFRA_FAILURE
elif status == common_pb2.CANCELED:
self.status_legacy = BuildStatus.COMPLETED
self.result = BuildResult.CANCELED
self.cancelation_reason = CancelationReason.CANCELED_EXPLICITLY
else: # pragma: no cover
assert False, status
def regenerate_lease_key(self):
"""Changes lease key to a different random int."""
while True:
new_key = random.randint(0, 1 << 31)
if new_key != self.lease_key: # pragma: no branch
self.lease_key = new_key
break
def clear_lease(self): # pragma: no cover
"""Clears build's lease attributes."""
self.lease_key = None
self.lease_expiration_date = None
self.leasee = None
def tags_to_protos(self, dest):
"""Adds non-hidden self.tags to a repeated StringPair container."""
for t in self.tags:
k, v = buildtags.parse(t)
if k not in buildtags.HIDDEN_TAG_KEYS:
dest.add(key=k, value=v)
