class PackageACLBody(ndb.Model):
"""Shared by PackageACL and PackageACLRevision.
Doesn't actually exist in the datastore by itself. Only inherited from.
"""
# Users granted the given role directly. Often only one account should be
# granted some role (e.g. a role account should be WRITER). It is annoying to
# manage one-account groups for cases like this.
users = auth.IdentityProperty(indexed=False, repeated=True)
# Groups granted the given role.
groups = ndb.StringProperty(indexed=False, repeated=True)
# Who made the last change.
modified_by = auth.IdentityProperty(indexed=True)
# When the last change was made.
modified_ts = ndb.DateTimeProperty(indexed=True)
class InstanceTag(ndb.Model):
"""Single tag of some package instance.
ID is hex-encoded SHA1 of the tag. Parent entity is corresponding
PackageInstance. The tag can't be made a part of the key because total key
length (including class names, all parent keys) is limited to 500 characters.
Tags can be pretty long.
Tags are separate entities (rather than repeated field in PackageInstance) for
two reasons:
* There can be many tags attached to an instance (e.g. 'git_revision:...'
tags for a package that doesn't change between revisions).
* PackageInstance entity is fetched pretty often, no need to pull all tags
all the time.
"""
# Disable useless in-memory per-request cache. It's harmful in tests.
_use_cache = False
# The tag itself, as key:value string.
tag = ndb.StringProperty()
# Who added this tag.
registered_by = auth.IdentityProperty()
# When the tag was added.
registered_ts = ndb.DateTimeProperty()
@property
def package_name(self):
"""Name of the package this tag belongs to."""
return self.key.parent().parent().string_id()
@property
def instance_id(self):
"""Package instance ID this tag belongs to."""
return self.key.parent().string_id()
class GroupImporterConfig(ndb.Model):
"""Singleton entity with group importer configuration JSON."""
config_proto = ndb.TextProperty()
config_revision = ndb.JsonProperty(
) # see config.py, _update_imports_config
modified_by = auth.IdentityProperty(indexed=False)
modified_ts = ndb.DateTimeProperty(auto_now=True, indexed=False)
class PackageInstance(ndb.Model):
"""Represents some uploaded package instance.
ID is package instance ID (SHA1 hex digest of package body).
Parent entity is Package(id=package_name).
"""
# Disable useless in-memory per-request cache. It's harmful in tests.
_use_cache = False
# Who registered the instance.
registered_by = auth.IdentityProperty()
# When the instance was registered.
registered_ts = ndb.DateTimeProperty()
# Names of processors scheduled for an instance or currently running.
processors_pending = ndb.StringProperty(repeated=True)
# Names of processors that successfully finished the processing.
processors_success = ndb.StringProperty(repeated=True)
# Names of processors that returned fatal error.
processors_failure = ndb.StringProperty(repeated=True)
@property
def package_name(self):
"""Name of the package this instance belongs to."""
return self.key.parent().string_id()
@property
def instance_id(self):
"""Package instance ID (SHA1 of package file content)."""
return self.key.string_id()
class VersionedFile(ndb.Model):
"""Versionned entity.
Root is ROOT_MODEL. id is datastore_utils.HIGH_KEY_ID - version number.
"""
created_ts = ndb.DateTimeProperty(indexed=False, auto_now_add=True)
who = auth.IdentityProperty(indexed=False)
content = ndb.BlobProperty(compressed=True)
ROOT_MODEL = datastore_utils.get_versioned_root_model('VersionedFileRoot')
@property
def version(self):
return datastore_utils.HIGH_KEY_ID - self.key.integer_id()
@classmethod
def fetch(cls, name):
"""Returns the current version of the instance."""
return datastore_utils.get_versioned_most_recent(
cls, cls._gen_root_key(name))
def store(self, name):
"""Stores a new version of the instance."""
# Create an incomplete key.
self.key = ndb.Key(self.__class__, None, parent=self._gen_root_key(name))
self.who = auth.get_current_identity()
return datastore_utils.store_new_version(self, self.ROOT_MODEL)
@classmethod
def _gen_root_key(cls, name):
return ndb.Key(cls.ROOT_MODEL, name)
class Package(ndb.Model):
"""Entity root for PackageInstance entities for some particular package.
Id is a package name.
"""
# Who registered the package.
registered_by = auth.IdentityProperty()
# When the package was registered.
registered_ts = ndb.DateTimeProperty()
@property
def package_name(self):
"""Name of the package."""
return self.key.string_id()
class PackageRef(ndb.Model):
"""A named reference to some instance ID.
ID is a reference name, parent entity is corresponding Package.
"""
# PackageInstance the ref points to.
instance_id = ndb.StringProperty()
# Who added or moved this reference.
modified_by = auth.IdentityProperty()
# When the reference was created or moved.
modified_ts = ndb.DateTimeProperty()
@property
def package_name(self):
"""Name of the package this ref belongs to."""
return self.key.parent().string_id()
class UploadSession(ndb.Model):
"""Some pending upload operation.
Entity id is autogenerated by the datastore. No parent entity.
"""
# Upload session never existed or already expired.
STATUS_MISSING = 0
# Client is still uploading the file.
STATUS_UPLOADING = 1
# Server is verifying the hash of the uploaded file.
STATUS_VERIFYING = 2
# The file is in the store and visible by all clients. Final state.
STATUS_PUBLISHED = 3
# Some other unexpected fatal error happened.
STATUS_ERROR = 4
# Hash algorithm to use to verify the content.
hash_algo = ndb.StringProperty(required=True, indexed=False)
# Expected hex digest of the file.
hash_digest = ndb.StringProperty(required=True, indexed=False)
# Full path in the GS to the temporary drop file that the client upload to.
temp_gs_location = ndb.TextProperty(required=True)
# Full path in the GS where to store the verified file.
final_gs_location = ndb.TextProperty(required=True)
# URL to put file content too.
upload_url = ndb.TextProperty(required=True)
# Status of the upload operation. See STATUS_* constants.
status = ndb.IntegerProperty(required=True,
choices=[
STATUS_ERROR,
STATUS_MISSING,
STATUS_PUBLISHED,
STATUS_UPLOADING,
STATUS_VERIFYING,
])
# For STATUS_ERROR may contain an error message.
error_message = ndb.TextProperty(required=False)
# Who started the upload.
created_by = auth.IdentityProperty(required=True)
# When the entity was created.
created_ts = ndb.DateTimeProperty(required=True, auto_now_add=True)
class Package(ndb.Model):
"""Entity root for PackageInstance entities for some particular package.
Id is a package name.
"""
# Disable useless in-memory per-request cache. It's harmful in tests.
_use_cache = False
# Who registered the package.
registered_by = auth.IdentityProperty()
# When the package was registered.
registered_ts = ndb.DateTimeProperty()
# If True, the package won't show up in listings, see 'list_packages'.
hidden = ndb.BooleanProperty()
@property
def package_name(self):
"""Name of the package."""
return self.key.string_id()
class PackageRef(ndb.Model):
"""A named reference to some instance ID.
ID is a reference name, parent entity is corresponding Package.
"""
# Disable useless in-memory per-request cache. It's harmful in tests.
_use_cache = False
# PackageInstance the ref points to.
instance_id = ndb.StringProperty()
# Who added or moved this reference.
modified_by = auth.IdentityProperty()
# When the reference was created or moved.
modified_ts = ndb.DateTimeProperty()
@property
def package_name(self):
"""Name of the package this ref belongs to."""
return self.key.parent().string_id()
@property
def ref(self):
"""Name of the ref (extracted from entity key)."""
return self.key.string_id()
class TaskRequest(ndb.Model):
"""Contains a user request.
Key id is a decreasing integer based on time since utils.EPOCH plus some
randomness on lower order bits. See new_request_key() for the complete gory
details.
This model is immutable.
"""
# Hashing algorithm used to hash TaskProperties to create its key.
HASHING_ALGO = hashlib.sha1
# Time this request was registered. It is set manually instead of using
# auto_now_add=True so that expiration_ts can be set very precisely relative
# to this property.
created_ts = ndb.DateTimeProperty(required=True)
# The name for this task request. It's only for description.
name = ndb.StringProperty(required=True)
# Authenticated client that triggered this task.
authenticated = auth.IdentityProperty()
# Which user to blame for this task. Can be arbitrary, not asserted by any
# credentials.
user = ndb.StringProperty(default='')
# Indicates what OAuth2 credentials the task uses when calling other services.
#
# Possible values are: 'none', 'bot' or <email>. For more information see
# swarming_rpcs.NewTaskRequest.
#
# This property exists only for informational purposes and for indexing. When
# actually getting an OAuth credentials, the original token (stored in hidden
# 'service_account_token' field) is used.
service_account = ndb.StringProperty(validator=_validate_service_account)
# The delegation token passed via 'service_account_token' when creating a new
# task that should by run with the authority of some service account. Can also
# be set to literal 'none' or 'bot'. See swarming_rpcs.NewTaskRequest for more
# information.
#
# This property never shows up in UI or API responses.
service_account_token = ndb.BlobProperty()
# The actual properties are embedded in this model.
properties = ndb.LocalStructuredProperty(
TaskProperties, compressed=True, required=True)
# Priority of the task to be run. A lower number is higher priority, thus will
# preempt requests with lower priority (higher numbers).
priority = ndb.IntegerProperty(
indexed=False, validator=_validate_priority, required=True)
# If the task request is not scheduled by this moment, it will be aborted by a
# cron job. It is saved instead of scheduling_expiration_secs so finding
# expired jobs is a simple query.
expiration_ts = ndb.DateTimeProperty(
indexed=True, validator=_validate_expiration, required=True)
# Tags that specify the category of the task.
tags = ndb.StringProperty(repeated=True, validator=_validate_tags)
# Set when a task (the parent) reentrantly create swarming tasks. Must be set
# to a valid task_id pointing to a TaskRunResult or be None.
parent_task_id = ndb.StringProperty(validator=_validate_task_run_id)
# PubSub topic to send task completion notification to.
pubsub_topic = ndb.StringProperty(indexed=False)
# Secret token to send as 'auth_token' attribute with PubSub messages.
pubsub_auth_token = ndb.StringProperty(indexed=False)
# Data to send in 'userdata' field of PubSub messages.
pubsub_userdata = ndb.StringProperty(indexed=False)
# This stores the computed properties_hash for this Task's properties object.
# It is set in init_new_request. It is None for non-idempotent tasks.
properties_hash = ndb.BlobProperty(indexed=False)
@property
def task_id(self):
"""Returns the TaskResultSummary packed id, not the task request key."""
return task_pack.pack_result_summary_key(
task_pack.request_key_to_result_summary_key(self.key))
@property
def expiration_secs(self):
"""Reconstructs this value from expiration_ts and created_ts. Integer."""
return int((self.expiration_ts - self.created_ts).total_seconds())
@property
def has_access(self):
"""Returns True if the current user has read-write access to this request.
This is used for:
* Read access: ability to read the task info and logs.
* Write access: ability to cancel the task.
Warning: This function looks at the current Authentication context.
"""
return (
acl.is_privileged_user() or
self.authenticated == auth.get_current_identity())
def to_dict(self):
"""Converts properties_hash to hex so it is json serializable."""
# to_dict() doesn't recurse correctly into ndb.LocalStructuredProperty!
out = super(TaskRequest, self).to_dict(
exclude=['pubsub_auth_token', 'properties', 'service_account_token'])
out['properties'] = self.properties.to_dict()
properties_hash = out['properties_hash']
out['properties_hash'] = (
properties_hash.encode('hex') if properties_hash else None)
return out
def _pre_put_hook(self):
"""Adds automatic tags."""
super(TaskRequest, self)._pre_put_hook()
self.properties._pre_put_hook()
if self.properties.is_terminate:
if not self.priority == 0:
raise datastore_errors.BadValueError(
'terminate request must be priority 0')
elif self.priority == 0:
raise datastore_errors.BadValueError(
'priority 0 can only be used for terminate request')
if (self.pubsub_topic and
not pubsub.validate_full_name(self.pubsub_topic, 'topics')):
raise datastore_errors.BadValueError(
'bad pubsub topic name - %s' % self.pubsub_topic)
if self.pubsub_auth_token and not self.pubsub_topic:
raise datastore_errors.BadValueError(
'pubsub_auth_token requires pubsub_topic')
if self.pubsub_userdata and not self.pubsub_topic:
raise datastore_errors.BadValueError(
'pubsub_userdata requires pubsub_topic')
class LeaseRequest(ndb.Model):
"""Datastore representation of a LeaseRequest.
Key:
id: Hash of the client + client-generated request ID which issued the
original rpc_messages.LeaseRequest instance. Used for easy deduplication.
kind: LeaseRequest. This root entity does not reference any parents.
"""
# DateTime indicating original datastore write time.
created_ts = ndb.DateTimeProperty(auto_now_add=True)
# Checksum of the rpc_messages.LeaseRequest instance. Used to compare incoming
# LeaseRequets for deduplication.
deduplication_checksum = ndb.StringProperty(required=True, indexed=False)
# ID of the CatalogMachineEntry provided for this lease.
machine_id = ndb.StringProperty()
# auth.model.Identity of the issuer of the original request.
owner = auth.IdentityProperty(required=True)
# Element of LeaseRequestStates giving the state of this request.
state = ndb.StringProperty(choices=LeaseRequestStates, required=True)
# rpc_messages.LeaseRequest instance representing the original request.
request = msgprop.MessageProperty(rpc_messages.LeaseRequest, required=True)
# rpc_messages.LeaseResponse instance representing the current response.
# This field will be updated as the request is being processed.
response = msgprop.MessageProperty(rpc_messages.LeaseResponse)
@classmethod
def compute_deduplication_checksum(cls, request):
"""Computes the deduplication checksum for the given request.
Args:
request: The rpc_messages.LeaseRequest instance to deduplicate.
Returns:
The deduplication checksum.
"""
return hashlib.sha1(protobuf.encode_message(request)).hexdigest()
@classmethod
def generate_key(cls, user, request):
"""Generates the key for the given request initiated by the given user.
Args:
user: An auth.model.Identity instance representing the requester.
request: The rpc_messages.LeaseRequest sent by the user.
Returns:
An ndb.Key instance.
"""
# Enforces per-user request ID uniqueness
return ndb.Key(
cls,
hashlib.sha1('%s\0%s' % (user, request.request_id)).hexdigest(),
)
@classmethod
def query_untriaged(cls):
"""Queries for untriaged LeaseRequests.
Yields:
Untriaged LeaseRequests in no guaranteed order.
"""
for request in cls.query(cls.state == LeaseRequestStates.UNTRIAGED):
yield request
class TaskRequest(ndb.Model):
"""Contains a user request.
Key id is a decreasing integer based on time since utils.EPOCH plus some
randomness on lower order bits. See _new_request_key() for the complete gory
details.
There is also "old style keys" which inherit from a fake root entity
TaskRequestShard.
TODO(maruel): Remove support 2015-02-01.
This model is immutable.
"""
# Time this request was registered. It is set manually instead of using
# auto_now_add=True so that expiration_ts can be set very precisely relative
# to this property.
created_ts = ndb.DateTimeProperty(required=True)
# The name for this task request. It's only for description.
name = ndb.StringProperty(required=True)
# Authenticated client that triggered this task.
authenticated = auth.IdentityProperty()
# Which user to blame for this task.
user = ndb.StringProperty(default='')
# The actual properties are embedded in this model.
properties = ndb.LocalStructuredProperty(
TaskProperties, compressed=True, required=True)
# Priority of the task to be run. A lower number is higher priority, thus will
# preempt requests with lower priority (higher numbers).
priority = ndb.IntegerProperty(
indexed=False, validator=_validate_priority, required=True)
# If the task request is not scheduled by this moment, it will be aborted by a
# cron job. It is saved instead of scheduling_expiration_secs so finding
# expired jobs is a simple query.
expiration_ts = ndb.DateTimeProperty(
indexed=True, validator=_validate_expiration, required=True)
# Tags that specify the category of the task.
tags = ndb.StringProperty(repeated=True, validator=_validate_tags)
# Set when a task (the parent) reentrantly create swarming tasks. Must be set
# to a valid task_id pointing to a TaskRunResult or be None.
parent_task_id = ndb.StringProperty(validator=_validate_task_run_id)
@property
def scheduling_expiration_secs(self):
"""Reconstructs this value from expiration_ts and created_ts."""
return (self.expiration_ts - self.created_ts).total_seconds()
def to_dict(self):
"""Converts properties_hash to hex so it is json serializable."""
out = super(TaskRequest, self).to_dict()
properties_hash = self.properties.properties_hash
out['properties_hash'] = (
properties_hash.encode('hex') if properties_hash else None)
return out
def _pre_put_hook(self):
"""Adds automatic tags."""
super(TaskRequest, self)._pre_put_hook()
self.tags.append('priority:%s' % self.priority)
self.tags.append('user:%s' % self.user)
for key, value in self.properties.dimensions.iteritems():
self.tags.append('%s:%s' % (key, value))
self.tags = sorted(set(self.tags))
class GlobalConfig(ndb.Model):
"""Singleton entity with the global configuration of the service.
All changes are stored in the revision log.
"""
# When this revision of configuration was created.
updated_ts = ndb.DateTimeProperty(indexed=False, auto_now_add=True)
# Who created this revision of configuration.
updated_by = auth.IdentityProperty(indexed=False)
@classmethod
def cached(cls):
"""Fetches config entry from local cache or datastore.
Bootstraps it if missing. May return slightly stale data but in most cases
doesn't do any RPCs. Should be used for read-only access to config.
"""
# @utils.cache_with_expiration can't be used directly with 'cached' since it
# will be applied to base class method, not a concrete implementation
# specific to a subclass. So build new class specific fetcher on the fly on
# a first attempt (it's not a big deal if it happens concurrently in MT
# environment, last one wins). Same can be achieved with metaclasses, but no
# one likes metaclasses.
if not cls._config_fetcher:
@utils.cache_with_expiration(expiration_sec=60)
def config_fetcher():
conf = cls.fetch()
if not conf:
conf = cls()
conf.set_defaults()
conf.store(updated_by=auth.get_service_self_identity())
return conf
cls._config_fetcher = staticmethod(config_fetcher)
return cls._config_fetcher()
@classmethod
def clear_cache(cls):
"""Clears the cache of .cached().
So the next call to .cached() returns the fresh instance from ndb.
"""
if cls._config_fetcher:
utils.clear_cache(cls._config_fetcher)
@classmethod
def fetch(cls):
"""Returns the current up-to-date version of the config entity.
Always fetches it from datastore. May return None if missing.
"""
return datastore_utils.get_versioned_most_recent(
cls, cls._get_root_key())
def store(self, updated_by=None):
"""Stores a new version of the config entity."""
# Create an incomplete key, to be completed by 'store_new_version'.
self.key = ndb.Key(self.__class__, None, parent=self._get_root_key())
self.updated_by = updated_by or auth.get_current_identity()
self.updated_ts = utils.utcnow()
return datastore_utils.store_new_version(self, self._get_root_model())
def modify(self, updated_by=None, **kwargs):
"""Applies |kwargs| dict to the entity and stores the entity if changed."""
dirty = False
for k, v in kwargs.iteritems():
assert k in self._properties, k
if getattr(self, k) != v:
setattr(self, k, v)
dirty = True
if dirty:
self.store(updated_by=updated_by)
return dirty
def set_defaults(self):
"""Fills in default values for empty config. Implemented by subclasses."""
### Private stuff.
_config_fetcher = None
@classmethod
def _get_root_model(cls):
return datastore_utils.get_versioned_root_model('%sRoot' %
cls.__name__)
@classmethod
def _get_root_key(cls):
return ndb.Key(cls._get_root_model(), 1)
class ConfigSettings(config.GlobalConfig): # Hostname of the config service. service_hostname = ndb.StringProperty(indexed=False) # Identity account used by config service. trusted_config_account = auth.IdentityProperty(indexed=False)
class GlobalConfig(ndb.Model):
"""Singleton entity with the global configuration of the service.
All changes are stored in the revision log.
"""
# When this revision of configuration was created.
updated_ts = ndb.DateTimeProperty(indexed=False, auto_now_add=True)
# Who created this revision of configuration.
updated_by = auth.IdentityProperty(indexed=False)
@classmethod
def cached_async(cls):
"""Fetches config entry from local cache or datastore.
Bootstraps it if missing. May return slightly stale data but in most cases
doesn't do any RPCs. Should be used for read-only access to config.
"""
# Build new class-specific fetcher function with cache on the fly on
# the first attempt (it's not a big deal if it happens concurrently in MT
# environment, last one wins). Same can be achieved with metaclasses, but no
# one likes metaclasses.
if not cls._config_fetcher_async:
@ndb.tasklet
def fetcher():
with fetcher.cache_lock:
expiry = fetcher.cache_expiry
if expiry is not None and utils.utcnow() < expiry:
raise ndb.Return(fetcher.cache_value)
# Do not lock while yielding, it would cause deadlock.
# Also do not cache a future, it might cross ndb context boundary.
# If there is no cached value, multiple concurrent requests will make
# multiple RPCs, but as soon as one of them updates cache, subsequent
# requests will use the cached value, for a minute.
conf = yield cls.fetch_async()
if not conf:
conf = cls()
conf.set_defaults()
yield conf.store_async(
updated_by=auth.get_service_self_identity())
with fetcher.cache_lock:
fetcher.cache_expiry = utils.utcnow() + datetime.timedelta(
minutes=1)
fetcher.cache_value = conf
raise ndb.Return(conf)
fetcher.cache_lock = threading.Lock()
fetcher.cache_expiry = None
fetcher.cache_value = None
cls._config_fetcher_async = staticmethod(fetcher)
return cls._config_fetcher_async()
cached = utils.sync_of(cached_async)
@classmethod
def clear_cache(cls):
"""Clears the cache of .cached().
So the next call to .cached() returns the fresh instance from ndb.
"""
if cls._config_fetcher_async:
cls._config_fetcher_async.cache_expiry = None
@classmethod
def fetch_async(cls):
"""Returns the current up-to-date version of the config entity.
Always fetches it from datastore. May return None if missing.
"""
return datastore_utils.get_versioned_most_recent_async(
cls, cls._get_root_key())
fetch = utils.sync_of(fetch_async)
def store_async(self, updated_by=None):
"""Stores a new version of the config entity."""
# Create an incomplete key, to be completed by 'store_new_version'.
self.key = ndb.Key(self.__class__, None, parent=self._get_root_key())
self.updated_by = updated_by or auth.get_current_identity()
self.updated_ts = utils.utcnow()
return datastore_utils.store_new_version_async(self,
self._get_root_model())
store = utils.sync_of(store_async)
def modify(self, updated_by=None, **kwargs):
"""Applies |kwargs| dict to the entity and stores the entity if changed."""
dirty = False
for k, v in kwargs.iteritems():
assert k in self._properties, k
if getattr(self, k) != v:
setattr(self, k, v)
dirty = True
if dirty:
self.store(updated_by=updated_by)
return dirty
def set_defaults(self):
"""Fills in default values for empty config. Implemented by subclasses."""
### Private stuff.
_config_fetcher_async = None
@classmethod
def _get_root_model(cls):
return datastore_utils.get_versioned_root_model('%sRoot' %
cls.__name__)
@classmethod
def _get_root_key(cls):
return ndb.Key(cls._get_root_model(), 1)
class Build(ndb.Model):
"""Describes a build.
Build key:
Build keys are autogenerated, monotonically decreasing integers.
That is, when sorted by key, new builds are first.
Build has no parent.
Build id is a 64 bits integer represented as a string to the user.
- 1 highest order bit is set to 0 to keep value positive.
- 43 bits are 43 lower bits of bitwise-inverted time since
BEGINING_OF_THE_WORLD at 1ms resolution.
It is good for 2**43 / 365.3 / 24 / 60 / 60 / 1000 = 278 years
or 2010 + 278 = year 2288.
- 16 bits are set to a random value. Assuming an instance is internally
consistent with itself, it can ensure to not reuse the same 16 bits in two
consecutive requests and/or throttle itself to one request per
millisecond. Using random value reduces to 2**-15 the probability of
collision on exact same timestamp at 1ms resolution, so a maximum
theoretical rate of 65536000 requests/sec but an effective rate in the
range of ~64k qps without much transaction conflicts. We should be fine.
- 4 bits are 0. This is to represent the 'version' of the entity
schema.
The idea is taken from Swarming TaskRequest entity:
https://code.google.com/p/swarming/source/browse/appengine/swarming/server/task_request.py#329
"""
status = msgprop.EnumProperty(BuildStatus, default=BuildStatus.SCHEDULED)
status_changed_time = ndb.DateTimeProperty(auto_now_add=True)
update_time = ndb.DateTimeProperty(auto_now=True)
# Creation time attributes.
create_time = ndb.DateTimeProperty(auto_now_add=True)
created_by = auth.IdentityProperty()
# a generic way to distinguish builds.
# Different buckets have different permissions.
bucket = ndb.StringProperty(required=True)
# a list of tags, where each tag is a string
# with ":" symbol. The first occurrence of ":" splits tag name and tag
# value. Contains only tags specified by the build request. Old Build
# entities do not have this field.
initial_tags = ndb.StringProperty(repeated=True, indexed=False)
# superset of initial_tags. May contain auto-added tags.
tags = ndb.StringProperty(repeated=True)
# immutable arbitrary build parameters.
parameters = ndb.JsonProperty()
# PubSub message parameters for build status change notifications.
pubsub_callback = ndb.StructuredProperty(PubSubCallback, indexed=False)
# id of the original build that this build was derived from.
retry_of = ndb.IntegerProperty()
# Lease-time attributes.
# current lease expiration date.
# The moment the build is leased, |lease_expiration_date| is set to
# (utcnow + lease_duration).
lease_expiration_date = ndb.DateTimeProperty()
# None if build is not leased, otherwise a random value.
# Changes every time a build is leased. Can be used to verify that a client
# is the leaseholder.
lease_key = ndb.IntegerProperty(indexed=False)
# True if the build is currently leased. Otherwise False
is_leased = ndb.ComputedProperty(lambda self: self.lease_key is not None)
leasee = auth.IdentityProperty()
never_leased = ndb.BooleanProperty()
# Start time attributes.
# a URL to a build-system-specific build, viewable by a human.
url = ndb.StringProperty(indexed=False)
# when the build started. Unknown for old builds.
start_time = ndb.DateTimeProperty()
# Completion time attributes.
complete_time = ndb.DateTimeProperty()
result = msgprop.EnumProperty(BuildResult)
result_details = ndb.JsonProperty()
cancelation_reason = msgprop.EnumProperty(CancelationReason)
failure_reason = msgprop.EnumProperty(FailureReason)
# Swarming integration
swarming_hostname = ndb.StringProperty()
swarming_task_id = ndb.StringProperty()
def _pre_put_hook(self):
"""Checks Build invariants before putting."""
super(Build, self)._pre_put_hook()
is_started = self.status == BuildStatus.STARTED
is_completed = self.status == BuildStatus.COMPLETED
is_canceled = self.result == BuildResult.CANCELED
is_failure = self.result == BuildResult.FAILURE
is_leased = self.lease_key is not None
assert (self.result is not None) == is_completed
assert (self.cancelation_reason is not None) == is_canceled
assert (self.failure_reason is not None) == is_failure
assert not (is_completed and is_leased)
assert (self.lease_expiration_date is not None) == is_leased
assert (self.leasee is not None) == is_leased
# no cover due to a bug in coverage (http://stackoverflow.com/a/35325514)
assert not self.tags or all(':' in t
for t in self.tags) # pragma: no cover
assert self.create_time
assert (self.complete_time is not None) == is_completed
assert not is_started or self.start_time
assert not self.start_time or self.start_time >= self.create_time
assert not self.complete_time or self.complete_time >= self.create_time
assert (not self.complete_time or not self.start_time
or self.complete_time >= self.start_time)
def regenerate_lease_key(self):
"""Changes lease key to a different random int."""
while True:
new_key = random.randint(0, 1 << 31)
if new_key != self.lease_key: # pragma: no branch
self.lease_key = new_key
break
def clear_lease(self): # pragma: no cover
"""Clears build's lease attributes."""
self.lease_key = None
self.lease_expiration_date = None
self.leasee = None
class Build(ndb.Model):
"""Describes a build.
Build key:
Build keys are autogenerated, monotonically decreasing integers.
That is, when sorted by key, new builds are first.
Build has no parent.
Build id is a 64 bits integer represented as a string to the user.
- 1 highest order bit is set to 0 to keep value positive.
- 43 bits are 43 lower bits of bitwise-inverted time since
BEGINING_OF_THE_WORLD at 1ms resolution.
It is good for 2**43 / 365.3 / 24 / 60 / 60 / 1000 = 278 years
or 2010 + 278 = year 2288.
- 16 bits are set to a random value. Assuming an instance is internally
consistent with itself, it can ensure to not reuse the same 16 bits in two
consecutive requests and/or throttle itself to one request per
millisecond. Using random value reduces to 2**-15 the probability of
collision on exact same timestamp at 1ms resolution, so a maximum
theoretical rate of 65536000 requests/sec but an effective rate in the
range of ~64k qps without much transaction conflicts. We should be fine.
- 4 bits are 0. This is to represent the 'version' of the entity
schema.
The idea is taken from Swarming TaskRequest entity:
https://code.google.com/p/swarming/source/browse/appengine/swarming/server/task_request.py#329
Attributes:
status (BuildStatus): status of the build.
bucket (string): a generic way to distinguish builds. Different buckets have
different permissions.
tags (list of string): a list of tags, where each tag is a string with ":"
symbol. The first occurance of ":" splits tag name and tag value.
parameters (dict): immutable arbitrary build parameters.
callback (Callback): push task parameters for build status changes.
lease_expiration_date (datetime): current lease expiration date.
The moment the build is leased, |lease_expiration_date| is set to
(utcnow + lease_duration).
lease_key (int): None if build is not leased, otherwise a random value.
Changes every time a build is leased. Can be used to verify that a client
is the leaseholder.
is_leased (bool): True if the build is currently leased. Otherwise False
url (str): a URL to a build-system-specific build, viewable by a human.
result (BuildResult): build result.
cancelation_reason (CancelationReason): why the build was canceled.
"""
status = msgprop.EnumProperty(BuildStatus, default=BuildStatus.SCHEDULED)
status_changed_time = ndb.DateTimeProperty(auto_now_add=True)
update_time = ndb.DateTimeProperty(auto_now=True)
# Creation time attributes.
create_time = ndb.DateTimeProperty(auto_now_add=True)
created_by = auth.IdentityProperty()
bucket = ndb.StringProperty(required=True)
tags = ndb.StringProperty(repeated=True)
parameters = ndb.JsonProperty()
callback = ndb.StructuredProperty(Callback, indexed=False)
# Lease-time attributes.
lease_expiration_date = ndb.DateTimeProperty()
lease_key = ndb.IntegerProperty(indexed=False)
is_leased = ndb.ComputedProperty(lambda self: self.lease_key is not None)
leasee = auth.IdentityProperty()
# Start time attributes.
url = ndb.StringProperty(indexed=False)
# Completion time attributes.
complete_time = ndb.DateTimeProperty()
result = msgprop.EnumProperty(BuildResult)
result_details = ndb.JsonProperty()
cancelation_reason = msgprop.EnumProperty(CancelationReason)
failure_reason = msgprop.EnumProperty(FailureReason)
def _pre_put_hook(self):
"""Checks Build invariants before putting."""
super(Build, self)._pre_put_hook()
is_completed = self.status == BuildStatus.COMPLETED
assert (self.result is not None) == is_completed
is_canceled = self.result == BuildResult.CANCELED
is_failure = self.result == BuildResult.FAILURE
assert (self.cancelation_reason is not None) == is_canceled
assert (self.failure_reason is not None) == is_failure
is_leased = self.lease_key is not None
assert not (is_completed and is_leased)
assert (self.lease_expiration_date is not None) == is_leased
assert (self.leasee is not None) == is_leased
assert not self.tags or all(':' in t for t in self.tags)
def regenerate_lease_key(self):
"""Changes lease key to a different random int."""
while True:
new_key = random.randint(0, 1 << 31)
if new_key != self.lease_key: # pragma: no branch
self.lease_key = new_key
break
class Build(ndb.Model):
"""Describes a build.
Build key:
Build keys are autogenerated, monotonically decreasing integers.
That is, when sorted by key, new builds are first.
Build has no parent.
Build id is a 64 bits integer represented as a string to the user.
- 1 highest order bit is set to 0 to keep value positive.
- 43 bits are 43 lower bits of bitwise-inverted time since
BEGINING_OF_THE_WORLD at 1ms resolution.
It is good for 2**43 / 365.3 / 24 / 60 / 60 / 1000 = 278 years
or 2010 + 278 = year 2288.
- 16 bits are set to a random value. Assuming an instance is internally
consistent with itself, it can ensure to not reuse the same 16 bits in two
consecutive requests and/or throttle itself to one request per
millisecond. Using random value reduces to 2**-15 the probability of
collision on exact same timestamp at 1ms resolution, so a maximum
theoretical rate of 65536000 requests/sec but an effective rate in the
range of ~64k qps without much transaction conflicts. We should be fine.
- 4 bits are 0. This is to represent the 'version' of the entity
schema.
The idea is taken from Swarming TaskRequest entity:
https://code.google.com/p/swarming/source/browse/appengine/swarming/server/task_request.py#329
"""
# ndb library sometimes silently ignores memcache errors
# => memcache is not synchronized with datastore
# => a build never finishes from the app code perspective
# => builder is stuck for days.
# We workaround this problem by setting a timeout.
_memcache_timeout = 600 # 10m
# Stores the build proto. The primary property of this entity.
# Majority of the other properties are either derivatives of this field or
# legacy.
#
# Does not include:
# output.properties: see BuildOutputProperties
# steps: see BuildSteps.
# tags: stored in tags attribute, because we have to index them anyway.
# input.properties: see BuildInputProperties.
# CAVEAT: field input.properties does exist during build creation, and
# moved into BuildInputProperties right before initial datastore.put.
# infra: see BuildInfra.
# CAVEAT: field infra does exist during build creation, and moved into
# BuildInfra right before initial datastore.put.
#
# Transition period: proto is either None or complete, i.e. created by
# creation.py or fix_builds.py.
proto = datastore_utils.ProtobufProperty(build_pb2.Build)
# A randomly generated key associated with the created swarming task.
# Embedded in a build token provided to a swarming task in secret bytes.
# Needed in case Buildbucket unintentionally creates multiple swarming tasks
# associated with the build.
# Populated by swarming.py on swarming task creation.
# A part of the message in build token (tokens.py) required for UpdateBuild
# api.
swarming_task_key = ndb.StringProperty(indexed=False)
# == proto-derived properties ================================================
#
# These properties are derived from "proto" properties.
# They are used to index builds.
status = ndb.ComputedProperty(lambda self: self.proto.status,
name='status_v2')
@property
def is_ended(self): # pragma: no cover
return is_terminal_status(self.proto.status)
incomplete = ndb.ComputedProperty(lambda self: not self.is_ended)
# ID of the LUCI project to which this build belongs.
project = ndb.ComputedProperty(lambda self: self.proto.builder.project)
# Indexed string "<project_id>/<bucket_name>".
# Example: "chromium/try".
# Prefix "luci.<project_id>." is stripped from bucket name.
bucket_id = ndb.ComputedProperty(lambda self: config.format_bucket_id(
self.proto.builder.project, self.proto.builder.bucket))
# Indexed string "<project_id>/<bucket_name>/<builder_name>".
# Example: "chromium/try/linux-rel".
# Prefix "luci.<project_id>." is stripped from bucket name.
builder_id = ndb.ComputedProperty(
lambda self: config.builder_id_string(self.proto.builder))
canary = ndb.ComputedProperty(lambda self: self.proto.canary)
# Value of proto.create_time.
# Making this property computed is not-entirely trivial because
# ComputedProperty saves it as int, as opposed to datetime.datetime.
# TODO(nodir): remove usages of create_time indices, rely on build id ordering
# instead.
create_time = ndb.DateTimeProperty()
# A list of colon-separated key-value pairs. Indexed.
# Used to populate tags in builds_to_protos_async, if requested.
tags = ndb.StringProperty(repeated=True)
# If True, the build won't affect monitoring and won't be surfaced in
# search results unless explicitly requested.
experimental = ndb.ComputedProperty(
lambda self: self.proto.input.experimental)
# Value of proto.created_by.
# Making this property computed is not-entirely trivial because
# ComputedProperty saves it as string, but IdentityProperty stores it
# as a blob property.
created_by = auth.IdentityProperty()
is_luci = ndb.BooleanProperty()
@property
def is_ended(self): # pragma: no cover
return self.proto.status not in (common_pb2.STATUS_UNSPECIFIED,
common_pb2.SCHEDULED,
common_pb2.STARTED)
# == Legacy properties =======================================================
status_legacy = msgprop.EnumProperty(BuildStatus,
default=BuildStatus.SCHEDULED,
name='status')
status_changed_time = ndb.DateTimeProperty(auto_now_add=True)
# immutable arbitrary build parameters.
parameters = datastore_utils.DeterministicJsonProperty(json_type=dict)
# PubSub message parameters for build status change notifications.
# TODO(nodir): replace with notification_pb2.NotificationConfig.
pubsub_callback = ndb.StructuredProperty(PubSubCallback, indexed=False)
# id of the original build that this build was derived from.
retry_of = ndb.IntegerProperty()
# a URL to a build-system-specific build, viewable by a human.
url = ndb.StringProperty(indexed=False)
# V1 status properties. Computed by _pre_put_hook.
result = msgprop.EnumProperty(BuildResult)
result_details = datastore_utils.DeterministicJsonProperty(json_type=dict)
cancelation_reason = msgprop.EnumProperty(CancelationReason)
failure_reason = msgprop.EnumProperty(FailureReason)
# Lease-time properties.
# TODO(nodir): move lease to a separate entity under Build.
# It would be more efficient.
# current lease expiration date.
# The moment the build is leased, |lease_expiration_date| is set to
# (utcnow + lease_duration).
lease_expiration_date = ndb.DateTimeProperty()
# None if build is not leased, otherwise a random value.
# Changes every time a build is leased. Can be used to verify that a client
# is the leaseholder.
lease_key = ndb.IntegerProperty(indexed=False)
# True if the build is currently leased. Otherwise False
is_leased = ndb.ComputedProperty(lambda self: self.lease_key is not None)
leasee = auth.IdentityProperty()
never_leased = ndb.BooleanProperty()
# ============================================================================
def _pre_put_hook(self):
"""Checks Build invariants before putting."""
super(Build, self)._pre_put_hook()
config.validate_project_id(self.proto.builder.project)
config.validate_bucket_name(self.proto.builder.bucket)
self.update_v1_status_fields()
self.proto.update_time.FromDatetime(utils.utcnow())
is_started = self.proto.status == common_pb2.STARTED
is_ended = self.is_ended
is_leased = self.lease_key is not None
assert not (is_ended and is_leased)
assert (self.lease_expiration_date is not None) == is_leased
assert (self.leasee is not None) == is_leased
tag_delm = buildtags.DELIMITER
assert not self.tags or all(tag_delm in t for t in self.tags)
assert self.proto.HasField('create_time')
assert self.proto.HasField('end_time') == is_ended
assert not is_started or self.proto.HasField('start_time')
def _ts_less(ts1, ts2):
return ts1.seconds and ts2.seconds and ts1.ToDatetime(
) < ts2.ToDatetime()
assert not _ts_less(self.proto.start_time, self.proto.create_time)
assert not _ts_less(self.proto.end_time, self.proto.create_time)
assert not _ts_less(self.proto.end_time, self.proto.start_time)
self.tags = sorted(set(self.tags))
def update_v1_status_fields(self):
"""Updates V1 status fields."""
self.status_legacy = None
self.result = None
self.failure_reason = None
self.cancelation_reason = None
status = self.proto.status
if status == common_pb2.SCHEDULED:
self.status_legacy = BuildStatus.SCHEDULED
elif status == common_pb2.STARTED:
self.status_legacy = BuildStatus.STARTED
elif status == common_pb2.SUCCESS:
self.status_legacy = BuildStatus.COMPLETED
self.result = BuildResult.SUCCESS
elif status == common_pb2.FAILURE:
self.status_legacy = BuildStatus.COMPLETED
self.result = BuildResult.FAILURE
self.failure_reason = FailureReason.BUILD_FAILURE
elif status == common_pb2.INFRA_FAILURE:
self.status_legacy = BuildStatus.COMPLETED
if self.proto.status_details.HasField('timeout'):
self.result = BuildResult.CANCELED
self.cancelation_reason = CancelationReason.TIMEOUT
else:
self.result = BuildResult.FAILURE
self.failure_reason = FailureReason.INFRA_FAILURE
elif status == common_pb2.CANCELED:
self.status_legacy = BuildStatus.COMPLETED
self.result = BuildResult.CANCELED
self.cancelation_reason = CancelationReason.CANCELED_EXPLICITLY
else: # pragma: no cover
assert False, status
def regenerate_lease_key(self):
"""Changes lease key to a different random int."""
while True:
new_key = random.randint(0, 1 << 31)
if new_key != self.lease_key: # pragma: no branch
self.lease_key = new_key
break
def clear_lease(self): # pragma: no cover
"""Clears build's lease attributes."""
self.lease_key = None
self.lease_expiration_date = None
self.leasee = None
def tags_to_protos(self, dest):
"""Adds non-hidden self.tags to a repeated StringPair container."""
for t in self.tags:
k, v = buildtags.parse(t)
if k not in buildtags.HIDDEN_TAG_KEYS:
dest.add(key=k, value=v)
class TaskRequest(ndb.Model):
"""Contains a user request.
Key id is a decreasing integer based on time since utils.EPOCH plus some
randomness on lower order bits. See _new_request_key() for the complete gory
details.
There is also "old style keys" which inherit from a fake root entity
TaskRequestShard.
TODO(maruel): Remove support 2015-10-01 once entities are deleted.
This model is immutable.
"""
# Time this request was registered. It is set manually instead of using
# auto_now_add=True so that expiration_ts can be set very precisely relative
# to this property.
created_ts = ndb.DateTimeProperty(required=True)
# The name for this task request. It's only for description.
name = ndb.StringProperty(required=True)
# Authenticated client that triggered this task.
authenticated = auth.IdentityProperty()
# Which user to blame for this task.
user = ndb.StringProperty(default='')
# The actual properties are embedded in this model.
properties = ndb.LocalStructuredProperty(TaskProperties,
compressed=True,
required=True)
# Priority of the task to be run. A lower number is higher priority, thus will
# preempt requests with lower priority (higher numbers).
priority = ndb.IntegerProperty(indexed=False,
validator=_validate_priority,
required=True)
# If the task request is not scheduled by this moment, it will be aborted by a
# cron job. It is saved instead of scheduling_expiration_secs so finding
# expired jobs is a simple query.
expiration_ts = ndb.DateTimeProperty(indexed=True,
validator=_validate_expiration,
required=True)
# Tags that specify the category of the task.
tags = ndb.StringProperty(repeated=True, validator=_validate_tags)
# Set when a task (the parent) reentrantly create swarming tasks. Must be set
# to a valid task_id pointing to a TaskRunResult or be None.
parent_task_id = ndb.StringProperty(validator=_validate_task_run_id)
# PubSub topic to send task completion notification to.
pubsub_topic = ndb.StringProperty(indexed=False)
# Secret token to send as 'auth_token' attribute with PubSub messages.
pubsub_auth_token = ndb.StringProperty(indexed=False)
# Data to send in 'userdata' field of PubSub messages.
pubsub_userdata = ndb.StringProperty(indexed=False)
@property
def task_id(self):
"""Returns the TaskResultSummary packed id, not the task request key."""
return task_pack.pack_result_summary_key(
task_pack.request_key_to_result_summary_key(self.key))
@property
def expiration_secs(self):
"""Reconstructs this value from expiration_ts and created_ts. Integer."""
return int((self.expiration_ts - self.created_ts).total_seconds())
def to_dict(self):
"""Converts properties_hash to hex so it is json serializable."""
out = super(TaskRequest, self).to_dict(exclude=['pubsub_auth_token'])
properties_hash = self.properties.properties_hash
out['properties_hash'] = (properties_hash.encode('hex')
if properties_hash else None)
return out
def _pre_put_hook(self):
"""Adds automatic tags."""
super(TaskRequest, self)._pre_put_hook()
self.properties._pre_put_hook()
if self.properties.is_terminate:
if not self.priority == 0:
raise datastore_errors.BadValueError(
'terminate request must be priority 0')
elif self.priority == 0:
raise datastore_errors.BadValueError(
'priority 0 can only be used for terminate request')
self.tags.append('priority:%s' % self.priority)
self.tags.append('user:%s' % self.user)
for key, value in self.properties.dimensions.iteritems():
self.tags.append('%s:%s' % (key, value))
self.tags = sorted(set(self.tags))
if (self.pubsub_topic and
not pubsub.validate_full_name(self.pubsub_topic, 'topics')):
raise datastore_errors.BadValueError('bad pubsub topic name - %s' %
self.pubsub_topic)
if self.pubsub_auth_token and not self.pubsub_topic:
raise datastore_errors.BadValueError(
'pubsub_auth_token requires pubsub_topic')
if self.pubsub_userdata and not self.pubsub_topic:
raise datastore_errors.BadValueError(
'pubsub_userdata requires pubsub_topic')
class GroupImporterConfig(ndb.Model):
"""Singleton entity with group importer configuration JSON."""
config = ndb.JsonProperty()
modified_by = auth.IdentityProperty(indexed=False)
modified_ts = ndb.DateTimeProperty(auto_now=True, indexed=False)
