class PackageACLBody(ndb.Model): """Shared by PackageACL and PackageACLRevision. Doesn't actually exist in the datastore by itself. Only inherited from. """ # Users granted the given role directly. Often only one account should be # granted some role (e.g. a role account should be WRITER). It is annoying to # manage one-account groups for cases like this. users = auth.IdentityProperty(indexed=False, repeated=True) # Groups granted the given role. groups = ndb.StringProperty(indexed=False, repeated=True) # Who made the last change. modified_by = auth.IdentityProperty(indexed=True) # When the last change was made. modified_ts = ndb.DateTimeProperty(indexed=True)
class InstanceTag(ndb.Model): """Single tag of some package instance. ID is hex-encoded SHA1 of the tag. Parent entity is corresponding PackageInstance. The tag can't be made a part of the key because total key length (including class names, all parent keys) is limited to 500 characters. Tags can be pretty long. Tags are separate entities (rather than repeated field in PackageInstance) for two reasons: * There can be many tags attached to an instance (e.g. 'git_revision:...' tags for a package that doesn't change between revisions). * PackageInstance entity is fetched pretty often, no need to pull all tags all the time. """ # Disable useless in-memory per-request cache. It's harmful in tests. _use_cache = False # The tag itself, as key:value string. tag = ndb.StringProperty() # Who added this tag. registered_by = auth.IdentityProperty() # When the tag was added. registered_ts = ndb.DateTimeProperty() @property def package_name(self): """Name of the package this tag belongs to.""" return self.key.parent().parent().string_id() @property def instance_id(self): """Package instance ID this tag belongs to.""" return self.key.parent().string_id()
class GroupImporterConfig(ndb.Model): """Singleton entity with group importer configuration JSON.""" config_proto = ndb.TextProperty() config_revision = ndb.JsonProperty( ) # see config.py, _update_imports_config modified_by = auth.IdentityProperty(indexed=False) modified_ts = ndb.DateTimeProperty(auto_now=True, indexed=False)
class PackageInstance(ndb.Model): """Represents some uploaded package instance. ID is package instance ID (SHA1 hex digest of package body). Parent entity is Package(id=package_name). """ # Disable useless in-memory per-request cache. It's harmful in tests. _use_cache = False # Who registered the instance. registered_by = auth.IdentityProperty() # When the instance was registered. registered_ts = ndb.DateTimeProperty() # Names of processors scheduled for an instance or currently running. processors_pending = ndb.StringProperty(repeated=True) # Names of processors that successfully finished the processing. processors_success = ndb.StringProperty(repeated=True) # Names of processors that returned fatal error. processors_failure = ndb.StringProperty(repeated=True) @property def package_name(self): """Name of the package this instance belongs to.""" return self.key.parent().string_id() @property def instance_id(self): """Package instance ID (SHA1 of package file content).""" return self.key.string_id()
class VersionedFile(ndb.Model): """Versionned entity. Root is ROOT_MODEL. id is datastore_utils.HIGH_KEY_ID - version number. """ created_ts = ndb.DateTimeProperty(indexed=False, auto_now_add=True) who = auth.IdentityProperty(indexed=False) content = ndb.BlobProperty(compressed=True) ROOT_MODEL = datastore_utils.get_versioned_root_model('VersionedFileRoot') @property def version(self): return datastore_utils.HIGH_KEY_ID - self.key.integer_id() @classmethod def fetch(cls, name): """Returns the current version of the instance.""" return datastore_utils.get_versioned_most_recent( cls, cls._gen_root_key(name)) def store(self, name): """Stores a new version of the instance.""" # Create an incomplete key. self.key = ndb.Key(self.__class__, None, parent=self._gen_root_key(name)) self.who = auth.get_current_identity() return datastore_utils.store_new_version(self, self.ROOT_MODEL) @classmethod def _gen_root_key(cls, name): return ndb.Key(cls.ROOT_MODEL, name)
class Package(ndb.Model): """Entity root for PackageInstance entities for some particular package. Id is a package name. """ # Who registered the package. registered_by = auth.IdentityProperty() # When the package was registered. registered_ts = ndb.DateTimeProperty() @property def package_name(self): """Name of the package.""" return self.key.string_id()
class PackageRef(ndb.Model): """A named reference to some instance ID. ID is a reference name, parent entity is corresponding Package. """ # PackageInstance the ref points to. instance_id = ndb.StringProperty() # Who added or moved this reference. modified_by = auth.IdentityProperty() # When the reference was created or moved. modified_ts = ndb.DateTimeProperty() @property def package_name(self): """Name of the package this ref belongs to.""" return self.key.parent().string_id()
class UploadSession(ndb.Model): """Some pending upload operation. Entity id is autogenerated by the datastore. No parent entity. """ # Upload session never existed or already expired. STATUS_MISSING = 0 # Client is still uploading the file. STATUS_UPLOADING = 1 # Server is verifying the hash of the uploaded file. STATUS_VERIFYING = 2 # The file is in the store and visible by all clients. Final state. STATUS_PUBLISHED = 3 # Some other unexpected fatal error happened. STATUS_ERROR = 4 # Hash algorithm to use to verify the content. hash_algo = ndb.StringProperty(required=True, indexed=False) # Expected hex digest of the file. hash_digest = ndb.StringProperty(required=True, indexed=False) # Full path in the GS to the temporary drop file that the client upload to. temp_gs_location = ndb.TextProperty(required=True) # Full path in the GS where to store the verified file. final_gs_location = ndb.TextProperty(required=True) # URL to put file content too. upload_url = ndb.TextProperty(required=True) # Status of the upload operation. See STATUS_* constants. status = ndb.IntegerProperty(required=True, choices=[ STATUS_ERROR, STATUS_MISSING, STATUS_PUBLISHED, STATUS_UPLOADING, STATUS_VERIFYING, ]) # For STATUS_ERROR may contain an error message. error_message = ndb.TextProperty(required=False) # Who started the upload. created_by = auth.IdentityProperty(required=True) # When the entity was created. created_ts = ndb.DateTimeProperty(required=True, auto_now_add=True)
class Package(ndb.Model): """Entity root for PackageInstance entities for some particular package. Id is a package name. """ # Disable useless in-memory per-request cache. It's harmful in tests. _use_cache = False # Who registered the package. registered_by = auth.IdentityProperty() # When the package was registered. registered_ts = ndb.DateTimeProperty() # If True, the package won't show up in listings, see 'list_packages'. hidden = ndb.BooleanProperty() @property def package_name(self): """Name of the package.""" return self.key.string_id()
class PackageRef(ndb.Model): """A named reference to some instance ID. ID is a reference name, parent entity is corresponding Package. """ # Disable useless in-memory per-request cache. It's harmful in tests. _use_cache = False # PackageInstance the ref points to. instance_id = ndb.StringProperty() # Who added or moved this reference. modified_by = auth.IdentityProperty() # When the reference was created or moved. modified_ts = ndb.DateTimeProperty() @property def package_name(self): """Name of the package this ref belongs to.""" return self.key.parent().string_id() @property def ref(self): """Name of the ref (extracted from entity key).""" return self.key.string_id()
class TaskRequest(ndb.Model): """Contains a user request. Key id is a decreasing integer based on time since utils.EPOCH plus some randomness on lower order bits. See new_request_key() for the complete gory details. This model is immutable. """ # Hashing algorithm used to hash TaskProperties to create its key. HASHING_ALGO = hashlib.sha1 # Time this request was registered. It is set manually instead of using # auto_now_add=True so that expiration_ts can be set very precisely relative # to this property. created_ts = ndb.DateTimeProperty(required=True) # The name for this task request. It's only for description. name = ndb.StringProperty(required=True) # Authenticated client that triggered this task. authenticated = auth.IdentityProperty() # Which user to blame for this task. Can be arbitrary, not asserted by any # credentials. user = ndb.StringProperty(default='') # Indicates what OAuth2 credentials the task uses when calling other services. # # Possible values are: 'none', 'bot' or <email>. For more information see # swarming_rpcs.NewTaskRequest. # # This property exists only for informational purposes and for indexing. When # actually getting an OAuth credentials, the original token (stored in hidden # 'service_account_token' field) is used. service_account = ndb.StringProperty(validator=_validate_service_account) # The delegation token passed via 'service_account_token' when creating a new # task that should by run with the authority of some service account. Can also # be set to literal 'none' or 'bot'. See swarming_rpcs.NewTaskRequest for more # information. # # This property never shows up in UI or API responses. service_account_token = ndb.BlobProperty() # The actual properties are embedded in this model. properties = ndb.LocalStructuredProperty( TaskProperties, compressed=True, required=True) # Priority of the task to be run. A lower number is higher priority, thus will # preempt requests with lower priority (higher numbers). priority = ndb.IntegerProperty( indexed=False, validator=_validate_priority, required=True) # If the task request is not scheduled by this moment, it will be aborted by a # cron job. It is saved instead of scheduling_expiration_secs so finding # expired jobs is a simple query. expiration_ts = ndb.DateTimeProperty( indexed=True, validator=_validate_expiration, required=True) # Tags that specify the category of the task. tags = ndb.StringProperty(repeated=True, validator=_validate_tags) # Set when a task (the parent) reentrantly create swarming tasks. Must be set # to a valid task_id pointing to a TaskRunResult or be None. parent_task_id = ndb.StringProperty(validator=_validate_task_run_id) # PubSub topic to send task completion notification to. pubsub_topic = ndb.StringProperty(indexed=False) # Secret token to send as 'auth_token' attribute with PubSub messages. pubsub_auth_token = ndb.StringProperty(indexed=False) # Data to send in 'userdata' field of PubSub messages. pubsub_userdata = ndb.StringProperty(indexed=False) # This stores the computed properties_hash for this Task's properties object. # It is set in init_new_request. It is None for non-idempotent tasks. properties_hash = ndb.BlobProperty(indexed=False) @property def task_id(self): """Returns the TaskResultSummary packed id, not the task request key.""" return task_pack.pack_result_summary_key( task_pack.request_key_to_result_summary_key(self.key)) @property def expiration_secs(self): """Reconstructs this value from expiration_ts and created_ts. Integer.""" return int((self.expiration_ts - self.created_ts).total_seconds()) @property def has_access(self): """Returns True if the current user has read-write access to this request. This is used for: * Read access: ability to read the task info and logs. * Write access: ability to cancel the task. Warning: This function looks at the current Authentication context. """ return ( acl.is_privileged_user() or self.authenticated == auth.get_current_identity()) def to_dict(self): """Converts properties_hash to hex so it is json serializable.""" # to_dict() doesn't recurse correctly into ndb.LocalStructuredProperty! out = super(TaskRequest, self).to_dict( exclude=['pubsub_auth_token', 'properties', 'service_account_token']) out['properties'] = self.properties.to_dict() properties_hash = out['properties_hash'] out['properties_hash'] = ( properties_hash.encode('hex') if properties_hash else None) return out def _pre_put_hook(self): """Adds automatic tags.""" super(TaskRequest, self)._pre_put_hook() self.properties._pre_put_hook() if self.properties.is_terminate: if not self.priority == 0: raise datastore_errors.BadValueError( 'terminate request must be priority 0') elif self.priority == 0: raise datastore_errors.BadValueError( 'priority 0 can only be used for terminate request') if (self.pubsub_topic and not pubsub.validate_full_name(self.pubsub_topic, 'topics')): raise datastore_errors.BadValueError( 'bad pubsub topic name - %s' % self.pubsub_topic) if self.pubsub_auth_token and not self.pubsub_topic: raise datastore_errors.BadValueError( 'pubsub_auth_token requires pubsub_topic') if self.pubsub_userdata and not self.pubsub_topic: raise datastore_errors.BadValueError( 'pubsub_userdata requires pubsub_topic')
class LeaseRequest(ndb.Model): """Datastore representation of a LeaseRequest. Key: id: Hash of the client + client-generated request ID which issued the original rpc_messages.LeaseRequest instance. Used for easy deduplication. kind: LeaseRequest. This root entity does not reference any parents. """ # DateTime indicating original datastore write time. created_ts = ndb.DateTimeProperty(auto_now_add=True) # Checksum of the rpc_messages.LeaseRequest instance. Used to compare incoming # LeaseRequets for deduplication. deduplication_checksum = ndb.StringProperty(required=True, indexed=False) # ID of the CatalogMachineEntry provided for this lease. machine_id = ndb.StringProperty() # auth.model.Identity of the issuer of the original request. owner = auth.IdentityProperty(required=True) # Element of LeaseRequestStates giving the state of this request. state = ndb.StringProperty(choices=LeaseRequestStates, required=True) # rpc_messages.LeaseRequest instance representing the original request. request = msgprop.MessageProperty(rpc_messages.LeaseRequest, required=True) # rpc_messages.LeaseResponse instance representing the current response. # This field will be updated as the request is being processed. response = msgprop.MessageProperty(rpc_messages.LeaseResponse) @classmethod def compute_deduplication_checksum(cls, request): """Computes the deduplication checksum for the given request. Args: request: The rpc_messages.LeaseRequest instance to deduplicate. Returns: The deduplication checksum. """ return hashlib.sha1(protobuf.encode_message(request)).hexdigest() @classmethod def generate_key(cls, user, request): """Generates the key for the given request initiated by the given user. Args: user: An auth.model.Identity instance representing the requester. request: The rpc_messages.LeaseRequest sent by the user. Returns: An ndb.Key instance. """ # Enforces per-user request ID uniqueness return ndb.Key( cls, hashlib.sha1('%s\0%s' % (user, request.request_id)).hexdigest(), ) @classmethod def query_untriaged(cls): """Queries for untriaged LeaseRequests. Yields: Untriaged LeaseRequests in no guaranteed order. """ for request in cls.query(cls.state == LeaseRequestStates.UNTRIAGED): yield request
class TaskRequest(ndb.Model): """Contains a user request. Key id is a decreasing integer based on time since utils.EPOCH plus some randomness on lower order bits. See _new_request_key() for the complete gory details. There is also "old style keys" which inherit from a fake root entity TaskRequestShard. TODO(maruel): Remove support 2015-02-01. This model is immutable. """ # Time this request was registered. It is set manually instead of using # auto_now_add=True so that expiration_ts can be set very precisely relative # to this property. created_ts = ndb.DateTimeProperty(required=True) # The name for this task request. It's only for description. name = ndb.StringProperty(required=True) # Authenticated client that triggered this task. authenticated = auth.IdentityProperty() # Which user to blame for this task. user = ndb.StringProperty(default='') # The actual properties are embedded in this model. properties = ndb.LocalStructuredProperty( TaskProperties, compressed=True, required=True) # Priority of the task to be run. A lower number is higher priority, thus will # preempt requests with lower priority (higher numbers). priority = ndb.IntegerProperty( indexed=False, validator=_validate_priority, required=True) # If the task request is not scheduled by this moment, it will be aborted by a # cron job. It is saved instead of scheduling_expiration_secs so finding # expired jobs is a simple query. expiration_ts = ndb.DateTimeProperty( indexed=True, validator=_validate_expiration, required=True) # Tags that specify the category of the task. tags = ndb.StringProperty(repeated=True, validator=_validate_tags) # Set when a task (the parent) reentrantly create swarming tasks. Must be set # to a valid task_id pointing to a TaskRunResult or be None. parent_task_id = ndb.StringProperty(validator=_validate_task_run_id) @property def scheduling_expiration_secs(self): """Reconstructs this value from expiration_ts and created_ts.""" return (self.expiration_ts - self.created_ts).total_seconds() def to_dict(self): """Converts properties_hash to hex so it is json serializable.""" out = super(TaskRequest, self).to_dict() properties_hash = self.properties.properties_hash out['properties_hash'] = ( properties_hash.encode('hex') if properties_hash else None) return out def _pre_put_hook(self): """Adds automatic tags.""" super(TaskRequest, self)._pre_put_hook() self.tags.append('priority:%s' % self.priority) self.tags.append('user:%s' % self.user) for key, value in self.properties.dimensions.iteritems(): self.tags.append('%s:%s' % (key, value)) self.tags = sorted(set(self.tags))
class GlobalConfig(ndb.Model): """Singleton entity with the global configuration of the service. All changes are stored in the revision log. """ # When this revision of configuration was created. updated_ts = ndb.DateTimeProperty(indexed=False, auto_now_add=True) # Who created this revision of configuration. updated_by = auth.IdentityProperty(indexed=False) @classmethod def cached(cls): """Fetches config entry from local cache or datastore. Bootstraps it if missing. May return slightly stale data but in most cases doesn't do any RPCs. Should be used for read-only access to config. """ # @utils.cache_with_expiration can't be used directly with 'cached' since it # will be applied to base class method, not a concrete implementation # specific to a subclass. So build new class specific fetcher on the fly on # a first attempt (it's not a big deal if it happens concurrently in MT # environment, last one wins). Same can be achieved with metaclasses, but no # one likes metaclasses. if not cls._config_fetcher: @utils.cache_with_expiration(expiration_sec=60) def config_fetcher(): conf = cls.fetch() if not conf: conf = cls() conf.set_defaults() conf.store(updated_by=auth.get_service_self_identity()) return conf cls._config_fetcher = staticmethod(config_fetcher) return cls._config_fetcher() @classmethod def clear_cache(cls): """Clears the cache of .cached(). So the next call to .cached() returns the fresh instance from ndb. """ if cls._config_fetcher: utils.clear_cache(cls._config_fetcher) @classmethod def fetch(cls): """Returns the current up-to-date version of the config entity. Always fetches it from datastore. May return None if missing. """ return datastore_utils.get_versioned_most_recent( cls, cls._get_root_key()) def store(self, updated_by=None): """Stores a new version of the config entity.""" # Create an incomplete key, to be completed by 'store_new_version'. self.key = ndb.Key(self.__class__, None, parent=self._get_root_key()) self.updated_by = updated_by or auth.get_current_identity() self.updated_ts = utils.utcnow() return datastore_utils.store_new_version(self, self._get_root_model()) def modify(self, updated_by=None, **kwargs): """Applies |kwargs| dict to the entity and stores the entity if changed.""" dirty = False for k, v in kwargs.iteritems(): assert k in self._properties, k if getattr(self, k) != v: setattr(self, k, v) dirty = True if dirty: self.store(updated_by=updated_by) return dirty def set_defaults(self): """Fills in default values for empty config. Implemented by subclasses.""" ### Private stuff. _config_fetcher = None @classmethod def _get_root_model(cls): return datastore_utils.get_versioned_root_model('%sRoot' % cls.__name__) @classmethod def _get_root_key(cls): return ndb.Key(cls._get_root_model(), 1)
class ConfigSettings(config.GlobalConfig): # Hostname of the config service. service_hostname = ndb.StringProperty(indexed=False) # Identity account used by config service. trusted_config_account = auth.IdentityProperty(indexed=False)
class GlobalConfig(ndb.Model): """Singleton entity with the global configuration of the service. All changes are stored in the revision log. """ # When this revision of configuration was created. updated_ts = ndb.DateTimeProperty(indexed=False, auto_now_add=True) # Who created this revision of configuration. updated_by = auth.IdentityProperty(indexed=False) @classmethod def cached_async(cls): """Fetches config entry from local cache or datastore. Bootstraps it if missing. May return slightly stale data but in most cases doesn't do any RPCs. Should be used for read-only access to config. """ # Build new class-specific fetcher function with cache on the fly on # the first attempt (it's not a big deal if it happens concurrently in MT # environment, last one wins). Same can be achieved with metaclasses, but no # one likes metaclasses. if not cls._config_fetcher_async: @ndb.tasklet def fetcher(): with fetcher.cache_lock: expiry = fetcher.cache_expiry if expiry is not None and utils.utcnow() < expiry: raise ndb.Return(fetcher.cache_value) # Do not lock while yielding, it would cause deadlock. # Also do not cache a future, it might cross ndb context boundary. # If there is no cached value, multiple concurrent requests will make # multiple RPCs, but as soon as one of them updates cache, subsequent # requests will use the cached value, for a minute. conf = yield cls.fetch_async() if not conf: conf = cls() conf.set_defaults() yield conf.store_async( updated_by=auth.get_service_self_identity()) with fetcher.cache_lock: fetcher.cache_expiry = utils.utcnow() + datetime.timedelta( minutes=1) fetcher.cache_value = conf raise ndb.Return(conf) fetcher.cache_lock = threading.Lock() fetcher.cache_expiry = None fetcher.cache_value = None cls._config_fetcher_async = staticmethod(fetcher) return cls._config_fetcher_async() cached = utils.sync_of(cached_async) @classmethod def clear_cache(cls): """Clears the cache of .cached(). So the next call to .cached() returns the fresh instance from ndb. """ if cls._config_fetcher_async: cls._config_fetcher_async.cache_expiry = None @classmethod def fetch_async(cls): """Returns the current up-to-date version of the config entity. Always fetches it from datastore. May return None if missing. """ return datastore_utils.get_versioned_most_recent_async( cls, cls._get_root_key()) fetch = utils.sync_of(fetch_async) def store_async(self, updated_by=None): """Stores a new version of the config entity.""" # Create an incomplete key, to be completed by 'store_new_version'. self.key = ndb.Key(self.__class__, None, parent=self._get_root_key()) self.updated_by = updated_by or auth.get_current_identity() self.updated_ts = utils.utcnow() return datastore_utils.store_new_version_async(self, self._get_root_model()) store = utils.sync_of(store_async) def modify(self, updated_by=None, **kwargs): """Applies |kwargs| dict to the entity and stores the entity if changed.""" dirty = False for k, v in kwargs.iteritems(): assert k in self._properties, k if getattr(self, k) != v: setattr(self, k, v) dirty = True if dirty: self.store(updated_by=updated_by) return dirty def set_defaults(self): """Fills in default values for empty config. Implemented by subclasses.""" ### Private stuff. _config_fetcher_async = None @classmethod def _get_root_model(cls): return datastore_utils.get_versioned_root_model('%sRoot' % cls.__name__) @classmethod def _get_root_key(cls): return ndb.Key(cls._get_root_model(), 1)
class Build(ndb.Model): """Describes a build. Build key: Build keys are autogenerated, monotonically decreasing integers. That is, when sorted by key, new builds are first. Build has no parent. Build id is a 64 bits integer represented as a string to the user. - 1 highest order bit is set to 0 to keep value positive. - 43 bits are 43 lower bits of bitwise-inverted time since BEGINING_OF_THE_WORLD at 1ms resolution. It is good for 2**43 / 365.3 / 24 / 60 / 60 / 1000 = 278 years or 2010 + 278 = year 2288. - 16 bits are set to a random value. Assuming an instance is internally consistent with itself, it can ensure to not reuse the same 16 bits in two consecutive requests and/or throttle itself to one request per millisecond. Using random value reduces to 2**-15 the probability of collision on exact same timestamp at 1ms resolution, so a maximum theoretical rate of 65536000 requests/sec but an effective rate in the range of ~64k qps without much transaction conflicts. We should be fine. - 4 bits are 0. This is to represent the 'version' of the entity schema. The idea is taken from Swarming TaskRequest entity: https://code.google.com/p/swarming/source/browse/appengine/swarming/server/task_request.py#329 """ status = msgprop.EnumProperty(BuildStatus, default=BuildStatus.SCHEDULED) status_changed_time = ndb.DateTimeProperty(auto_now_add=True) update_time = ndb.DateTimeProperty(auto_now=True) # Creation time attributes. create_time = ndb.DateTimeProperty(auto_now_add=True) created_by = auth.IdentityProperty() # a generic way to distinguish builds. # Different buckets have different permissions. bucket = ndb.StringProperty(required=True) # a list of tags, where each tag is a string # with ":" symbol. The first occurrence of ":" splits tag name and tag # value. Contains only tags specified by the build request. Old Build # entities do not have this field. initial_tags = ndb.StringProperty(repeated=True, indexed=False) # superset of initial_tags. May contain auto-added tags. tags = ndb.StringProperty(repeated=True) # immutable arbitrary build parameters. parameters = ndb.JsonProperty() # PubSub message parameters for build status change notifications. pubsub_callback = ndb.StructuredProperty(PubSubCallback, indexed=False) # id of the original build that this build was derived from. retry_of = ndb.IntegerProperty() # Lease-time attributes. # current lease expiration date. # The moment the build is leased, |lease_expiration_date| is set to # (utcnow + lease_duration). lease_expiration_date = ndb.DateTimeProperty() # None if build is not leased, otherwise a random value. # Changes every time a build is leased. Can be used to verify that a client # is the leaseholder. lease_key = ndb.IntegerProperty(indexed=False) # True if the build is currently leased. Otherwise False is_leased = ndb.ComputedProperty(lambda self: self.lease_key is not None) leasee = auth.IdentityProperty() never_leased = ndb.BooleanProperty() # Start time attributes. # a URL to a build-system-specific build, viewable by a human. url = ndb.StringProperty(indexed=False) # when the build started. Unknown for old builds. start_time = ndb.DateTimeProperty() # Completion time attributes. complete_time = ndb.DateTimeProperty() result = msgprop.EnumProperty(BuildResult) result_details = ndb.JsonProperty() cancelation_reason = msgprop.EnumProperty(CancelationReason) failure_reason = msgprop.EnumProperty(FailureReason) # Swarming integration swarming_hostname = ndb.StringProperty() swarming_task_id = ndb.StringProperty() def _pre_put_hook(self): """Checks Build invariants before putting.""" super(Build, self)._pre_put_hook() is_started = self.status == BuildStatus.STARTED is_completed = self.status == BuildStatus.COMPLETED is_canceled = self.result == BuildResult.CANCELED is_failure = self.result == BuildResult.FAILURE is_leased = self.lease_key is not None assert (self.result is not None) == is_completed assert (self.cancelation_reason is not None) == is_canceled assert (self.failure_reason is not None) == is_failure assert not (is_completed and is_leased) assert (self.lease_expiration_date is not None) == is_leased assert (self.leasee is not None) == is_leased # no cover due to a bug in coverage (http://stackoverflow.com/a/35325514) assert not self.tags or all(':' in t for t in self.tags) # pragma: no cover assert self.create_time assert (self.complete_time is not None) == is_completed assert not is_started or self.start_time assert not self.start_time or self.start_time >= self.create_time assert not self.complete_time or self.complete_time >= self.create_time assert (not self.complete_time or not self.start_time or self.complete_time >= self.start_time) def regenerate_lease_key(self): """Changes lease key to a different random int.""" while True: new_key = random.randint(0, 1 << 31) if new_key != self.lease_key: # pragma: no branch self.lease_key = new_key break def clear_lease(self): # pragma: no cover """Clears build's lease attributes.""" self.lease_key = None self.lease_expiration_date = None self.leasee = None
class Build(ndb.Model): """Describes a build. Build key: Build keys are autogenerated, monotonically decreasing integers. That is, when sorted by key, new builds are first. Build has no parent. Build id is a 64 bits integer represented as a string to the user. - 1 highest order bit is set to 0 to keep value positive. - 43 bits are 43 lower bits of bitwise-inverted time since BEGINING_OF_THE_WORLD at 1ms resolution. It is good for 2**43 / 365.3 / 24 / 60 / 60 / 1000 = 278 years or 2010 + 278 = year 2288. - 16 bits are set to a random value. Assuming an instance is internally consistent with itself, it can ensure to not reuse the same 16 bits in two consecutive requests and/or throttle itself to one request per millisecond. Using random value reduces to 2**-15 the probability of collision on exact same timestamp at 1ms resolution, so a maximum theoretical rate of 65536000 requests/sec but an effective rate in the range of ~64k qps without much transaction conflicts. We should be fine. - 4 bits are 0. This is to represent the 'version' of the entity schema. The idea is taken from Swarming TaskRequest entity: https://code.google.com/p/swarming/source/browse/appengine/swarming/server/task_request.py#329 Attributes: status (BuildStatus): status of the build. bucket (string): a generic way to distinguish builds. Different buckets have different permissions. tags (list of string): a list of tags, where each tag is a string with ":" symbol. The first occurance of ":" splits tag name and tag value. parameters (dict): immutable arbitrary build parameters. callback (Callback): push task parameters for build status changes. lease_expiration_date (datetime): current lease expiration date. The moment the build is leased, |lease_expiration_date| is set to (utcnow + lease_duration). lease_key (int): None if build is not leased, otherwise a random value. Changes every time a build is leased. Can be used to verify that a client is the leaseholder. is_leased (bool): True if the build is currently leased. Otherwise False url (str): a URL to a build-system-specific build, viewable by a human. result (BuildResult): build result. cancelation_reason (CancelationReason): why the build was canceled. """ status = msgprop.EnumProperty(BuildStatus, default=BuildStatus.SCHEDULED) status_changed_time = ndb.DateTimeProperty(auto_now_add=True) update_time = ndb.DateTimeProperty(auto_now=True) # Creation time attributes. create_time = ndb.DateTimeProperty(auto_now_add=True) created_by = auth.IdentityProperty() bucket = ndb.StringProperty(required=True) tags = ndb.StringProperty(repeated=True) parameters = ndb.JsonProperty() callback = ndb.StructuredProperty(Callback, indexed=False) # Lease-time attributes. lease_expiration_date = ndb.DateTimeProperty() lease_key = ndb.IntegerProperty(indexed=False) is_leased = ndb.ComputedProperty(lambda self: self.lease_key is not None) leasee = auth.IdentityProperty() # Start time attributes. url = ndb.StringProperty(indexed=False) # Completion time attributes. complete_time = ndb.DateTimeProperty() result = msgprop.EnumProperty(BuildResult) result_details = ndb.JsonProperty() cancelation_reason = msgprop.EnumProperty(CancelationReason) failure_reason = msgprop.EnumProperty(FailureReason) def _pre_put_hook(self): """Checks Build invariants before putting.""" super(Build, self)._pre_put_hook() is_completed = self.status == BuildStatus.COMPLETED assert (self.result is not None) == is_completed is_canceled = self.result == BuildResult.CANCELED is_failure = self.result == BuildResult.FAILURE assert (self.cancelation_reason is not None) == is_canceled assert (self.failure_reason is not None) == is_failure is_leased = self.lease_key is not None assert not (is_completed and is_leased) assert (self.lease_expiration_date is not None) == is_leased assert (self.leasee is not None) == is_leased assert not self.tags or all(':' in t for t in self.tags) def regenerate_lease_key(self): """Changes lease key to a different random int.""" while True: new_key = random.randint(0, 1 << 31) if new_key != self.lease_key: # pragma: no branch self.lease_key = new_key break
class Build(ndb.Model): """Describes a build. Build key: Build keys are autogenerated, monotonically decreasing integers. That is, when sorted by key, new builds are first. Build has no parent. Build id is a 64 bits integer represented as a string to the user. - 1 highest order bit is set to 0 to keep value positive. - 43 bits are 43 lower bits of bitwise-inverted time since BEGINING_OF_THE_WORLD at 1ms resolution. It is good for 2**43 / 365.3 / 24 / 60 / 60 / 1000 = 278 years or 2010 + 278 = year 2288. - 16 bits are set to a random value. Assuming an instance is internally consistent with itself, it can ensure to not reuse the same 16 bits in two consecutive requests and/or throttle itself to one request per millisecond. Using random value reduces to 2**-15 the probability of collision on exact same timestamp at 1ms resolution, so a maximum theoretical rate of 65536000 requests/sec but an effective rate in the range of ~64k qps without much transaction conflicts. We should be fine. - 4 bits are 0. This is to represent the 'version' of the entity schema. The idea is taken from Swarming TaskRequest entity: https://code.google.com/p/swarming/source/browse/appengine/swarming/server/task_request.py#329 """ # ndb library sometimes silently ignores memcache errors # => memcache is not synchronized with datastore # => a build never finishes from the app code perspective # => builder is stuck for days. # We workaround this problem by setting a timeout. _memcache_timeout = 600 # 10m # Stores the build proto. The primary property of this entity. # Majority of the other properties are either derivatives of this field or # legacy. # # Does not include: # output.properties: see BuildOutputProperties # steps: see BuildSteps. # tags: stored in tags attribute, because we have to index them anyway. # input.properties: see BuildInputProperties. # CAVEAT: field input.properties does exist during build creation, and # moved into BuildInputProperties right before initial datastore.put. # infra: see BuildInfra. # CAVEAT: field infra does exist during build creation, and moved into # BuildInfra right before initial datastore.put. # # Transition period: proto is either None or complete, i.e. created by # creation.py or fix_builds.py. proto = datastore_utils.ProtobufProperty(build_pb2.Build) # A randomly generated key associated with the created swarming task. # Embedded in a build token provided to a swarming task in secret bytes. # Needed in case Buildbucket unintentionally creates multiple swarming tasks # associated with the build. # Populated by swarming.py on swarming task creation. # A part of the message in build token (tokens.py) required for UpdateBuild # api. swarming_task_key = ndb.StringProperty(indexed=False) # == proto-derived properties ================================================ # # These properties are derived from "proto" properties. # They are used to index builds. status = ndb.ComputedProperty(lambda self: self.proto.status, name='status_v2') @property def is_ended(self): # pragma: no cover return is_terminal_status(self.proto.status) incomplete = ndb.ComputedProperty(lambda self: not self.is_ended) # ID of the LUCI project to which this build belongs. project = ndb.ComputedProperty(lambda self: self.proto.builder.project) # Indexed string "<project_id>/<bucket_name>". # Example: "chromium/try". # Prefix "luci.<project_id>." is stripped from bucket name. bucket_id = ndb.ComputedProperty(lambda self: config.format_bucket_id( self.proto.builder.project, self.proto.builder.bucket)) # Indexed string "<project_id>/<bucket_name>/<builder_name>". # Example: "chromium/try/linux-rel". # Prefix "luci.<project_id>." is stripped from bucket name. builder_id = ndb.ComputedProperty( lambda self: config.builder_id_string(self.proto.builder)) canary = ndb.ComputedProperty(lambda self: self.proto.canary) # Value of proto.create_time. # Making this property computed is not-entirely trivial because # ComputedProperty saves it as int, as opposed to datetime.datetime. # TODO(nodir): remove usages of create_time indices, rely on build id ordering # instead. create_time = ndb.DateTimeProperty() # A list of colon-separated key-value pairs. Indexed. # Used to populate tags in builds_to_protos_async, if requested. tags = ndb.StringProperty(repeated=True) # If True, the build won't affect monitoring and won't be surfaced in # search results unless explicitly requested. experimental = ndb.ComputedProperty( lambda self: self.proto.input.experimental) # Value of proto.created_by. # Making this property computed is not-entirely trivial because # ComputedProperty saves it as string, but IdentityProperty stores it # as a blob property. created_by = auth.IdentityProperty() is_luci = ndb.BooleanProperty() @property def is_ended(self): # pragma: no cover return self.proto.status not in (common_pb2.STATUS_UNSPECIFIED, common_pb2.SCHEDULED, common_pb2.STARTED) # == Legacy properties ======================================================= status_legacy = msgprop.EnumProperty(BuildStatus, default=BuildStatus.SCHEDULED, name='status') status_changed_time = ndb.DateTimeProperty(auto_now_add=True) # immutable arbitrary build parameters. parameters = datastore_utils.DeterministicJsonProperty(json_type=dict) # PubSub message parameters for build status change notifications. # TODO(nodir): replace with notification_pb2.NotificationConfig. pubsub_callback = ndb.StructuredProperty(PubSubCallback, indexed=False) # id of the original build that this build was derived from. retry_of = ndb.IntegerProperty() # a URL to a build-system-specific build, viewable by a human. url = ndb.StringProperty(indexed=False) # V1 status properties. Computed by _pre_put_hook. result = msgprop.EnumProperty(BuildResult) result_details = datastore_utils.DeterministicJsonProperty(json_type=dict) cancelation_reason = msgprop.EnumProperty(CancelationReason) failure_reason = msgprop.EnumProperty(FailureReason) # Lease-time properties. # TODO(nodir): move lease to a separate entity under Build. # It would be more efficient. # current lease expiration date. # The moment the build is leased, |lease_expiration_date| is set to # (utcnow + lease_duration). lease_expiration_date = ndb.DateTimeProperty() # None if build is not leased, otherwise a random value. # Changes every time a build is leased. Can be used to verify that a client # is the leaseholder. lease_key = ndb.IntegerProperty(indexed=False) # True if the build is currently leased. Otherwise False is_leased = ndb.ComputedProperty(lambda self: self.lease_key is not None) leasee = auth.IdentityProperty() never_leased = ndb.BooleanProperty() # ============================================================================ def _pre_put_hook(self): """Checks Build invariants before putting.""" super(Build, self)._pre_put_hook() config.validate_project_id(self.proto.builder.project) config.validate_bucket_name(self.proto.builder.bucket) self.update_v1_status_fields() self.proto.update_time.FromDatetime(utils.utcnow()) is_started = self.proto.status == common_pb2.STARTED is_ended = self.is_ended is_leased = self.lease_key is not None assert not (is_ended and is_leased) assert (self.lease_expiration_date is not None) == is_leased assert (self.leasee is not None) == is_leased tag_delm = buildtags.DELIMITER assert not self.tags or all(tag_delm in t for t in self.tags) assert self.proto.HasField('create_time') assert self.proto.HasField('end_time') == is_ended assert not is_started or self.proto.HasField('start_time') def _ts_less(ts1, ts2): return ts1.seconds and ts2.seconds and ts1.ToDatetime( ) < ts2.ToDatetime() assert not _ts_less(self.proto.start_time, self.proto.create_time) assert not _ts_less(self.proto.end_time, self.proto.create_time) assert not _ts_less(self.proto.end_time, self.proto.start_time) self.tags = sorted(set(self.tags)) def update_v1_status_fields(self): """Updates V1 status fields.""" self.status_legacy = None self.result = None self.failure_reason = None self.cancelation_reason = None status = self.proto.status if status == common_pb2.SCHEDULED: self.status_legacy = BuildStatus.SCHEDULED elif status == common_pb2.STARTED: self.status_legacy = BuildStatus.STARTED elif status == common_pb2.SUCCESS: self.status_legacy = BuildStatus.COMPLETED self.result = BuildResult.SUCCESS elif status == common_pb2.FAILURE: self.status_legacy = BuildStatus.COMPLETED self.result = BuildResult.FAILURE self.failure_reason = FailureReason.BUILD_FAILURE elif status == common_pb2.INFRA_FAILURE: self.status_legacy = BuildStatus.COMPLETED if self.proto.status_details.HasField('timeout'): self.result = BuildResult.CANCELED self.cancelation_reason = CancelationReason.TIMEOUT else: self.result = BuildResult.FAILURE self.failure_reason = FailureReason.INFRA_FAILURE elif status == common_pb2.CANCELED: self.status_legacy = BuildStatus.COMPLETED self.result = BuildResult.CANCELED self.cancelation_reason = CancelationReason.CANCELED_EXPLICITLY else: # pragma: no cover assert False, status def regenerate_lease_key(self): """Changes lease key to a different random int.""" while True: new_key = random.randint(0, 1 << 31) if new_key != self.lease_key: # pragma: no branch self.lease_key = new_key break def clear_lease(self): # pragma: no cover """Clears build's lease attributes.""" self.lease_key = None self.lease_expiration_date = None self.leasee = None def tags_to_protos(self, dest): """Adds non-hidden self.tags to a repeated StringPair container.""" for t in self.tags: k, v = buildtags.parse(t) if k not in buildtags.HIDDEN_TAG_KEYS: dest.add(key=k, value=v)
class TaskRequest(ndb.Model): """Contains a user request. Key id is a decreasing integer based on time since utils.EPOCH plus some randomness on lower order bits. See _new_request_key() for the complete gory details. There is also "old style keys" which inherit from a fake root entity TaskRequestShard. TODO(maruel): Remove support 2015-10-01 once entities are deleted. This model is immutable. """ # Time this request was registered. It is set manually instead of using # auto_now_add=True so that expiration_ts can be set very precisely relative # to this property. created_ts = ndb.DateTimeProperty(required=True) # The name for this task request. It's only for description. name = ndb.StringProperty(required=True) # Authenticated client that triggered this task. authenticated = auth.IdentityProperty() # Which user to blame for this task. user = ndb.StringProperty(default='') # The actual properties are embedded in this model. properties = ndb.LocalStructuredProperty(TaskProperties, compressed=True, required=True) # Priority of the task to be run. A lower number is higher priority, thus will # preempt requests with lower priority (higher numbers). priority = ndb.IntegerProperty(indexed=False, validator=_validate_priority, required=True) # If the task request is not scheduled by this moment, it will be aborted by a # cron job. It is saved instead of scheduling_expiration_secs so finding # expired jobs is a simple query. expiration_ts = ndb.DateTimeProperty(indexed=True, validator=_validate_expiration, required=True) # Tags that specify the category of the task. tags = ndb.StringProperty(repeated=True, validator=_validate_tags) # Set when a task (the parent) reentrantly create swarming tasks. Must be set # to a valid task_id pointing to a TaskRunResult or be None. parent_task_id = ndb.StringProperty(validator=_validate_task_run_id) # PubSub topic to send task completion notification to. pubsub_topic = ndb.StringProperty(indexed=False) # Secret token to send as 'auth_token' attribute with PubSub messages. pubsub_auth_token = ndb.StringProperty(indexed=False) # Data to send in 'userdata' field of PubSub messages. pubsub_userdata = ndb.StringProperty(indexed=False) @property def task_id(self): """Returns the TaskResultSummary packed id, not the task request key.""" return task_pack.pack_result_summary_key( task_pack.request_key_to_result_summary_key(self.key)) @property def expiration_secs(self): """Reconstructs this value from expiration_ts and created_ts. Integer.""" return int((self.expiration_ts - self.created_ts).total_seconds()) def to_dict(self): """Converts properties_hash to hex so it is json serializable.""" out = super(TaskRequest, self).to_dict(exclude=['pubsub_auth_token']) properties_hash = self.properties.properties_hash out['properties_hash'] = (properties_hash.encode('hex') if properties_hash else None) return out def _pre_put_hook(self): """Adds automatic tags.""" super(TaskRequest, self)._pre_put_hook() self.properties._pre_put_hook() if self.properties.is_terminate: if not self.priority == 0: raise datastore_errors.BadValueError( 'terminate request must be priority 0') elif self.priority == 0: raise datastore_errors.BadValueError( 'priority 0 can only be used for terminate request') self.tags.append('priority:%s' % self.priority) self.tags.append('user:%s' % self.user) for key, value in self.properties.dimensions.iteritems(): self.tags.append('%s:%s' % (key, value)) self.tags = sorted(set(self.tags)) if (self.pubsub_topic and not pubsub.validate_full_name(self.pubsub_topic, 'topics')): raise datastore_errors.BadValueError('bad pubsub topic name - %s' % self.pubsub_topic) if self.pubsub_auth_token and not self.pubsub_topic: raise datastore_errors.BadValueError( 'pubsub_auth_token requires pubsub_topic') if self.pubsub_userdata and not self.pubsub_topic: raise datastore_errors.BadValueError( 'pubsub_userdata requires pubsub_topic')
class GroupImporterConfig(ndb.Model): """Singleton entity with group importer configuration JSON.""" config = ndb.JsonProperty() modified_by = auth.IdentityProperty(indexed=False) modified_ts = ndb.DateTimeProperty(auto_now=True, indexed=False)