def _pre_put_hook(self): """Adds automatic tags.""" super(TaskRequest, self)._pre_put_hook() self.properties._pre_put_hook() if self.properties.is_terminate: if not self.priority == 0: raise datastore_errors.BadValueError( 'terminate request must be priority 0') elif self.priority == 0: raise datastore_errors.BadValueError( 'priority 0 can only be used for terminate request') self.tags.append('priority:%s' % self.priority) self.tags.append('user:%s' % self.user) for key, value in self.properties.dimensions.iteritems(): self.tags.append('%s:%s' % (key, value)) self.tags = sorted(set(self.tags)) if (self.pubsub_topic and not pubsub.validate_full_name(self.pubsub_topic, 'topics')): raise datastore_errors.BadValueError('bad pubsub topic name - %s' % self.pubsub_topic) if self.pubsub_auth_token and not self.pubsub_topic: raise datastore_errors.BadValueError( 'pubsub_auth_token requires pubsub_topic') if self.pubsub_userdata and not self.pubsub_topic: raise datastore_errors.BadValueError( 'pubsub_userdata requires pubsub_topic')
def from_websafe_string(cursor): """Gets a Cursor given its websafe serialized form. The serialized form of a cursor may change in a non-backwards compatible way. In this case cursors must be regenerated from a new Query request. Args: cursor: A serialized cursor as returned by .to_websafe_string. Returns: A Cursor. Raises: datastore_errors.BadValueError if the cursor argument is not a string type of does not represent a serialized cursor. """ if not isinstance(cursor, basestring): raise datastore_errors.BadValueError( 'cursor argument should be str or unicode (%r)' % (cursor, )) try: decoded_bytes = base64.b64decode( str(cursor).replace('-', '+').replace('_', '/')) except (ValueError, TypeError), e: raise datastore_errors.BadValueError( 'Invalid cursor %s. Details: %s' % (cursor, e))
def ToPropertyPb(name, values): """Creates a type-specific onestore property PB from a property name and a value or list of values. Determines the type of property based on the type of the value(s). If name is invalid, Serialize throws a BadPropertyError. If values is an unsupported type, or an empty list, or a list with elements of different types, Serialize throws a BadValueError. Args: # the property name name: string # either a supported type or a list of them. if a list, all # of the list's elements should be of the same type values: string, int, long, float, datetime, Key, or list Returns: # a list of or single StringProperty, Int64Property, BoolProperty, # DoubleProperty, PointProperty, UserProperty, or ReferenceProperty. [entity_pb.*Property, ...] """ ValidateString(name, 'property name', datastore_errors.BadPropertyError) if RESERVED_PROPERTY_NAME.match(name): raise datastore_errors.BadPropertyError('%s is a reserved property name.' % name) if isinstance(values, tuple): raise datastore_errors.BadValueError( 'May not use tuple property value; property %s is %s.' % (name, repr(values))) if isinstance(values, list): multiple = True else: multiple = False values = [values] def long_if_int(val): if isinstance(val, int) and not isinstance(val, bool): return long(val) else: return val values = [long_if_int(v) for v in values] try: proptype = values[0].__class__ for v in values: if v is not None: if (v.__class__ is not proptype and not (v.__class__ in (str, unicode) and proptype in (str, unicode))): raise datastore_errors.BadValueError( 'Values for property %s have mismatched types: %s (a %s) and ' '%s (a %s).' % (name, values[0], proptype, v, typename(v))) elif (isinstance(v, Key) and not v.has_id_or_name()): raise datastore_errors.BadValueError( 'Incomplete key found for reference property %s.' % name) except (KeyError, ValueError, TypeError, IndexError, AttributeError), msg: raise datastore_errors.BadValueError( 'Error type checking values for property %s: %s' % (name, msg))
def _pre_put_hook(self): super(TaskProperties, self)._pre_put_hook() if self.commands: raise datastore_errors.BadValueError( 'commands is not supported anymore') if not self.is_terminate: if bool(self.command) == bool(self.inputs_ref): raise datastore_errors.BadValueError('use one of command or inputs_ref') if self.extra_args and not self.inputs_ref: raise datastore_errors.BadValueError('extra_args require inputs_ref') if self.inputs_ref: self.inputs_ref._pre_put_hook() package_names = set() for p in self.packages: p._pre_put_hook() if p.package_name in package_names: raise datastore_errors.BadValueError( 'package %s is specified more than once' % p.package_name) package_names.add(p.package_name) self.packages.sort(key=lambda p: p.package_name) if self.idempotent: pinned = lambda p: cipd.is_pinned_version(p.version) if self.packages and any(not pinned(p) for p in self.packages): raise datastore_errors.BadValueError( 'an idempotent task cannot have unpinned packages; ' 'use instance IDs or tags as package versions')
def __init__(self, lat, lon=None): if lon is None: try: split = lat.split(',') lat, lon = split except (AttributeError, ValueError): raise datastore_errors.BadValueError( 'Expected a "lat,long" formatted string; received %s (a %s).' % (lat, typename(lat))) try: lat = float(lat) lon = float(lon) if abs(lat) > 90: raise datastore_errors.BadValueError( 'Latitude must be between -90 and 90; received %f' % lat) if abs(lon) > 180: raise datastore_errors.BadValueError( 'Longitude must be between -180 and 180; received %f' % lon) except (TypeError, ValueError): raise datastore_errors.BadValueError( 'Expected floats for lat and long; received %s (a %s) and %s (a %s).' % (lat, typename(lat), lon, typename(lon))) self.lat = lat self.lon = lon
def _validate(self, value): if (not isinstance(value, basestring)): raise datastore_errors.BadValueError('Invalid phone') phone = re.sub(r'[^0-9]', '', value) if (not re.match(r'^1?[0-9]{10}$', phone)): raise datastore_errors.BadValueError('Invalid phone: ' + value) return re.sub(r'^1?([0-9]{3})([0-9]{3})([0-9]{4})$', r'(\1) \2-\3', phone)
def _pre_put_hook(self): if not self.name: raise datastore_errors.BadValueError('name is not specified') if not CACHE_NAME_RE.match(self.name): raise datastore_errors.BadValueError( 'name "%s" does not match %s' % (self.name, CACHE_NAME_RE.pattern)) _validate_rel_path('Cache path', self.path)
def Validate(self, value): if not isinstance(value, Key): raise datastore_errors.BadValueError('Expected Key, got %r' % (value,)) # Reject incomplete keys. last = value.pairs()[-1] if not last[-1]: raise datastore_errors.BadValueError('Expected complete Key, got %r' % (value,)) return value
def _validate_dimensions_flat(obj): """Validates obj.dimensions_flat; throws BadValueError if invalid.""" if not obj.dimensions_flat: raise datastore_errors.BadValueError( '%s.dimensions_flat is empty' % obj.__class__.__name__) if len(obj.dimensions_flat) != len(set(obj.dimensions_flat)): raise datastore_errors.BadValueError( '%s.dimensions_flat has duplicate entries' % obj.__class__.__name__) if sorted(obj.dimensions_flat) != obj.dimensions_flat: raise datastore_errors.BadValueError( '%s.dimensions_flat must be sorted' % obj.__class__.__name__)
def _validate_dimensions(prop, value): """Validates TaskProperties.dimensions.""" # pylint: disable=W0212 if not value: raise datastore_errors.BadValueError(u'%s must be specified' % prop._name) _validate_dict_of_strings(prop, value) for key in value: if not re.match(DIMENSION_KEY_RE, key): raise datastore_errors.BadValueError( u'key %r doesn\'t match %s' % (key, DIMENSION_KEY_RE)) if u'pool' not in value and u'id' not in value: raise datastore_errors.BadValueError( u'At least one of \'id\' or \'pool\' must be used as %s' % prop._name)
def _pre_put_hook(self): super(TaskProperties, self)._pre_put_hook() if not self.is_terminate: if len(self.commands or []) > 1: raise datastore_errors.BadValueError('only one command is supported') if bool(self.commands) == bool(self.inputs_ref): raise datastore_errors.BadValueError('use one of command or inputs_ref') if self.data and not self.commands: raise datastore_errors.BadValueError('data requires commands') if self.extra_args and not self.inputs_ref: raise datastore_errors.BadValueError('extra_args require inputs_ref') if self.inputs_ref: self.inputs_ref._pre_put_hook()
def __init__(self, rating): super(Rating, self).__init__(self, rating) if isinstance(rating, float) or isinstance(rating, complex): raise datastore_errors.BadValueError( 'Expected int or long; received %s (a %s).' % (rating, typename(rating))) try: if long(rating) < Rating.MIN or long(rating) > Rating.MAX: raise datastore_errors.BadValueError() except ValueError: raise datastore_errors.BadValueError( 'Expected int or long; received %s (a %s).' % (rating, typename(rating)))
def _validate_hostname(prop, value): # pylint: disable=unused-argument if value: # It must be https://*.appspot.com or https?://* parsed = urlparse.urlparse(value) if not parsed.netloc: raise datastore_errors.BadValueError( '%s must be valid hostname, not %s' % (prop._name, value)) if parsed.netloc.endswith('appspot.com'): if parsed.scheme != 'https': raise datastore_errors.BadValueError( '%s must be https://, not %s' % (prop._name, value)) elif parsed.scheme not in ('http', 'https'): raise datastore_errors.BadValueError( '%s must be https:// or http://, not %s' % (prop._name, value))
def _pre_put_hook(self): super(BqState, self)._pre_put_hook() if bool(self.recent) != bool(self.oldest): raise datastore_errors.BadValueError( 'Internal error; recent and oldest must both be set') if self.oldest: if self.oldest >= self.recent: raise datastore_errors.BadValueError( 'Internal error; oldest >= recent') if self.oldest.second or self.oldest.microsecond: raise datastore_errors.BadValueError( 'Internal error; oldest has seconds') if self.recent.second or self.recent.microsecond: raise datastore_errors.BadValueError( 'Internal error; recent has seconds')
def FromPropertyPb(pb): """Converts a onestore property PB to a python value. Args: pb: entity_pb.Property Returns: # return type is determined by the type of the argument string, int, bool, double, users.User, or one of the atom or gd types """ if not isinstance(pb, entity_pb.Property): raise datastore_errors.BadValueError( 'Expected PropertyValue; received %s (a %s).' % (pb, typename(pb))) pbval = pb.value() if (pbval.has_stringvalue()): value = pbval.stringvalue() if pb.meaning() != entity_pb.Property.BLOB: value = unicode(value.decode('utf-8')) elif pbval.has_pointvalue(): value = (pbval.pointvalue().x(), pbval.pointvalue().y()) elif pbval.has_uservalue(): email = unicode(pbval.uservalue().email().decode('utf-8')) auth_domain = unicode(pbval.uservalue().auth_domain().decode('utf-8')) nickname = unicode(pbval.uservalue().nickname().decode('utf-8')) value = users.User(email=email, _auth_domain=auth_domain, nickname=nickname) elif pbval.has_referencevalue(): value = FromReferenceProperty(pbval) elif pbval.has_int64value(): value = long(pbval.int64value()) elif pbval.has_booleanvalue(): value = bool(pbval.booleanvalue()) elif pbval.has_doublevalue(): value = float(pbval.doublevalue()) else: if pb.multiple(): raise datastore_errors.BadValueError( 'Record indicated as multiple, but has no values.') else: value = None try: if pb.has_meaning() and pb.meaning() in _PROPERTY_CONVERSIONS: value = _PROPERTY_CONVERSIONS[pb.meaning()](value) except (KeyError, ValueError, IndexError, TypeError, AttributeError), msg: raise datastore_errors.BadValueError( 'Error converting pb: %s\nException was: %s' % (pb, msg))
def validate(self, value): value = super(GenderProperty, self).validate(value) if value is not None and value not in self.values: raise datastore_errors.BadValueError( "Property %s must be '%s' or '%s'" % (self.name, self.values[0], self.values[1])) return value
def _pre_put_hook(self): super(TaskToRun, self)._pre_put_hook() if self.expiration_ts is None and self.queue_number: raise datastore_errors.BadValueError( ('%s.queue_number must be None when expiration_ts is None' % self.__class__.__name__))
def validate(self, value): if not isinstance(value, array.array) or value.typecode != self.typecode: raise datastore_errors.BadValueError( "Property %s must be an array instance with typecode %s" % (self.name, self.typecode)) value = super(ArrayProperty, self).validate(value) return value
def _validate_grace(prop, value): """Validates grace_period_secs in TaskProperties.""" if not (0 <= value <= _ONE_DAY_SECS): # pylint: disable=W0212 raise datastore_errors.BadValueError( '%s (%ds) must be between %ds and one day' % (prop._name, value, 0))
def create(cls, user_email, name, hidden, protect, color, description=None): """Creates a new tag. Args: user_email: str, email of the user creating the tag. name: str, a unique name for the tag. hidden: bool, whether a tag is hidden in the frontend UI. protect: bool, whether a tag is protected from user manipulation. color: str, the UI color of the tag in human-readable format. description: Optional[str], a description for the tag. Returns: The new Tag entity. Raises: datastore_errors.BadValueError: If the tag name is an empty string. """ tag = cls(name=name, hidden=hidden, protect=protect, color=color, description=description) if not name: raise datastore_errors.BadValueError( 'The tag name must not be empty.') tag.put() logging.info('Creating a new tag with name %r.', name) tag.stream_to_bq(user_email, 'Created a new tag with name %r.' % name) return tag
def PercentageValidator(_, value): """Validates that the total number of lines is greater than 0.""" if value <= 0: raise datastore_errors.BadValueError( 'total_lines is expected to be greater than 0.') return value
class Cursor(_BaseComponent): """An immutable class that represents a relative position in a query. The position denoted by a Cursor is relative to a result in a query even if the result has been removed from the given query. Usually to position immediately after the last result returned by a batch. A cursor should only be used on a query with an identical signature to the one that produced it. """ @datastore_rpc._positional(1) def __init__(self, _cursor_pb=None): """Constructor. A Cursor constructed with no arguments points the first result of any query. If such a Cursor is used as an end_cursor no results will ever be returned. """ super(Cursor, self).__init__() if _cursor_pb is not None: if not isinstance(_cursor_pb, datastore_pb.CompiledCursor): raise datastore_errors.BadArgumentError( '_cursor_pb argument should be datastore_pb.CompiledCursor (%r)' % (_cursor_pb, )) self.__compiled_cursor = _cursor_pb else: self.__compiled_cursor = datastore_pb.CompiledCursor() def to_bytes(self): """Serialize cursor as a byte string.""" return self.__compiled_cursor.Encode() @staticmethod def from_bytes(cursor): """Gets a Cursor given its byte string serialized form. The serialized form of a cursor may change in a non-backwards compatible way. In this case cursors must be regenerated from a new Query request. Args: cursor: A serialized cursor as returned by .to_bytes. Returns: A Cursor. Raises: datastore_errors.BadValueError if the cursor argument does not represent a serialized cursor. """ try: cursor_pb = datastore_pb.CompiledCursor(cursor) except (ValueError, TypeError), e: raise datastore_errors.BadValueError( 'Invalid cursor %s. Details: %s' % (cursor, e)) except Exception, e: if e.__class__.__name__ == 'ProtocolBufferDecodeError': raise datastore_errors.BadValueError( 'Invalid cursor %s. Details: %s' % (cursor, e)) else: raise
def _validate_socket(prop, value): # type: (ndb.StringProperty, object) -> unicode socket_types = get_config(NAMESPACE).odoo.product_ids.keys() if value in socket_types: return value else: raise datastore_errors.BadValueError('Value %r for property %s is not an allowed choice' % (value, prop._name))
def _validate_timeout(prop, value): """Validates timeouts in seconds in TaskProperties.""" if not (_MIN_TIMEOUT_SECS <= value <= _ONE_DAY_SECS): # pylint: disable=W0212 raise datastore_errors.BadValueError( '%s (%ds) must be between %ds and one day' % (prop._name, value, _MIN_TIMEOUT_SECS))
def _pre_put_hook(self): """Adds automatic tags.""" super(TaskRequest, self)._pre_put_hook() self.properties._pre_put_hook() if self.properties.is_terminate: if not self.priority == 0: raise datastore_errors.BadValueError( 'terminate request must be priority 0') elif self.priority == 0: raise datastore_errors.BadValueError( 'priority 0 can only be used for terminate request') self.tags.append('priority:%s' % self.priority) self.tags.append('user:%s' % self.user) for key, value in self.properties.dimensions.iteritems(): self.tags.append('%s:%s' % (key, value)) self.tags = sorted(set(self.tags))
def _pre_put_hook(self): """Use extra validation that cannot be validated throught 'validator'.""" super(_TaskResultCommon, self)._pre_put_hook() if self.state == State.EXPIRED: if self.failure or self.exit_code is not None: raise datastore_errors.BadValueError( 'Unexpected State, a task can\'t fail if it hasn\'t started yet' ) if self.state == State.TIMED_OUT and not self.failure: raise datastore_errors.BadValueError( 'Timeout implies task failure') if not self.modified_ts: raise datastore_errors.BadValueError('Must update .modified_ts') if self.state in State.STATES_DONE: if self.duration is None: raise datastore_errors.BadValueError( 'duration must be set with state %s' % State.to_string(self.state)) # Allow exit_code to be missing for TIMED_OUT. if self.state != State.TIMED_OUT: if self.exit_code is None: raise datastore_errors.BadValueError( 'exit_code must be set with state %s' % State.to_string(self.state)) elif self.state != State.BOT_DIED: # Allow duration and exit_code to be either missing or set for BOT_DIED, # but they should be not present for any running/pending states. if self.duration is not None: raise datastore_errors.BadValueError( 'duration must not be set with state %s' % State.to_string(self.state)) if self.exit_code is not None: raise datastore_errors.BadValueError( 'exit_code must not be set with state %s' % State.to_string(self.state)) if self.deduped_from: if self.state != State.COMPLETED: raise datastore_errors.BadValueError( 'state(%d) must be COMPLETED on deduped task %s' % (self.state, self.deduped_from)) if self.failure: raise datastore_errors.BadValueError( 'failure can\'t be True on deduped task %s' % self.deduped_from) self.children_task_ids = sorted(set(self.children_task_ids), key=lambda x: int(x, 16))
def validate(self, value): value = super(PickledProperty, self).validate(value) if value is not None and self.force_type and \ not isinstance(value, self.force_type): raise datastore_errors.BadValueError( 'Property %s must be of type "%s".' % (self.name, self.force_type)) return value
def __init__(self, link): super(Link, self).__init__(self, link) ValidateString(link, 'link', max_len=_MAX_LINK_PROPERTY_LENGTH) scheme, domain, path, params, query, fragment = urlparse.urlparse(link) if (not scheme or (scheme != 'file' and not domain) or (scheme == 'file' and not path)): raise datastore_errors.BadValueError('Invalid URL: %s' % link)
def create(cls, name, title=None, body=None): """Creates a model and entity.""" if not name: raise datastore_errors.BadValueError( 'The Template name must not be empty.') entity = cls(title=title, body=body) template = cls.get_by_id(name) if template is not None: raise datastore_errors.BadValueError( 'Create template: A Template entity with name %r already exists.' % name) entity.key = ndb.Key(cls, name) entity.put() logging.info('Creating a new template with name %r.', name) cls.cached_templates = [] return entity
def _validate(self, val): val = super(CoercedFloatProperty, self).validate(val) if val is None: return val try: return float(val) except (TypeError, ValueError): raise datastore_errors.BadValueError( 'Property %s must be a float' % self.name)