def _pre_put_hook(self):
"""Adds automatic tags."""
super(TaskRequest, self)._pre_put_hook()
self.properties._pre_put_hook()
if self.properties.is_terminate:
if not self.priority == 0:
raise datastore_errors.BadValueError(
'terminate request must be priority 0')
elif self.priority == 0:
raise datastore_errors.BadValueError(
'priority 0 can only be used for terminate request')
self.tags.append('priority:%s' % self.priority)
self.tags.append('user:%s' % self.user)
for key, value in self.properties.dimensions.iteritems():
self.tags.append('%s:%s' % (key, value))
self.tags = sorted(set(self.tags))
if (self.pubsub_topic and
not pubsub.validate_full_name(self.pubsub_topic, 'topics')):
raise datastore_errors.BadValueError('bad pubsub topic name - %s' %
self.pubsub_topic)
if self.pubsub_auth_token and not self.pubsub_topic:
raise datastore_errors.BadValueError(
'pubsub_auth_token requires pubsub_topic')
if self.pubsub_userdata and not self.pubsub_topic:
raise datastore_errors.BadValueError(
'pubsub_userdata requires pubsub_topic')
def from_websafe_string(cursor):
"""Gets a Cursor given its websafe serialized form.
The serialized form of a cursor may change in a non-backwards compatible
way. In this case cursors must be regenerated from a new Query request.
Args:
cursor: A serialized cursor as returned by .to_websafe_string.
Returns:
A Cursor.
Raises:
datastore_errors.BadValueError if the cursor argument is not a string
type of does not represent a serialized cursor.
"""
if not isinstance(cursor, basestring):
raise datastore_errors.BadValueError(
'cursor argument should be str or unicode (%r)' % (cursor, ))
try:
decoded_bytes = base64.b64decode(
str(cursor).replace('-', '+').replace('_', '/'))
except (ValueError, TypeError), e:
raise datastore_errors.BadValueError(
'Invalid cursor %s. Details: %s' % (cursor, e))
def ToPropertyPb(name, values):
"""Creates a type-specific onestore property PB from a property name and a
value or list of values. Determines the type of property based on the type
of the value(s).
If name is invalid, Serialize throws a BadPropertyError. If values is
an unsupported type, or an empty list, or a list with elements of different
types, Serialize throws a BadValueError.
Args:
# the property name
name: string
# either a supported type or a list of them. if a list, all
# of the list's elements should be of the same type
values: string, int, long, float, datetime, Key, or list
Returns:
# a list of or single StringProperty, Int64Property, BoolProperty,
# DoubleProperty, PointProperty, UserProperty, or ReferenceProperty.
[entity_pb.*Property, ...]
"""
ValidateString(name, 'property name', datastore_errors.BadPropertyError)
if RESERVED_PROPERTY_NAME.match(name):
raise datastore_errors.BadPropertyError('%s is a reserved property name.' %
name)
if isinstance(values, tuple):
raise datastore_errors.BadValueError(
'May not use tuple property value; property %s is %s.' %
(name, repr(values)))
if isinstance(values, list):
multiple = True
else:
multiple = False
values = [values]
def long_if_int(val):
if isinstance(val, int) and not isinstance(val, bool):
return long(val)
else:
return val
values = [long_if_int(v) for v in values]
try:
proptype = values[0].__class__
for v in values:
if v is not None:
if (v.__class__ is not proptype and not
(v.__class__ in (str, unicode) and proptype in (str, unicode))):
raise datastore_errors.BadValueError(
'Values for property %s have mismatched types: %s (a %s) and '
'%s (a %s).' % (name, values[0], proptype, v, typename(v)))
elif (isinstance(v, Key) and not v.has_id_or_name()):
raise datastore_errors.BadValueError(
'Incomplete key found for reference property %s.' % name)
except (KeyError, ValueError, TypeError, IndexError, AttributeError), msg:
raise datastore_errors.BadValueError(
'Error type checking values for property %s: %s' % (name, msg))
def _pre_put_hook(self):
super(TaskProperties, self)._pre_put_hook()
if self.commands:
raise datastore_errors.BadValueError(
'commands is not supported anymore')
if not self.is_terminate:
if bool(self.command) == bool(self.inputs_ref):
raise datastore_errors.BadValueError('use one of command or inputs_ref')
if self.extra_args and not self.inputs_ref:
raise datastore_errors.BadValueError('extra_args require inputs_ref')
if self.inputs_ref:
self.inputs_ref._pre_put_hook()
package_names = set()
for p in self.packages:
p._pre_put_hook()
if p.package_name in package_names:
raise datastore_errors.BadValueError(
'package %s is specified more than once' % p.package_name)
package_names.add(p.package_name)
self.packages.sort(key=lambda p: p.package_name)
if self.idempotent:
pinned = lambda p: cipd.is_pinned_version(p.version)
if self.packages and any(not pinned(p) for p in self.packages):
raise datastore_errors.BadValueError(
'an idempotent task cannot have unpinned packages; '
'use instance IDs or tags as package versions')
def __init__(self, lat, lon=None):
if lon is None:
try:
split = lat.split(',')
lat, lon = split
except (AttributeError, ValueError):
raise datastore_errors.BadValueError(
'Expected a "lat,long" formatted string; received %s (a %s).' %
(lat, typename(lat)))
try:
lat = float(lat)
lon = float(lon)
if abs(lat) > 90:
raise datastore_errors.BadValueError(
'Latitude must be between -90 and 90; received %f' % lat)
if abs(lon) > 180:
raise datastore_errors.BadValueError(
'Longitude must be between -180 and 180; received %f' % lon)
except (TypeError, ValueError):
raise datastore_errors.BadValueError(
'Expected floats for lat and long; received %s (a %s) and %s (a %s).' %
(lat, typename(lat), lon, typename(lon)))
self.lat = lat
self.lon = lon
def _validate(self, value):
if (not isinstance(value, basestring)):
raise datastore_errors.BadValueError('Invalid phone')
phone = re.sub(r'[^0-9]', '', value)
if (not re.match(r'^1?[0-9]{10}$', phone)):
raise datastore_errors.BadValueError('Invalid phone: ' + value)
return re.sub(r'^1?([0-9]{3})([0-9]{3})([0-9]{4})$', r'(\1) \2-\3',
phone)
def _pre_put_hook(self):
if not self.name:
raise datastore_errors.BadValueError('name is not specified')
if not CACHE_NAME_RE.match(self.name):
raise datastore_errors.BadValueError(
'name "%s" does not match %s' % (self.name, CACHE_NAME_RE.pattern))
_validate_rel_path('Cache path', self.path)
def Validate(self, value):
if not isinstance(value, Key):
raise datastore_errors.BadValueError('Expected Key, got %r' % (value,))
# Reject incomplete keys.
last = value.pairs()[-1]
if not last[-1]:
raise datastore_errors.BadValueError('Expected complete Key, got %r' %
(value,))
return value
def _validate_dimensions_flat(obj):
"""Validates obj.dimensions_flat; throws BadValueError if invalid."""
if not obj.dimensions_flat:
raise datastore_errors.BadValueError(
'%s.dimensions_flat is empty' % obj.__class__.__name__)
if len(obj.dimensions_flat) != len(set(obj.dimensions_flat)):
raise datastore_errors.BadValueError(
'%s.dimensions_flat has duplicate entries' % obj.__class__.__name__)
if sorted(obj.dimensions_flat) != obj.dimensions_flat:
raise datastore_errors.BadValueError(
'%s.dimensions_flat must be sorted' % obj.__class__.__name__)
def _validate_dimensions(prop, value):
"""Validates TaskProperties.dimensions."""
# pylint: disable=W0212
if not value:
raise datastore_errors.BadValueError(u'%s must be specified' % prop._name)
_validate_dict_of_strings(prop, value)
for key in value:
if not re.match(DIMENSION_KEY_RE, key):
raise datastore_errors.BadValueError(
u'key %r doesn\'t match %s' % (key, DIMENSION_KEY_RE))
if u'pool' not in value and u'id' not in value:
raise datastore_errors.BadValueError(
u'At least one of \'id\' or \'pool\' must be used as %s' % prop._name)
def _pre_put_hook(self):
super(TaskProperties, self)._pre_put_hook()
if not self.is_terminate:
if len(self.commands or []) > 1:
raise datastore_errors.BadValueError('only one command is supported')
if bool(self.commands) == bool(self.inputs_ref):
raise datastore_errors.BadValueError('use one of command or inputs_ref')
if self.data and not self.commands:
raise datastore_errors.BadValueError('data requires commands')
if self.extra_args and not self.inputs_ref:
raise datastore_errors.BadValueError('extra_args require inputs_ref')
if self.inputs_ref:
self.inputs_ref._pre_put_hook()
def __init__(self, rating):
super(Rating, self).__init__(self, rating)
if isinstance(rating, float) or isinstance(rating, complex):
raise datastore_errors.BadValueError(
'Expected int or long; received %s (a %s).' %
(rating, typename(rating)))
try:
if long(rating) < Rating.MIN or long(rating) > Rating.MAX:
raise datastore_errors.BadValueError()
except ValueError:
raise datastore_errors.BadValueError(
'Expected int or long; received %s (a %s).' %
(rating, typename(rating)))
def _validate_hostname(prop, value):
# pylint: disable=unused-argument
if value:
# It must be https://*.appspot.com or https?://*
parsed = urlparse.urlparse(value)
if not parsed.netloc:
raise datastore_errors.BadValueError(
'%s must be valid hostname, not %s' % (prop._name, value))
if parsed.netloc.endswith('appspot.com'):
if parsed.scheme != 'https':
raise datastore_errors.BadValueError(
'%s must be https://, not %s' % (prop._name, value))
elif parsed.scheme not in ('http', 'https'):
raise datastore_errors.BadValueError(
'%s must be https:// or http://, not %s' % (prop._name, value))
def _pre_put_hook(self):
super(BqState, self)._pre_put_hook()
if bool(self.recent) != bool(self.oldest):
raise datastore_errors.BadValueError(
'Internal error; recent and oldest must both be set')
if self.oldest:
if self.oldest >= self.recent:
raise datastore_errors.BadValueError(
'Internal error; oldest >= recent')
if self.oldest.second or self.oldest.microsecond:
raise datastore_errors.BadValueError(
'Internal error; oldest has seconds')
if self.recent.second or self.recent.microsecond:
raise datastore_errors.BadValueError(
'Internal error; recent has seconds')
def FromPropertyPb(pb):
"""Converts a onestore property PB to a python value.
Args:
pb: entity_pb.Property
Returns:
# return type is determined by the type of the argument
string, int, bool, double, users.User, or one of the atom or gd types
"""
if not isinstance(pb, entity_pb.Property):
raise datastore_errors.BadValueError(
'Expected PropertyValue; received %s (a %s).' % (pb, typename(pb)))
pbval = pb.value()
if (pbval.has_stringvalue()):
value = pbval.stringvalue()
if pb.meaning() != entity_pb.Property.BLOB:
value = unicode(value.decode('utf-8'))
elif pbval.has_pointvalue():
value = (pbval.pointvalue().x(), pbval.pointvalue().y())
elif pbval.has_uservalue():
email = unicode(pbval.uservalue().email().decode('utf-8'))
auth_domain = unicode(pbval.uservalue().auth_domain().decode('utf-8'))
nickname = unicode(pbval.uservalue().nickname().decode('utf-8'))
value = users.User(email=email, _auth_domain=auth_domain, nickname=nickname)
elif pbval.has_referencevalue():
value = FromReferenceProperty(pbval)
elif pbval.has_int64value():
value = long(pbval.int64value())
elif pbval.has_booleanvalue():
value = bool(pbval.booleanvalue())
elif pbval.has_doublevalue():
value = float(pbval.doublevalue())
else:
if pb.multiple():
raise datastore_errors.BadValueError(
'Record indicated as multiple, but has no values.')
else:
value = None
try:
if pb.has_meaning() and pb.meaning() in _PROPERTY_CONVERSIONS:
value = _PROPERTY_CONVERSIONS[pb.meaning()](value)
except (KeyError, ValueError, IndexError, TypeError, AttributeError), msg:
raise datastore_errors.BadValueError(
'Error converting pb: %s\nException was: %s' % (pb, msg))
def validate(self, value):
value = super(GenderProperty, self).validate(value)
if value is not None and value not in self.values:
raise datastore_errors.BadValueError(
"Property %s must be '%s' or '%s'" %
(self.name, self.values[0], self.values[1]))
return value
def _pre_put_hook(self):
super(TaskToRun, self)._pre_put_hook()
if self.expiration_ts is None and self.queue_number:
raise datastore_errors.BadValueError(
('%s.queue_number must be None when expiration_ts is None' %
self.__class__.__name__))
def validate(self, value):
if not isinstance(value, array.array) or value.typecode != self.typecode:
raise datastore_errors.BadValueError(
"Property %s must be an array instance with typecode %s"
% (self.name, self.typecode))
value = super(ArrayProperty, self).validate(value)
return value
def _validate_grace(prop, value):
"""Validates grace_period_secs in TaskProperties."""
if not (0 <= value <= _ONE_DAY_SECS):
# pylint: disable=W0212
raise datastore_errors.BadValueError(
'%s (%ds) must be between %ds and one day' %
(prop._name, value, 0))
def create(cls,
user_email,
name,
hidden,
protect,
color,
description=None):
"""Creates a new tag.
Args:
user_email: str, email of the user creating the tag.
name: str, a unique name for the tag.
hidden: bool, whether a tag is hidden in the frontend UI.
protect: bool, whether a tag is protected from user manipulation.
color: str, the UI color of the tag in human-readable format.
description: Optional[str], a description for the tag.
Returns:
The new Tag entity.
Raises:
datastore_errors.BadValueError: If the tag name is an empty string.
"""
tag = cls(name=name,
hidden=hidden,
protect=protect,
color=color,
description=description)
if not name:
raise datastore_errors.BadValueError(
'The tag name must not be empty.')
tag.put()
logging.info('Creating a new tag with name %r.', name)
tag.stream_to_bq(user_email, 'Created a new tag with name %r.' % name)
return tag
def PercentageValidator(_, value):
"""Validates that the total number of lines is greater than 0."""
if value <= 0:
raise datastore_errors.BadValueError(
'total_lines is expected to be greater than 0.')
return value
class Cursor(_BaseComponent):
"""An immutable class that represents a relative position in a query.
The position denoted by a Cursor is relative to a result in a query even
if the result has been removed from the given query. Usually to position
immediately after the last result returned by a batch.
A cursor should only be used on a query with an identical signature to the
one that produced it.
"""
@datastore_rpc._positional(1)
def __init__(self, _cursor_pb=None):
"""Constructor.
A Cursor constructed with no arguments points the first result of any
query. If such a Cursor is used as an end_cursor no results will ever be
returned.
"""
super(Cursor, self).__init__()
if _cursor_pb is not None:
if not isinstance(_cursor_pb, datastore_pb.CompiledCursor):
raise datastore_errors.BadArgumentError(
'_cursor_pb argument should be datastore_pb.CompiledCursor (%r)'
% (_cursor_pb, ))
self.__compiled_cursor = _cursor_pb
else:
self.__compiled_cursor = datastore_pb.CompiledCursor()
def to_bytes(self):
"""Serialize cursor as a byte string."""
return self.__compiled_cursor.Encode()
@staticmethod
def from_bytes(cursor):
"""Gets a Cursor given its byte string serialized form.
The serialized form of a cursor may change in a non-backwards compatible
way. In this case cursors must be regenerated from a new Query request.
Args:
cursor: A serialized cursor as returned by .to_bytes.
Returns:
A Cursor.
Raises:
datastore_errors.BadValueError if the cursor argument does not represent a
serialized cursor.
"""
try:
cursor_pb = datastore_pb.CompiledCursor(cursor)
except (ValueError, TypeError), e:
raise datastore_errors.BadValueError(
'Invalid cursor %s. Details: %s' % (cursor, e))
except Exception, e:
if e.__class__.__name__ == 'ProtocolBufferDecodeError':
raise datastore_errors.BadValueError(
'Invalid cursor %s. Details: %s' % (cursor, e))
else:
raise
def _validate_socket(prop, value):
# type: (ndb.StringProperty, object) -> unicode
socket_types = get_config(NAMESPACE).odoo.product_ids.keys()
if value in socket_types:
return value
else:
raise datastore_errors.BadValueError('Value %r for property %s is not an allowed choice' % (value, prop._name))
def _validate_timeout(prop, value):
"""Validates timeouts in seconds in TaskProperties."""
if not (_MIN_TIMEOUT_SECS <= value <= _ONE_DAY_SECS):
# pylint: disable=W0212
raise datastore_errors.BadValueError(
'%s (%ds) must be between %ds and one day' %
(prop._name, value, _MIN_TIMEOUT_SECS))
def _pre_put_hook(self):
"""Adds automatic tags."""
super(TaskRequest, self)._pre_put_hook()
self.properties._pre_put_hook()
if self.properties.is_terminate:
if not self.priority == 0:
raise datastore_errors.BadValueError(
'terminate request must be priority 0')
elif self.priority == 0:
raise datastore_errors.BadValueError(
'priority 0 can only be used for terminate request')
self.tags.append('priority:%s' % self.priority)
self.tags.append('user:%s' % self.user)
for key, value in self.properties.dimensions.iteritems():
self.tags.append('%s:%s' % (key, value))
self.tags = sorted(set(self.tags))
def _pre_put_hook(self):
"""Use extra validation that cannot be validated throught 'validator'."""
super(_TaskResultCommon, self)._pre_put_hook()
if self.state == State.EXPIRED:
if self.failure or self.exit_code is not None:
raise datastore_errors.BadValueError(
'Unexpected State, a task can\'t fail if it hasn\'t started yet'
)
if self.state == State.TIMED_OUT and not self.failure:
raise datastore_errors.BadValueError(
'Timeout implies task failure')
if not self.modified_ts:
raise datastore_errors.BadValueError('Must update .modified_ts')
if self.state in State.STATES_DONE:
if self.duration is None:
raise datastore_errors.BadValueError(
'duration must be set with state %s' %
State.to_string(self.state))
# Allow exit_code to be missing for TIMED_OUT.
if self.state != State.TIMED_OUT:
if self.exit_code is None:
raise datastore_errors.BadValueError(
'exit_code must be set with state %s' %
State.to_string(self.state))
elif self.state != State.BOT_DIED:
# Allow duration and exit_code to be either missing or set for BOT_DIED,
# but they should be not present for any running/pending states.
if self.duration is not None:
raise datastore_errors.BadValueError(
'duration must not be set with state %s' %
State.to_string(self.state))
if self.exit_code is not None:
raise datastore_errors.BadValueError(
'exit_code must not be set with state %s' %
State.to_string(self.state))
if self.deduped_from:
if self.state != State.COMPLETED:
raise datastore_errors.BadValueError(
'state(%d) must be COMPLETED on deduped task %s' %
(self.state, self.deduped_from))
if self.failure:
raise datastore_errors.BadValueError(
'failure can\'t be True on deduped task %s' %
self.deduped_from)
self.children_task_ids = sorted(set(self.children_task_ids),
key=lambda x: int(x, 16))
def validate(self, value):
value = super(PickledProperty, self).validate(value)
if value is not None and self.force_type and \
not isinstance(value, self.force_type):
raise datastore_errors.BadValueError(
'Property %s must be of type "%s".' %
(self.name, self.force_type))
return value
def __init__(self, link):
super(Link, self).__init__(self, link)
ValidateString(link, 'link', max_len=_MAX_LINK_PROPERTY_LENGTH)
scheme, domain, path, params, query, fragment = urlparse.urlparse(link)
if (not scheme or (scheme != 'file' and not domain) or
(scheme == 'file' and not path)):
raise datastore_errors.BadValueError('Invalid URL: %s' % link)
def create(cls, name, title=None, body=None):
"""Creates a model and entity."""
if not name:
raise datastore_errors.BadValueError(
'The Template name must not be empty.')
entity = cls(title=title,
body=body)
template = cls.get_by_id(name)
if template is not None:
raise datastore_errors.BadValueError(
'Create template: A Template entity with name %r already exists.' %
name)
entity.key = ndb.Key(cls, name)
entity.put()
logging.info('Creating a new template with name %r.', name)
cls.cached_templates = []
return entity
def _validate(self, val):
val = super(CoercedFloatProperty, self).validate(val)
if val is None:
return val
try:
return float(val)
except (TypeError, ValueError):
raise datastore_errors.BadValueError(
'Property %s must be a float' % self.name)
