def get_scatter_prop(element_list): """ Gets the scatter property for an entity's key path. This will return a property for only a small percentage of entities. Args: element_list: A list of entity_pb.Path_Element objects. Returns: An entity_pb.Property object or None. """ def id_from_element(element): if element.has_name(): return element.name() elif element.has_id(): return str(element.id()) else: return '' to_hash = ''.join([id_from_element(element) for element in element_list]) full_hash = mmh3.hash(to_hash) hash_bytes = struct.pack('i', full_hash)[0:2] hash_int = struct.unpack('H', hash_bytes)[0] if hash_int >= dbconstants.SCATTER_PROPORTION: return None scatter_property = entity_pb.Property() scatter_property.set_name('__scatter__') scatter_property.set_meaning(entity_pb.Property.BYTESTRING) scatter_property.set_multiple(False) property_value = scatter_property.mutable_value() property_value.set_stringvalue(hash_bytes) return scatter_property
def _GetScatterProperty(entity_proto): """Gets the scatter property for an object. For ease of implementation, this is not synchronized with the actual value on the App Engine server, but should work equally well. Note: This property may change, either here or in production. No client other than the mapper framework should rely on it directly. Returns: The PropertyValue of the scatter property or None if this entity should not have a scatter property. """ hash_obj = _MD5_FUNC() for element in entity_proto.key().path().element_list(): if element.has_name(): hash_obj.update(element.name()) elif element.has_id(): hash_obj.update(str(element.id())) hash_bytes = hash_obj.digest()[0:2] (hash_int, ) = struct.unpack('H', hash_bytes) if hash_int >= _SCATTER_PROPORTION: return None scatter_property = entity_pb.Property() scatter_property.set_name('__scatter__') scatter_property.set_meaning(entity_pb.Property.BYTESTRING) scatter_property.set_multiple(False) property_value = scatter_property.mutable_value() property_value.set_stringvalue(hash_bytes) return scatter_property
def _get_scatter_property(entity_proto): key_name = entity_proto.key().path().element_list()[-1].name() if key_name in key_names_to_vals: scatter_property = entity_pb.Property() scatter_property.set_name(datastore_types.SCATTER_SPECIAL_PROPERTY) scatter_property.set_meaning(entity_pb.Property.BYTESTRING) scatter_property.set_multiple(False) property_value = scatter_property.mutable_value() property_value.set_stringvalue(str(key_names_to_vals[key_name])) return scatter_property
'Values for property %s have mismatched types: %s (a %s) and ' '%s (a %s).' % (name, values[0], proptype, v, typename(v))) elif (isinstance(v, Key) and not v.has_id_or_name()): raise datastore_errors.BadValueError( 'Incomplete key found for reference property %s.' % name) except (KeyError, ValueError, TypeError, IndexError, AttributeError), msg: raise datastore_errors.BadValueError( 'Error type checking values for property %s: %s' % (name, msg)) if proptype not in _PROPERTY_TYPES: raise datastore_errors.BadValueError( 'Unsupported type for property %s: %s' % (name, proptype)) pbs = [] for v in values: pb = entity_pb.Property() pb.set_name(name.encode('utf-8')) pb.set_multiple(multiple) if _PROPERTY_MEANINGS.has_key(proptype): pb.set_meaning(_PROPERTY_MEANINGS[proptype]) pbvalue = pb.mutable_value() if v is None: pass elif isinstance(v, Blob): pbvalue.set_stringvalue(v) elif isinstance(v, (basestring, IM)): if not isinstance(v, Text): if isinstance(v, Link): max_len = _MAX_LINK_PROPERTY_LENGTH else: