def apply_path_filter(self, op, path, ancestor_path=()): if not isinstance(path, tuple): path = Path.flatten(path) remaining_path = path[len(ancestor_path):] if self._ancestor else path if not remaining_path: raise InternalError(u'Path filter must be within ancestor') start = Path.pack(remaining_path, omit_terminator=True) # Since the commit versionstamp could potentially start with 0xFF, this # selection scans up to the next possible path value. stop = start + six.int2byte(Path.MIN_ID_MARKER) index = -2 if op == Query_Filter.EQUAL: self._set_start(index, start) self._set_stop(index, stop) self._set_stop(index + 1, b'\xFF') return if op == Query_Filter.GREATER_THAN_OR_EQUAL: self._set_start(index, start) elif op == Query_Filter.GREATER_THAN: self._set_start(index, stop) elif op == Query_Filter.LESS_THAN_OR_EQUAL: self._set_stop(index, stop) elif op == Query_Filter.LESS_THAN: self._set_stop(index, start) else: raise BadRequest(u'Unexpected filter operation')
def encode_key(self, path, original_versionstamp, deleted_versionstamp=None): """ Encodes a key for a deleted version index entry. Args: path: A tuple or protobuf path object. original_versionstamp: A 10-byte string specifying the entity version's original commit versionstamp. deleted_versionstamp: A 10-byte string specifying the entity version's delete versionstamp or None. Returns: A string containing an FDB key. If deleted_versionstamp was None, the key should be used with set_versionstamped_key. """ if not isinstance(path, tuple): path = Path.flatten(path) scatter_byte = hash_tuple(path) key = b''.join([ self.directory.rawPrefix, scatter_byte, deleted_versionstamp or b'\x00' * VERSIONSTAMP_SIZE, Path.pack(path), original_versionstamp ]) if not deleted_versionstamp: versionstamp_index = len( self.directory.rawPrefix) + len(scatter_byte) key += encode_versionstamp_index(versionstamp_index) return key
def encode_query_key(self, txid, namespace, ancestor_path): if not isinstance(ancestor_path, tuple): ancestor_path = Path.flatten(ancestor_path) section_prefix = self._txid_prefix(txid) + self.QUERIES encoded_ancestor = Text.encode(namespace) + Path.pack(ancestor_path[:2]) return section_prefix + encoded_ancestor
def encode_key(self, ancestor_path, encoded_values, remaining_path, commit_versionstamp): ancestor_path = Path.pack(ancestor_path) if ancestor_path else b'' remaining_path = Path.pack(remaining_path) key = b''.join( (self.directory.rawPrefix, ancestor_path) + tuple(encoded_values) + (remaining_path, commit_versionstamp or b'\x00' * VERSIONSTAMP_SIZE)) if not commit_versionstamp: key += encode_versionstamp_index(len(key) - VERSIONSTAMP_SIZE) return key
def _encode_path_prefix(self, path): """ Encodes the portion of the key up to and including the path. Args: path: A tuple or protobuf path object. Returns: A string containing the path prefix. """ if not isinstance(path, tuple): path = Path.flatten(path) return b''.join([self.directory.rawPrefix, hash_tuple(path), Path.pack(path)])
def set_ancestor(self, ancestor_path): if not ancestor_path: return index = 1 if self._ancestor else -2 if self._ancestor: self._set_start(index, Path.pack(ancestor_path)) self._set_stop(index, Path.pack(ancestor_path)) self._set_stop(index + 1, b'\xFF') else: prefix = Path.pack(ancestor_path, omit_terminator=True) self._set_start(index, prefix) self._set_stop(index, prefix + b'\xFF')
def _unpack_keys(self, blob): keys = [] pos = 0 while pos < len(blob): namespace, pos = Text.decode(blob, pos) path, pos = Path.unpack(blob, pos) key = entity_pb.Reference() key.set_app(self.project_id) key.set_name_space(namespace) key.mutable_path().MergeFrom(Path.decode(path)) keys.append(key) return keys
def encode_key(self, path, commit_versionstamp): key = b''.join([self.directory.rawPrefix, Path.pack(path), commit_versionstamp or b'\x00' * VERSIONSTAMP_SIZE]) if not commit_versionstamp: key += encode_versionstamp_index(len(key) - VERSIONSTAMP_SIZE) return key
def format_prop_val(prop_value): if prop_value.has_int64value(): return prop_value.int64value() elif prop_value.has_booleanvalue(): return prop_value.booleanvalue() elif prop_value.has_stringvalue(): return repr(prop_value.stringvalue()) elif prop_value.has_doublevalue(): return prop_value.doublevalue() elif prop_value.has_pointvalue(): point_val = prop_value.pointvalue() return point_val.x(), point_val.y() elif prop_value.has_uservalue(): user_val = prop_value.uservalue() details = { 'email': user_val.email(), 'auth_domain': user_val.auth_domain() } if user_val.has_nickname(): details['nickname'] = user_val.nickname() if user_val.has_federated_identity(): details['federatedIdentity'] = user_val.federated_identity() if user_val.has_federated_provider(): details['federatedProvider'] = user_val.federated_provider() return json.dumps(details) elif prop_value.has_referencevalue(): return Path.flatten(prop_value.referencevalue()) else: return None
def get_latest(self, tr, key, read_versionstamp=None, include_data=True, snapshot=False): """ Gets the newest entity version for the given read versionstamp. Args: tr: An FDB transaction. key: A protubuf reference object. read_versionstamp: A 10-byte string specifying the FDB read versionstamp. Newer versionstamps are ignored. include_data: A boolean specifying whether or not to fetch all of the entity's Key-Values. snapshot: If True, the read will not cause a transaction conflict. """ data_ns = yield self._data_ns_from_key(tr, key) desired_slice = data_ns.get_slice(key.path(), read_versionstamp=read_versionstamp) last_entry = yield self._last_version(tr, data_ns, desired_slice, include_data, snapshot=snapshot) if last_entry is None: last_entry = VersionEntry(data_ns.project_id, data_ns.namespace, Path.flatten(key.path())) raise gen.Return(last_entry)
def decode(self, kv): value, pos = decode_value(kv.key, len(self.directory.rawPrefix)) path = Path.unpack(kv.key, pos)[0] commit_versionstamp = kv.key[self.versionstamp_slice] deleted_versionstamp = kv.value or None return PropertyEntry(self.project_id, self.namespace, path, self.prop_name, value, commit_versionstamp, deleted_versionstamp)
def key(self): key = entity_pb.Reference() key.set_app(self.project_id) if self.namespace is not None: key.set_name_space(self.namespace) key.mutable_path().MergeFrom(Path.decode(self.path)) return key
def from_key(cls, key): project_id = decode_str(key.app()) namespace = None if key.has_name_space(): namespace = decode_str(key.name_space()) path = Path.flatten(key.path()) return cls(project_id, namespace, path)
def decode(self, kv): pos = len(self.directory.rawPrefix) properties = [] if self.ancestor: ancestor_path, pos = Path.unpack(kv.key, pos) else: ancestor_path = () for prop_name, direction in self.order_info: value, pos = decode_value(kv.key, pos, direction == Query_Order.DESCENDING) properties.append((prop_name, value)) remaining_path = Path.unpack(kv.key, pos)[0] path = ancestor_path + remaining_path commit_versionstamp = kv.key[self.versionstamp_slice] deleted_versionstamp = kv.value or None return CompositeEntry(self.project_id, self.namespace, path, properties, commit_versionstamp, deleted_versionstamp)
def encode_key(self, group_path): """ Encodes a key for a given entity group. Args: group_path: A tuple containing path elements. Returns: A byte string containing the relevant FDB key. """ return b''.join([ self.directory.rawPrefix, hash_tuple(group_path), Text.encode(group_path[0]) + Path.encode_id_or_name(group_path[1])])
def encode_key(self, path): """ Encodes a key for a safe read versionstamp entry. Args: path: A tuple or protobuf path object. Returns: A string containing an FDB key. """ if not isinstance(path, tuple): path = Path.flatten(path) entity_group = path[:2] return self.directory.rawPrefix + hash_tuple(entity_group)
def decode(self, kv): """ Decodes a KV to a DeletedVersionEntry. Args: kv: An FDB KeyValue object. Returns: A DeletedVersionEntry object. """ deleted_versionstamp = kv.key[self.del_versionstamp_slice] path = Path.unpack(kv.key, self.path_slice.start)[0] original_versionstamp = kv.key[self.original_versionstamp_slice] return DeletedVersionEntry(self.project_id, self.namespace, path, original_versionstamp, deleted_versionstamp)
def encode(self, path): """ Creates a Key-Value tuple for updating a group's commit versionstamp. Args: path: A tuple or protobuf path object. Returns: A (key, value) tuple suitable for set_versionstamped_value. """ if not isinstance(path, tuple): path = Path.flatten(path) group_path = path[:2] return (self.encode_key(group_path), b'\x00' * VERSIONSTAMP_SIZE + encode_versionstamp_index(0))
def allocate_ids_request(self, app_id, http_request_data): """ High level function for getting unique identifiers for entities. Args: app_id: Name of the application. http_request_data: Stores the protocol buffer request from the AppServer. Returns: Returns an encoded response. Raises: NotImplementedError: when requesting a max id. """ request = datastore_pb.AllocateIdsRequest(http_request_data) if request.has_max() and request.has_size(): raise gen.Return( ('', datastore_pb.Error.BAD_REQUEST, 'Both size and max cannot be set.')) if not (request.has_max() or request.has_size()): raise gen.Return( ('', datastore_pb.Error.BAD_REQUEST, 'Either size or max must be set.')) if not request.has_model_key(): raise gen.Return(('', datastore_pb.Error.BAD_REQUEST, 'Model key must be set')) namespace = six.text_type(request.model_key().name_space()) path_prefix = Path.flatten(request.model_key().path(), allow_partial=True)[:-1] if request.has_size(): coroutine = datastore_access.allocate_size args = (app_id, namespace, path_prefix, request.size()) else: coroutine = datastore_access.allocate_max args = (app_id, namespace, path_prefix, request.max()) try: start, end = yield coroutine(*args) except dbconstants.AppScaleBadArg as error: raise gen.Return(('', datastore_pb.Error.BAD_REQUEST, str(error))) except dbconstants.AppScaleDBConnectionError as error: raise gen.Return(('', datastore_pb.Error.INTERNAL_ERROR, str(error))) response = datastore_pb.AllocateIdsResponse() response.set_start(start) response.set_end(end) raise gen.Return((response.Encode(), 0, ''))
def _get_index_keys(self, tr, entity, commit_versionstamp=None): has_index = commit_versionstamp is None project_id = decode_str(entity.key().app()) namespace = decode_str(entity.key().name_space()) path = Path.flatten(entity.key().path()) kind = path[-2] stats = IndexStatsSummary() kindless_index = yield self._kindless_index(tr, project_id, namespace) kind_index = yield self._kind_index(tr, project_id, namespace, kind) composite_indexes = yield self._get_indexes(tr, project_id, namespace, kind) kindless_key = kindless_index.encode_key(path, commit_versionstamp) kind_key = kind_index.encode_key(path, commit_versionstamp) stats.add_kindless_key(kindless_key, has_index) stats.add_kind_key(kind_key, has_index) all_keys = [kindless_key, kind_key] entity_prop_names = [] for prop in entity.property_list(): prop_name = decode_str(prop.name()) entity_prop_names.append(prop_name) index = yield self._single_prop_index(tr, project_id, namespace, kind, prop_name) prop_key = index.encode_key(prop.value(), path, commit_versionstamp) stats.add_prop_key(prop, prop_key, has_index) all_keys.append(prop_key) scatter_val = get_scatter_val(path) if scatter_val is not None: index = yield self._single_prop_index(tr, project_id, namespace, kind, SCATTER_PROP) all_keys.append( index.encode_key(scatter_val, path, commit_versionstamp)) for index in composite_indexes: # If the entity does not have the relevant props for the index, skip it. if not all(index_prop_name in entity_prop_names for index_prop_name in index.prop_names): continue composite_keys = index.encode_keys(entity.property_list(), path, commit_versionstamp) stats.add_composite_keys(index.id, composite_keys, has_index) all_keys.extend(composite_keys) raise gen.Return((all_keys, stats))
def _enforce_max_groups(mutations): """ Raises an exception if too many groups were modified. """ mutated_groups = set() for mutation in mutations: if isinstance(mutation, entity_pb.Reference): key = mutation else: key = mutation.key() namespace = decode_str(key.name_space()) flat_group = (namespace, ) + Path.flatten(key.path())[:2] mutated_groups.add(flat_group) if len(mutated_groups) > 25: raise BadRequest( u'Too many entity groups modified in transaction')
def decode_metadata(self, txid, kvs): lookup_rpcs = defaultdict(list) queried_groups = set() mutation_rpcs = [] rpc_type_index = len(self._txid_prefix(txid)) current_versionstamp = None for kv in kvs: rpc_type = kv.key[rpc_type_index] pos = rpc_type_index + 1 if rpc_type == self.QUERIES: namespace, pos = Text.decode(kv.key, pos) group_path = Path.unpack(kv.key, pos)[0] queried_groups.add((namespace, group_path)) continue rpc_versionstamp = kv.key[pos:pos + VERSIONSTAMP_SIZE] if rpc_type == self.LOOKUPS: lookup_rpcs[rpc_versionstamp].append(kv.value) elif rpc_type in (self.PUTS, self.DELETES): if current_versionstamp == rpc_versionstamp: mutation_rpcs[-1].append(kv.value) else: current_versionstamp = rpc_versionstamp mutation_rpcs.append([rpc_type, kv.value]) else: raise InternalError(u'Unrecognized RPC type') lookups = dict() mutations = [] for chunks in six.itervalues(lookup_rpcs): lookups.update([(key.SerializeToString(), key) for key in self._unpack_keys(b''.join(chunks))]) for rpc_info in mutation_rpcs: rpc_type = rpc_info[0] blob = b''.join(rpc_info[1:]) if rpc_type == self.PUTS: mutations.extend(self._unpack_entities(blob)) else: mutations.extend(self._unpack_keys(blob)) return list(six.itervalues(lookups)), queried_groups, mutations
def decode(self, kvs): """ Decodes Key-Values to a version entry. Args: kvs: An iterable containing KeyValue objects. Returns: A VersionEntry object. """ path = Path.unpack(kvs[0].key, self.path_slice.start)[0] commit_versionstamp = kvs[0].key[self.versionstamp_slice] first_index = ord(kvs[0].key[self.index_slice]) version = Int64.decode_bare(kvs[-1].value[self.version_slice]) if first_index == 0: encoded_entity = b''.join([kv.value for kv in kvs])[self.entity_slice] else: encoded_entity = VersionEntry.INCOMPLETE return VersionEntry(self.project_id, self.namespace, path, commit_versionstamp, version, encoded_entity)
def encode_key(self, path_prefix): scatter_byte = hash_tuple(path_prefix) encoded_path = Path.pack(path_prefix, omit_terminator=True, allow_partial=True) return self.directory.rawPrefix + scatter_byte + encoded_path
def group(self): group = entity_pb.Path() group.add_element().MergeFrom(Path.decode_element(self.path[:2])) return group
def decode(self, kv): path = Path.unpack(kv.key, len(self.directory.rawPrefix))[0] commit_versionstamp = kv.key[self.versionstamp_slice] deleted_versionstamp = kv.value or None return IndexEntry(self.project_id, self.namespace, path, commit_versionstamp, deleted_versionstamp)
def get_iterator(self, tr, query, read_versionstamp=None): project_id = decode_str(query.app()) namespace = decode_str(query.name_space()) filter_props = group_filters(query) ancestor_path = tuple() if query.has_ancestor(): ancestor_path = Path.flatten(query.ancestor().path()) start_cursor = None if query.has_compiled_cursor(): start_cursor = ListCursor(query)._GetLastResult() end_cursor = None if query.has_end_compiled_cursor(): end_compiled = query.end_compiled_cursor() end_cursor = ListCursor(query)._DecodeCompiledCursor( end_compiled)[0] rpc_limit, check_more_results = self.rpc_limit(query) fetch_limit = rpc_limit if check_more_results: fetch_limit += 1 if query.has_kind() and query.kind() == u'__namespace__': project_dir = yield self._directory_cache.get(tr, (project_id, )) raise gen.Return( NamespaceIterator(tr, self._tornado_fdb, project_dir)) elif query.has_kind() and query.kind() == u'__kind__': project_dir = yield self._directory_cache.get(tr, (project_id, )) raise gen.Return( KindIterator(tr, self._tornado_fdb, project_dir, namespace)) elif query.has_kind() and query.kind() == u'__property__': project_dir = yield self._directory_cache.get(tr, (project_id, )) raise gen.Return( PropertyIterator(tr, self._tornado_fdb, project_dir, namespace)) index = yield self._get_perfect_index(tr, query) reverse = get_scan_direction(query, index) == Query_Order.DESCENDING if index is None: if not all(prop.equality for prop in filter_props): raise BadRequest(u'Query not supported') indexes = [] equality_props = [ filter_prop for filter_prop in filter_props if filter_prop.name == KEY_PROP ] if len(equality_props) > 1: raise BadRequest(u'Only one equality key filter is supported') equality_prop = next(iter(equality_props), None) other_props = [ filter_prop for filter_prop in filter_props if filter_prop.name != KEY_PROP ] for filter_prop in other_props: index = yield self._single_prop_index(tr, project_id, namespace, decode_str(query.kind()), filter_prop.name) for op, value in filter_prop.filters: tmp_filter_prop = FilterProperty(filter_prop.name, [(op, value)]) if equality_prop is not None: tmp_filter_props = (tmp_filter_prop, equality_prop) else: tmp_filter_props = (tmp_filter_prop, ) slice = index.get_slice(tmp_filter_props, ancestor_path, start_cursor, end_cursor) indexes.append([index, slice, filter_prop.name, value]) raise gen.Return( MergeJoinIterator(tr, self._tornado_fdb, filter_props, indexes, fetch_limit, read_versionstamp, ancestor_path, snapshot=True)) equality_prop = next( (filter_prop for filter_prop in filter_props if filter_prop.equality), None) if equality_prop is not None and len(equality_prop.filters) > 1: indexes = [] for op, value in equality_prop.filters: tmp_filter_props = [] for filter_prop in filter_props: if filter_prop.name == equality_prop.name: tmp_filter_props.append( FilterProperty(filter_prop.name, [(op, value)])) else: tmp_filter_props.append(filter_prop) desired_slice = index.get_slice(tmp_filter_props, ancestor_path, start_cursor, end_cursor, reverse) indexes.append( [index, desired_slice, equality_prop.name, value]) raise gen.Return( MergeJoinIterator(tr, self._tornado_fdb, filter_props, indexes, fetch_limit, read_versionstamp, ancestor_path, snapshot=True)) desired_slice = index.get_slice(filter_props, ancestor_path, start_cursor, end_cursor, reverse) iterator = IndexIterator(tr, self._tornado_fdb, index, desired_slice, fetch_limit, reverse, read_versionstamp, snapshot=True) raise gen.Return(iterator)
def _encode_keys(self, keys): return b''.join( [Text.encode(decode_str(key.name_space())) + Path.pack(key.path()) for key in keys])