def apply_path_filter(self, op, path, ancestor_path=()):
if not isinstance(path, tuple):
path = Path.flatten(path)
remaining_path = path[len(ancestor_path):] if self._ancestor else path
if not remaining_path:
raise InternalError(u'Path filter must be within ancestor')
start = Path.pack(remaining_path, omit_terminator=True)
# Since the commit versionstamp could potentially start with 0xFF, this
# selection scans up to the next possible path value.
stop = start + six.int2byte(Path.MIN_ID_MARKER)
index = -2
if op == Query_Filter.EQUAL:
self._set_start(index, start)
self._set_stop(index, stop)
self._set_stop(index + 1, b'\xFF')
return
if op == Query_Filter.GREATER_THAN_OR_EQUAL:
self._set_start(index, start)
elif op == Query_Filter.GREATER_THAN:
self._set_start(index, stop)
elif op == Query_Filter.LESS_THAN_OR_EQUAL:
self._set_stop(index, stop)
elif op == Query_Filter.LESS_THAN:
self._set_stop(index, start)
else:
raise BadRequest(u'Unexpected filter operation')
def encode_key(self,
path,
original_versionstamp,
deleted_versionstamp=None):
""" Encodes a key for a deleted version index entry.
Args:
path: A tuple or protobuf path object.
original_versionstamp: A 10-byte string specifying the entity version's
original commit versionstamp.
deleted_versionstamp: A 10-byte string specifying the entity version's
delete versionstamp or None.
Returns:
A string containing an FDB key. If deleted_versionstamp was None, the key
should be used with set_versionstamped_key.
"""
if not isinstance(path, tuple):
path = Path.flatten(path)
scatter_byte = hash_tuple(path)
key = b''.join([
self.directory.rawPrefix, scatter_byte, deleted_versionstamp
or b'\x00' * VERSIONSTAMP_SIZE,
Path.pack(path), original_versionstamp
])
if not deleted_versionstamp:
versionstamp_index = len(
self.directory.rawPrefix) + len(scatter_byte)
key += encode_versionstamp_index(versionstamp_index)
return key
def encode_query_key(self, txid, namespace, ancestor_path):
if not isinstance(ancestor_path, tuple):
ancestor_path = Path.flatten(ancestor_path)
section_prefix = self._txid_prefix(txid) + self.QUERIES
encoded_ancestor = Text.encode(namespace) + Path.pack(ancestor_path[:2])
return section_prefix + encoded_ancestor
def encode_key(self, ancestor_path, encoded_values, remaining_path,
commit_versionstamp):
ancestor_path = Path.pack(ancestor_path) if ancestor_path else b''
remaining_path = Path.pack(remaining_path)
key = b''.join(
(self.directory.rawPrefix, ancestor_path) + tuple(encoded_values) +
(remaining_path, commit_versionstamp or b'\x00' * VERSIONSTAMP_SIZE))
if not commit_versionstamp:
key += encode_versionstamp_index(len(key) - VERSIONSTAMP_SIZE)
return key
def _encode_path_prefix(self, path):
""" Encodes the portion of the key up to and including the path.
Args:
path: A tuple or protobuf path object.
Returns:
A string containing the path prefix.
"""
if not isinstance(path, tuple):
path = Path.flatten(path)
return b''.join([self.directory.rawPrefix, hash_tuple(path),
Path.pack(path)])
def set_ancestor(self, ancestor_path):
if not ancestor_path:
return
index = 1 if self._ancestor else -2
if self._ancestor:
self._set_start(index, Path.pack(ancestor_path))
self._set_stop(index, Path.pack(ancestor_path))
self._set_stop(index + 1, b'\xFF')
else:
prefix = Path.pack(ancestor_path, omit_terminator=True)
self._set_start(index, prefix)
self._set_stop(index, prefix + b'\xFF')
def _unpack_keys(self, blob):
keys = []
pos = 0
while pos < len(blob):
namespace, pos = Text.decode(blob, pos)
path, pos = Path.unpack(blob, pos)
key = entity_pb.Reference()
key.set_app(self.project_id)
key.set_name_space(namespace)
key.mutable_path().MergeFrom(Path.decode(path))
keys.append(key)
return keys
def encode_key(self, path, commit_versionstamp):
key = b''.join([self.directory.rawPrefix, Path.pack(path),
commit_versionstamp or b'\x00' * VERSIONSTAMP_SIZE])
if not commit_versionstamp:
key += encode_versionstamp_index(len(key) - VERSIONSTAMP_SIZE)
return key
def format_prop_val(prop_value):
if prop_value.has_int64value():
return prop_value.int64value()
elif prop_value.has_booleanvalue():
return prop_value.booleanvalue()
elif prop_value.has_stringvalue():
return repr(prop_value.stringvalue())
elif prop_value.has_doublevalue():
return prop_value.doublevalue()
elif prop_value.has_pointvalue():
point_val = prop_value.pointvalue()
return point_val.x(), point_val.y()
elif prop_value.has_uservalue():
user_val = prop_value.uservalue()
details = {
'email': user_val.email(),
'auth_domain': user_val.auth_domain()
}
if user_val.has_nickname():
details['nickname'] = user_val.nickname()
if user_val.has_federated_identity():
details['federatedIdentity'] = user_val.federated_identity()
if user_val.has_federated_provider():
details['federatedProvider'] = user_val.federated_provider()
return json.dumps(details)
elif prop_value.has_referencevalue():
return Path.flatten(prop_value.referencevalue())
else:
return None
def get_latest(self,
tr,
key,
read_versionstamp=None,
include_data=True,
snapshot=False):
""" Gets the newest entity version for the given read versionstamp.
Args:
tr: An FDB transaction.
key: A protubuf reference object.
read_versionstamp: A 10-byte string specifying the FDB read versionstamp.
Newer versionstamps are ignored.
include_data: A boolean specifying whether or not to fetch all of the
entity's Key-Values.
snapshot: If True, the read will not cause a transaction conflict.
"""
data_ns = yield self._data_ns_from_key(tr, key)
desired_slice = data_ns.get_slice(key.path(),
read_versionstamp=read_versionstamp)
last_entry = yield self._last_version(tr,
data_ns,
desired_slice,
include_data,
snapshot=snapshot)
if last_entry is None:
last_entry = VersionEntry(data_ns.project_id, data_ns.namespace,
Path.flatten(key.path()))
raise gen.Return(last_entry)
def decode(self, kv):
value, pos = decode_value(kv.key, len(self.directory.rawPrefix))
path = Path.unpack(kv.key, pos)[0]
commit_versionstamp = kv.key[self.versionstamp_slice]
deleted_versionstamp = kv.value or None
return PropertyEntry(self.project_id, self.namespace, path, self.prop_name,
value, commit_versionstamp, deleted_versionstamp)
def key(self):
key = entity_pb.Reference()
key.set_app(self.project_id)
if self.namespace is not None:
key.set_name_space(self.namespace)
key.mutable_path().MergeFrom(Path.decode(self.path))
return key
def from_key(cls, key):
project_id = decode_str(key.app())
namespace = None
if key.has_name_space():
namespace = decode_str(key.name_space())
path = Path.flatten(key.path())
return cls(project_id, namespace, path)
def decode(self, kv):
pos = len(self.directory.rawPrefix)
properties = []
if self.ancestor:
ancestor_path, pos = Path.unpack(kv.key, pos)
else:
ancestor_path = ()
for prop_name, direction in self.order_info:
value, pos = decode_value(kv.key, pos,
direction == Query_Order.DESCENDING)
properties.append((prop_name, value))
remaining_path = Path.unpack(kv.key, pos)[0]
path = ancestor_path + remaining_path
commit_versionstamp = kv.key[self.versionstamp_slice]
deleted_versionstamp = kv.value or None
return CompositeEntry(self.project_id, self.namespace, path, properties,
commit_versionstamp, deleted_versionstamp)
def encode_key(self, group_path):
""" Encodes a key for a given entity group.
Args:
group_path: A tuple containing path elements.
Returns:
A byte string containing the relevant FDB key.
"""
return b''.join([
self.directory.rawPrefix, hash_tuple(group_path),
Text.encode(group_path[0]) + Path.encode_id_or_name(group_path[1])])
def encode_key(self, path):
""" Encodes a key for a safe read versionstamp entry.
Args:
path: A tuple or protobuf path object.
Returns:
A string containing an FDB key.
"""
if not isinstance(path, tuple):
path = Path.flatten(path)
entity_group = path[:2]
return self.directory.rawPrefix + hash_tuple(entity_group)
def decode(self, kv):
""" Decodes a KV to a DeletedVersionEntry.
Args:
kv: An FDB KeyValue object.
Returns:
A DeletedVersionEntry object.
"""
deleted_versionstamp = kv.key[self.del_versionstamp_slice]
path = Path.unpack(kv.key, self.path_slice.start)[0]
original_versionstamp = kv.key[self.original_versionstamp_slice]
return DeletedVersionEntry(self.project_id, self.namespace, path,
original_versionstamp, deleted_versionstamp)
def encode(self, path):
""" Creates a Key-Value tuple for updating a group's commit versionstamp.
Args:
path: A tuple or protobuf path object.
Returns:
A (key, value) tuple suitable for set_versionstamped_value.
"""
if not isinstance(path, tuple):
path = Path.flatten(path)
group_path = path[:2]
return (self.encode_key(group_path),
b'\x00' * VERSIONSTAMP_SIZE + encode_versionstamp_index(0))
def allocate_ids_request(self, app_id, http_request_data):
""" High level function for getting unique identifiers for entities.
Args:
app_id: Name of the application.
http_request_data: Stores the protocol buffer request from the
AppServer.
Returns:
Returns an encoded response.
Raises:
NotImplementedError: when requesting a max id.
"""
request = datastore_pb.AllocateIdsRequest(http_request_data)
if request.has_max() and request.has_size():
raise gen.Return(
('', datastore_pb.Error.BAD_REQUEST,
'Both size and max cannot be set.'))
if not (request.has_max() or request.has_size()):
raise gen.Return(
('', datastore_pb.Error.BAD_REQUEST,
'Either size or max must be set.'))
if not request.has_model_key():
raise gen.Return(('', datastore_pb.Error.BAD_REQUEST,
'Model key must be set'))
namespace = six.text_type(request.model_key().name_space())
path_prefix = Path.flatten(request.model_key().path(),
allow_partial=True)[:-1]
if request.has_size():
coroutine = datastore_access.allocate_size
args = (app_id, namespace, path_prefix, request.size())
else:
coroutine = datastore_access.allocate_max
args = (app_id, namespace, path_prefix, request.max())
try:
start, end = yield coroutine(*args)
except dbconstants.AppScaleBadArg as error:
raise gen.Return(('', datastore_pb.Error.BAD_REQUEST, str(error)))
except dbconstants.AppScaleDBConnectionError as error:
raise gen.Return(('', datastore_pb.Error.INTERNAL_ERROR, str(error)))
response = datastore_pb.AllocateIdsResponse()
response.set_start(start)
response.set_end(end)
raise gen.Return((response.Encode(), 0, ''))
def _get_index_keys(self, tr, entity, commit_versionstamp=None):
has_index = commit_versionstamp is None
project_id = decode_str(entity.key().app())
namespace = decode_str(entity.key().name_space())
path = Path.flatten(entity.key().path())
kind = path[-2]
stats = IndexStatsSummary()
kindless_index = yield self._kindless_index(tr, project_id, namespace)
kind_index = yield self._kind_index(tr, project_id, namespace, kind)
composite_indexes = yield self._get_indexes(tr, project_id, namespace,
kind)
kindless_key = kindless_index.encode_key(path, commit_versionstamp)
kind_key = kind_index.encode_key(path, commit_versionstamp)
stats.add_kindless_key(kindless_key, has_index)
stats.add_kind_key(kind_key, has_index)
all_keys = [kindless_key, kind_key]
entity_prop_names = []
for prop in entity.property_list():
prop_name = decode_str(prop.name())
entity_prop_names.append(prop_name)
index = yield self._single_prop_index(tr, project_id, namespace,
kind, prop_name)
prop_key = index.encode_key(prop.value(), path,
commit_versionstamp)
stats.add_prop_key(prop, prop_key, has_index)
all_keys.append(prop_key)
scatter_val = get_scatter_val(path)
if scatter_val is not None:
index = yield self._single_prop_index(tr, project_id, namespace,
kind, SCATTER_PROP)
all_keys.append(
index.encode_key(scatter_val, path, commit_versionstamp))
for index in composite_indexes:
# If the entity does not have the relevant props for the index, skip it.
if not all(index_prop_name in entity_prop_names
for index_prop_name in index.prop_names):
continue
composite_keys = index.encode_keys(entity.property_list(), path,
commit_versionstamp)
stats.add_composite_keys(index.id, composite_keys, has_index)
all_keys.extend(composite_keys)
raise gen.Return((all_keys, stats))
def _enforce_max_groups(mutations):
""" Raises an exception if too many groups were modified. """
mutated_groups = set()
for mutation in mutations:
if isinstance(mutation, entity_pb.Reference):
key = mutation
else:
key = mutation.key()
namespace = decode_str(key.name_space())
flat_group = (namespace, ) + Path.flatten(key.path())[:2]
mutated_groups.add(flat_group)
if len(mutated_groups) > 25:
raise BadRequest(
u'Too many entity groups modified in transaction')
def decode_metadata(self, txid, kvs):
lookup_rpcs = defaultdict(list)
queried_groups = set()
mutation_rpcs = []
rpc_type_index = len(self._txid_prefix(txid))
current_versionstamp = None
for kv in kvs:
rpc_type = kv.key[rpc_type_index]
pos = rpc_type_index + 1
if rpc_type == self.QUERIES:
namespace, pos = Text.decode(kv.key, pos)
group_path = Path.unpack(kv.key, pos)[0]
queried_groups.add((namespace, group_path))
continue
rpc_versionstamp = kv.key[pos:pos + VERSIONSTAMP_SIZE]
if rpc_type == self.LOOKUPS:
lookup_rpcs[rpc_versionstamp].append(kv.value)
elif rpc_type in (self.PUTS, self.DELETES):
if current_versionstamp == rpc_versionstamp:
mutation_rpcs[-1].append(kv.value)
else:
current_versionstamp = rpc_versionstamp
mutation_rpcs.append([rpc_type, kv.value])
else:
raise InternalError(u'Unrecognized RPC type')
lookups = dict()
mutations = []
for chunks in six.itervalues(lookup_rpcs):
lookups.update([(key.SerializeToString(), key)
for key in self._unpack_keys(b''.join(chunks))])
for rpc_info in mutation_rpcs:
rpc_type = rpc_info[0]
blob = b''.join(rpc_info[1:])
if rpc_type == self.PUTS:
mutations.extend(self._unpack_entities(blob))
else:
mutations.extend(self._unpack_keys(blob))
return list(six.itervalues(lookups)), queried_groups, mutations
def decode(self, kvs):
""" Decodes Key-Values to a version entry.
Args:
kvs: An iterable containing KeyValue objects.
Returns:
A VersionEntry object.
"""
path = Path.unpack(kvs[0].key, self.path_slice.start)[0]
commit_versionstamp = kvs[0].key[self.versionstamp_slice]
first_index = ord(kvs[0].key[self.index_slice])
version = Int64.decode_bare(kvs[-1].value[self.version_slice])
if first_index == 0:
encoded_entity = b''.join([kv.value for kv in kvs])[self.entity_slice]
else:
encoded_entity = VersionEntry.INCOMPLETE
return VersionEntry(self.project_id, self.namespace, path,
commit_versionstamp, version, encoded_entity)
def encode_key(self, path_prefix):
scatter_byte = hash_tuple(path_prefix)
encoded_path = Path.pack(path_prefix,
omit_terminator=True,
allow_partial=True)
return self.directory.rawPrefix + scatter_byte + encoded_path
def group(self):
group = entity_pb.Path()
group.add_element().MergeFrom(Path.decode_element(self.path[:2]))
return group
def decode(self, kv):
path = Path.unpack(kv.key, len(self.directory.rawPrefix))[0]
commit_versionstamp = kv.key[self.versionstamp_slice]
deleted_versionstamp = kv.value or None
return IndexEntry(self.project_id, self.namespace, path,
commit_versionstamp, deleted_versionstamp)
def get_iterator(self, tr, query, read_versionstamp=None):
project_id = decode_str(query.app())
namespace = decode_str(query.name_space())
filter_props = group_filters(query)
ancestor_path = tuple()
if query.has_ancestor():
ancestor_path = Path.flatten(query.ancestor().path())
start_cursor = None
if query.has_compiled_cursor():
start_cursor = ListCursor(query)._GetLastResult()
end_cursor = None
if query.has_end_compiled_cursor():
end_compiled = query.end_compiled_cursor()
end_cursor = ListCursor(query)._DecodeCompiledCursor(
end_compiled)[0]
rpc_limit, check_more_results = self.rpc_limit(query)
fetch_limit = rpc_limit
if check_more_results:
fetch_limit += 1
if query.has_kind() and query.kind() == u'__namespace__':
project_dir = yield self._directory_cache.get(tr, (project_id, ))
raise gen.Return(
NamespaceIterator(tr, self._tornado_fdb, project_dir))
elif query.has_kind() and query.kind() == u'__kind__':
project_dir = yield self._directory_cache.get(tr, (project_id, ))
raise gen.Return(
KindIterator(tr, self._tornado_fdb, project_dir, namespace))
elif query.has_kind() and query.kind() == u'__property__':
project_dir = yield self._directory_cache.get(tr, (project_id, ))
raise gen.Return(
PropertyIterator(tr, self._tornado_fdb, project_dir,
namespace))
index = yield self._get_perfect_index(tr, query)
reverse = get_scan_direction(query, index) == Query_Order.DESCENDING
if index is None:
if not all(prop.equality for prop in filter_props):
raise BadRequest(u'Query not supported')
indexes = []
equality_props = [
filter_prop for filter_prop in filter_props
if filter_prop.name == KEY_PROP
]
if len(equality_props) > 1:
raise BadRequest(u'Only one equality key filter is supported')
equality_prop = next(iter(equality_props), None)
other_props = [
filter_prop for filter_prop in filter_props
if filter_prop.name != KEY_PROP
]
for filter_prop in other_props:
index = yield self._single_prop_index(tr, project_id,
namespace,
decode_str(query.kind()),
filter_prop.name)
for op, value in filter_prop.filters:
tmp_filter_prop = FilterProperty(filter_prop.name,
[(op, value)])
if equality_prop is not None:
tmp_filter_props = (tmp_filter_prop, equality_prop)
else:
tmp_filter_props = (tmp_filter_prop, )
slice = index.get_slice(tmp_filter_props, ancestor_path,
start_cursor, end_cursor)
indexes.append([index, slice, filter_prop.name, value])
raise gen.Return(
MergeJoinIterator(tr,
self._tornado_fdb,
filter_props,
indexes,
fetch_limit,
read_versionstamp,
ancestor_path,
snapshot=True))
equality_prop = next(
(filter_prop
for filter_prop in filter_props if filter_prop.equality), None)
if equality_prop is not None and len(equality_prop.filters) > 1:
indexes = []
for op, value in equality_prop.filters:
tmp_filter_props = []
for filter_prop in filter_props:
if filter_prop.name == equality_prop.name:
tmp_filter_props.append(
FilterProperty(filter_prop.name, [(op, value)]))
else:
tmp_filter_props.append(filter_prop)
desired_slice = index.get_slice(tmp_filter_props,
ancestor_path, start_cursor,
end_cursor, reverse)
indexes.append(
[index, desired_slice, equality_prop.name, value])
raise gen.Return(
MergeJoinIterator(tr,
self._tornado_fdb,
filter_props,
indexes,
fetch_limit,
read_versionstamp,
ancestor_path,
snapshot=True))
desired_slice = index.get_slice(filter_props, ancestor_path,
start_cursor, end_cursor, reverse)
iterator = IndexIterator(tr,
self._tornado_fdb,
index,
desired_slice,
fetch_limit,
reverse,
read_versionstamp,
snapshot=True)
raise gen.Return(iterator)
def _encode_keys(self, keys):
return b''.join(
[Text.encode(decode_str(key.name_space())) + Path.pack(key.path())
for key in keys])
