def get_index_kv_from_tuple(tuple_list, reverse=False):
""" Returns keys/value of indexes for a set of entities.
Args:
tuple_list: A list of tuples of prefix and pb entities
reverse: if these keys are for the descending table
Returns:
A list of keys and values of indexes
"""
all_rows = []
for prefix, e in tuple_list:
for p in e.property_list():
val = str(encode_index_pb(p.value()))
if reverse:
val = helper_functions.reverse_lex(val)
params = [prefix,
get_entity_kind(e),
p.name(),
val,
str(encode_index_pb(e.key().path()))]
index_key = get_index_key_from_params(params)
p_vals = [index_key,
buffer(prefix + dbconstants.KEY_DELIMITER) + \
encode_index_pb(e.key().path())]
all_rows.append(p_vals)
return tuple(ii for ii in all_rows)
def get_index_kv_from_tuple(tuple_list, reverse=False):
""" Returns keys/value of indexes for a set of entities.
Args:
tuple_list: A list of tuples of prefix and pb entities
reverse: if these keys are for the descending table
Returns:
A list of keys and values of indexes
"""
all_rows = []
for prefix, e in tuple_list:
for p in e.property_list():
val = str(encode_index_pb(p.value()))
if reverse:
val = helper_functions.reverse_lex(val)
params = [
prefix,
get_entity_kind(e),
p.name(), val,
str(encode_index_pb(e.key().path()))
]
index_key = get_index_key_from_params(params)
p_vals = [index_key,
buffer(prefix + dbconstants.KEY_DELIMITER) + \
encode_index_pb(e.key().path())]
all_rows.append(p_vals)
return tuple(ii for ii in all_rows)
def index_deletions(old_entity, new_entity, composite_indices=()):
""" Get a list of index deletions needed for updating an entity. For changing
an existing entity, this involves examining the property list of both
entities to see which index entries need to be removed.
Args:
old_entity: An entity object.
new_entity: An entity object.
composite_indices: A list or tuple of composite indices.
Returns:
A list of dictionaries representing mutation operations.
"""
ds_static = datastore_server.DatastoreDistributed
deletions = []
app_id = datastore_server.clean_app_id(old_entity.key().app())
namespace = old_entity.key().name_space()
kind = ds_static.get_entity_kind(old_entity.key())
entity_key = str(ds_static.encode_index_pb(old_entity.key().path()))
new_props = {}
for prop in new_entity.property_list():
if prop.name() not in new_props:
new_props[prop.name()] = []
new_props[prop.name()].append(prop)
changed_props = {}
for prop in old_entity.property_list():
if prop.name() in new_props and prop in new_props[prop.name()]:
continue
if prop.name() not in changed_props:
changed_props[prop.name()] = []
changed_props[prop.name()].append(prop)
value = str(ds_static.encode_index_pb(prop.value()))
key = dbconstants.KEY_DELIMITER.join([app_id, namespace, kind, prop.name(), value, entity_key])
deletions.append({"table": dbconstants.ASC_PROPERTY_TABLE, "key": key, "operation": TxnActions.DELETE})
reverse_key = dbconstants.KEY_DELIMITER.join(
[app_id, namespace, kind, prop.name(), helper_functions.reverse_lex(value), entity_key]
)
deletions.append({"table": dbconstants.DSC_PROPERTY_TABLE, "key": reverse_key, "operation": TxnActions.DELETE})
changed_prop_names = set(changed_props.keys())
for index in composite_indices:
if index.definition().entity_type() != kind:
continue
index_props = set(prop.name() for prop in index.definition().property_list())
if index_props.isdisjoint(changed_prop_names):
continue
old_entries = set(ds_static.get_composite_index_keys(index, old_entity))
new_entries = set(ds_static.get_composite_index_keys(index, new_entity))
for entry in old_entries - new_entries:
deletions.append({"table": dbconstants.COMPOSITE_TABLE, "key": entry, "operation": TxnActions.DELETE})
return deletions
def get_index_kv_from_tuple(tuple_list, reverse=False):
""" Returns keys/value of indexes for a set of entities.
Args:
tuple_list: A list of tuples of prefix and pb entities
reverse: if these keys are for the descending table
Returns:
A list of keys and values of indexes
"""
all_rows = []
for prefix, entity in tuple_list:
# Give some entities a property that makes it easy to sample keys.
scatter_prop = get_scatter_prop(entity.key().path().element_list())
if scatter_prop is not None:
# Prevent the original entity from being modified.
prop_list = [prop for prop in entity.property_list()] + [scatter_prop]
else:
prop_list = entity.property_list()
for prop in prop_list:
val = str(encode_index_pb(prop.value()))
if reverse:
val = helper_functions.reverse_lex(val)
params = [prefix,
get_entity_kind(entity),
prop.name(),
val,
str(encode_index_pb(entity.key().path()))]
index_key = get_index_key_from_params(params)
p_vals = [index_key,
buffer(prefix + dbconstants.KEY_DELIMITER) + \
encode_index_pb(entity.key().path())]
all_rows.append(p_vals)
return tuple(all_rows)
def get_composite_index_keys(index, entity):
""" Creates keys to the composite index table for a given entity.
Keys are built as such:
app_id/ns/composite_id/ancestor/valuevaluevalue..../entity_key
Components explained:
ns: The namespace of the entity.
composite_id: The composite ID assigned to this index upon creation.
ancestor: The root ancestor path (only if the query this index is for
has an ancestor)
value(s): The string representation of mulitiple properties.
entity_key: The entity key (full path) used as a means of having a unique
identifier. This prevents two entities with the same values from
colliding.
Args:
index: A datastore_pb.CompositeIndex.
entity: A entity_pb.EntityProto.
Returns:
A list of strings representing keys to the composite table.
"""
composite_id = index.id()
definition = index.definition()
app_id = clean_app_id(entity.key().app())
name_space = entity.key().name_space()
ent_key = encode_index_pb(entity.key().path())
pre_comp_index_key = "{0}{1}{2}{4}{3}{4}".format(app_id,
dbconstants.KEY_DELIMITER,
name_space, composite_id,
dbconstants.KEY_DELIMITER)
if definition.ancestor() == 1:
ancestor_list = get_ancestor_paths_from_ent_key(ent_key)
property_list_names = [prop.name() for prop in entity.property_list()]
multivalue_dict = {}
for prop in entity.property_list():
if prop.name() not in property_list_names:
continue
value = str(encode_index_pb(prop.value()))
if prop.name() in multivalue_dict:
multivalue_dict[prop.name()].append(value)
else:
multivalue_dict[prop.name()] = [value]
# Build lists for which we'll get all combinations of indexes.
lists_of_prop_list = []
for prop in definition.property_list():
# Check to make sure the entity has the required items. If not then we
# do not create an index for the composite index.
# The definition can also have a key as a part of the index, but this
# is not repeated.
if prop.name() == "__key__":
value = str(encode_index_pb(entity.key().path()))
if prop.direction() == entity_pb.Index_Property.DESCENDING:
value = helper_functions.reverse_lex(value)
lists_of_prop_list.append([value])
elif prop.name() not in multivalue_dict:
return []
else:
my_list = multivalue_dict[prop.name()]
if prop.direction() == entity_pb.Index_Property.DESCENDING:
for index, item in enumerate(my_list):
my_list[index] = helper_functions.reverse_lex(item)
lists_of_prop_list.append(my_list)
# Get all combinations of the composite indexes.
all_combinations = []
if len(lists_of_prop_list) == 1:
for item in lists_of_prop_list[0]:
all_combinations.append([item])
elif len(lists_of_prop_list) > 1:
all_combinations = list(itertools.product(*lists_of_prop_list))
# We should throw an exception if the number of combinations is
# more than 20000. We currently do not.
# https://developers.google.com/appengine/docs/python/datastore/
# #Python_Quotas_and_limits
all_keys = []
for combo in all_combinations:
index_value = ""
for prop_value in combo:
index_value += str(prop_value) + dbconstants.KEY_DELIMITER
# We append the ent key to have unique keys if entities happen
# to share the same index values (and ancestor).
if definition.ancestor() == 1:
for ancestor in ancestor_list:
pre_comp_key = pre_comp_index_key + "{0}{1}".format(
ancestor, dbconstants.KEY_DELIMITER)
composite_key = "{0}{1}{2}".format(pre_comp_key, index_value,
ent_key)
all_keys.append(composite_key)
else:
composite_key = "{0}{1}{2}".format(pre_comp_index_key, index_value,
ent_key)
all_keys.append(composite_key)
return all_keys
def get_composite_index_keys(index, entity):
""" Creates keys to the composite index table for a given entity.
Keys are built as such:
app_id/ns/composite_id/ancestor/valuevaluevalue..../entity_key
Components explained:
ns: The namespace of the entity.
composite_id: The composite ID assigned to this index upon creation.
ancestor: The root ancestor path (only if the query this index is for
has an ancestor)
value(s): The string representation of mulitiple properties.
entity_key: The entity key (full path) used as a means of having a unique
identifier. This prevents two entities with the same values from
colliding.
Args:
index: A datastore_pb.CompositeIndex.
entity: A entity_pb.EntityProto.
Returns:
A list of strings representing keys to the composite table.
"""
composite_id = index.id()
definition = index.definition()
app_id = clean_app_id(entity.key().app())
name_space = entity.key().name_space()
ent_key = encode_index_pb(entity.key().path())
pre_comp_index_key = "{0}{1}{2}{4}{3}{4}".format(app_id,
dbconstants.KEY_DELIMITER,
name_space, composite_id,
dbconstants.KEY_DELIMITER)
if definition.ancestor() == 1:
ancestor_list = get_ancestor_paths_from_ent_key(ent_key)
property_list_names = [prop.name() for prop in entity.property_list()]
multivalue_dict = {}
for prop in entity.property_list():
if prop.name() not in property_list_names:
continue
value = str(encode_index_pb(prop.value()))
if prop.name() in multivalue_dict:
multivalue_dict[prop.name()].append(value)
else:
multivalue_dict[prop.name()] = [value]
# Build lists for which we'll get all combinations of indexes.
lists_of_prop_list = []
for prop in definition.property_list():
# Check to make sure the entity has the required items. If not then we
# do not create an index for the composite index.
# The definition can also have a key as a part of the index, but this
# is not repeated.
if prop.name() == "__key__":
value = str(encode_index_pb(entity.key().path()))
if prop.direction() == entity_pb.Index_Property.DESCENDING:
value = helper_functions.reverse_lex(value)
lists_of_prop_list.append([value])
elif prop.name() not in multivalue_dict:
return []
else:
my_list = multivalue_dict[prop.name()]
if prop.direction() == entity_pb.Index_Property.DESCENDING:
for index, item in enumerate(my_list):
my_list[index] = helper_functions.reverse_lex(item)
lists_of_prop_list.append(my_list)
# Get all combinations of the composite indexes.
all_combinations = []
if len(lists_of_prop_list) == 1:
for item in lists_of_prop_list[0]:
all_combinations.append([item])
elif len(lists_of_prop_list) > 1:
all_combinations = list(itertools.product(*lists_of_prop_list))
# We should throw an exception if the number of combinations is
# more than 20000. We currently do not.
# https://developers.google.com/appengine/docs/python/datastore/
# #Python_Quotas_and_limits
all_keys = []
for combo in all_combinations:
index_value = ""
for prop_value in combo:
index_value += str(prop_value) + dbconstants.KEY_DELIMITER
# We append the ent key to have unique keys if entities happen
# to share the same index values (and ancestor).
if definition.ancestor() == 1:
for ancestor in ancestor_list:
pre_comp_key = pre_comp_index_key + "{0}{1}".format(
ancestor, dbconstants.KEY_DELIMITER)
composite_key = "{0}{1}{2}".format(pre_comp_key, index_value,
ent_key)
all_keys.append(composite_key)
else:
composite_key = "{0}{1}{2}".format(pre_comp_index_key, index_value,
ent_key)
all_keys.append(composite_key)
return all_keys
def index_deletions(old_entity, new_entity, composite_indices=()):
""" Get a list of index deletions needed for updating an entity. For changing
an existing entity, this involves examining the property list of both
entities to see which index entries need to be removed.
Args:
old_entity: An entity object.
new_entity: An entity object.
composite_indices: A list or tuple of composite indices.
Returns:
A list of dictionaries representing mutation operations.
"""
ds_static = datastore_server.DatastoreDistributed
deletions = []
app_id = datastore_server.clean_app_id(old_entity.key().app())
namespace = old_entity.key().name_space()
kind = ds_static.get_entity_kind(old_entity.key())
entity_key = str(ds_static.encode_index_pb(old_entity.key().path()))
new_props = {}
for prop in new_entity.property_list():
if prop.name() not in new_props:
new_props[prop.name()] = []
new_props[prop.name()].append(prop)
changed_props = {}
for prop in old_entity.property_list():
if prop.name() in new_props and prop in new_props[prop.name()]:
continue
if prop.name() not in changed_props:
changed_props[prop.name()] = []
changed_props[prop.name()].append(prop)
value = str(ds_static.encode_index_pb(prop.value()))
key = dbconstants.KEY_DELIMITER.join(
[app_id, namespace, kind,
prop.name(), value, entity_key])
deletions.append({
'table': dbconstants.ASC_PROPERTY_TABLE,
'key': key,
'operation': TxnActions.DELETE
})
reverse_key = dbconstants.KEY_DELIMITER.join([
app_id, namespace, kind,
prop.name(),
helper_functions.reverse_lex(value), entity_key
])
deletions.append({
'table': dbconstants.DSC_PROPERTY_TABLE,
'key': reverse_key,
'operation': TxnActions.DELETE
})
changed_prop_names = set(changed_props.keys())
for index in composite_indices:
if index.definition().entity_type() != kind:
continue
index_props = set(prop.name()
for prop in index.definition().property_list())
if index_props.isdisjoint(changed_prop_names):
continue
old_entries = set(ds_static.get_composite_index_keys(
index, old_entity))
new_entries = set(ds_static.get_composite_index_keys(
index, new_entity))
for entry in (old_entries - new_entries):
deletions.append({
'table': dbconstants.COMPOSITE_TABLE,
'key': entry,
'operation': TxnActions.DELETE
})
return deletions
