def get_index_kv_from_tuple(tuple_list, reverse=False):
""" Returns keys/value of indexes for a set of entities.
Args:
tuple_list: A list of tuples of prefix and pb entities
reverse: if these keys are for the descending table
Returns:
A list of keys and values of indexes
"""
all_rows = []
for prefix, entity in tuple_list:
# Give some entities a property that makes it easy to sample keys.
scatter_prop = get_scatter_prop(entity.key().path().element_list())
if scatter_prop is not None:
# Prevent the original entity from being modified.
prop_list = [prop
for prop in entity.property_list()] + [scatter_prop]
else:
prop_list = entity.property_list()
for prop in prop_list:
val = str(encode_index_pb(prop.value()))
if reverse:
val = helper_functions.reverse_lex(val)
params = [
prefix,
get_entity_kind(entity),
prop.name(), val,
str(encode_index_pb(entity.key().path()))
]
index_key = get_index_key_from_params(params)
p_vals = [index_key,
buffer(prefix + dbconstants.KEY_DELIMITER) + \
encode_index_pb(entity.key().path())]
all_rows.append(p_vals)
return tuple(all_rows)
def get_index_kv_from_tuple(tuple_list, reverse=False):
""" Returns keys/value of indexes for a set of entities.
Args:
tuple_list: A list of tuples of prefix and pb entities
reverse: if these keys are for the descending table
Returns:
A list of keys and values of indexes
"""
all_rows = []
for prefix, entity in tuple_list:
# Give some entities a property that makes it easy to sample keys.
scatter_prop = get_scatter_prop(entity.key().path().element_list())
if scatter_prop is not None:
# Prevent the original entity from being modified.
prop_list = [prop for prop in entity.property_list()] + [scatter_prop]
else:
prop_list = entity.property_list()
for prop in prop_list:
val = str(encode_index_pb(prop.value()))
if reverse:
val = helper_functions.reverse_lex(val)
params = [prefix,
get_entity_kind(entity),
prop.name(),
val,
str(encode_index_pb(entity.key().path()))]
index_key = get_index_key_from_params(params)
p_vals = [index_key,
buffer(prefix + dbconstants.KEY_DELIMITER) + \
encode_index_pb(entity.key().path())]
all_rows.append(p_vals)
return tuple(all_rows)
def get_composite_index_keys(index, entity):
""" Creates keys to the composite index table for a given entity.
Keys are built as such:
app_id/ns/composite_id/ancestor/valuevaluevalue..../entity_key
Components explained:
ns: The namespace of the entity.
composite_id: The composite ID assigned to this index upon creation.
ancestor: The root ancestor path (only if the query this index is for
has an ancestor)
value(s): The string representation of mulitiple properties.
entity_key: The entity key (full path) used as a means of having a unique
identifier. This prevents two entities with the same values from
colliding.
Args:
index: A datastore_pb.CompositeIndex.
entity: A entity_pb.EntityProto.
Returns:
A list of strings representing keys to the composite table.
"""
composite_id = index.id()
definition = index.definition()
app_id = clean_app_id(entity.key().app())
name_space = entity.key().name_space()
ent_key = encode_index_pb(entity.key().path())
pre_comp_index_key = "{0}{1}{2}{4}{3}{4}".format(app_id,
dbconstants.KEY_DELIMITER,
name_space, composite_id,
dbconstants.KEY_DELIMITER)
if definition.ancestor() == 1:
ancestor_list = get_ancestor_paths_from_ent_key(ent_key)
property_list_names = [prop.name() for prop in entity.property_list()]
multivalue_dict = {}
for prop in entity.property_list():
if prop.name() not in property_list_names:
continue
value = str(encode_index_pb(prop.value()))
if prop.name() in multivalue_dict:
multivalue_dict[prop.name()].append(value)
else:
multivalue_dict[prop.name()] = [value]
# Build lists for which we'll get all combinations of indexes.
lists_of_prop_list = []
for prop in definition.property_list():
# Check to make sure the entity has the required items. If not then we
# do not create an index for the composite index.
# The definition can also have a key as a part of the index, but this
# is not repeated.
if prop.name() == "__key__":
value = str(encode_index_pb(entity.key().path()))
if prop.direction() == entity_pb.Index_Property.DESCENDING:
value = helper_functions.reverse_lex(value)
lists_of_prop_list.append([value])
elif prop.name() not in multivalue_dict:
return []
else:
my_list = multivalue_dict[prop.name()]
if prop.direction() == entity_pb.Index_Property.DESCENDING:
for index, item in enumerate(my_list):
my_list[index] = helper_functions.reverse_lex(item)
lists_of_prop_list.append(my_list)
# Get all combinations of the composite indexes.
all_combinations = []
if len(lists_of_prop_list) == 1:
for item in lists_of_prop_list[0]:
all_combinations.append([item])
elif len(lists_of_prop_list) > 1:
all_combinations = list(itertools.product(*lists_of_prop_list))
# We should throw an exception if the number of combinations is
# more than 20000. We currently do not.
# https://developers.google.com/appengine/docs/python/datastore/
# #Python_Quotas_and_limits
all_keys = []
for combo in all_combinations:
index_value = ""
for prop_value in combo:
index_value += str(prop_value) + dbconstants.KEY_DELIMITER
# We append the ent key to have unique keys if entities happen
# to share the same index values (and ancestor).
if definition.ancestor() == 1:
for ancestor in ancestor_list:
pre_comp_key = pre_comp_index_key + "{0}{1}".format(
ancestor, dbconstants.KEY_DELIMITER)
composite_key = "{0}{1}{2}".format(pre_comp_key, index_value,
ent_key)
all_keys.append(composite_key)
else:
composite_key = "{0}{1}{2}".format(pre_comp_index_key, index_value,
ent_key)
all_keys.append(composite_key)
return all_keys
def get_composite_index_keys(index, entity):
""" Creates keys to the composite index table for a given entity.
Keys are built as such:
app_id/ns/composite_id/ancestor/valuevaluevalue..../entity_key
Components explained:
ns: The namespace of the entity.
composite_id: The composite ID assigned to this index upon creation.
ancestor: The root ancestor path (only if the query this index is for
has an ancestor)
value(s): The string representation of mulitiple properties.
entity_key: The entity key (full path) used as a means of having a unique
identifier. This prevents two entities with the same values from
colliding.
Args:
index: A datastore_pb.CompositeIndex.
entity: A entity_pb.EntityProto.
Returns:
A list of strings representing keys to the composite table.
"""
composite_id = index.id()
definition = index.definition()
app_id = clean_app_id(entity.key().app())
name_space = entity.key().name_space()
ent_key = encode_index_pb(entity.key().path())
pre_comp_index_key = "{0}{1}{2}{4}{3}{4}".format(app_id,
dbconstants.KEY_DELIMITER,
name_space, composite_id,
dbconstants.KEY_DELIMITER)
if definition.ancestor() == 1:
ancestor_list = get_ancestor_paths_from_ent_key(ent_key)
property_list_names = [prop.name() for prop in entity.property_list()]
multivalue_dict = {}
for prop in entity.property_list():
if prop.name() not in property_list_names:
continue
value = str(encode_index_pb(prop.value()))
if prop.name() in multivalue_dict:
multivalue_dict[prop.name()].append(value)
else:
multivalue_dict[prop.name()] = [value]
# Build lists for which we'll get all combinations of indexes.
lists_of_prop_list = []
for prop in definition.property_list():
# Check to make sure the entity has the required items. If not then we
# do not create an index for the composite index.
# The definition can also have a key as a part of the index, but this
# is not repeated.
if prop.name() == "__key__":
value = str(encode_index_pb(entity.key().path()))
if prop.direction() == entity_pb.Index_Property.DESCENDING:
value = helper_functions.reverse_lex(value)
lists_of_prop_list.append([value])
elif prop.name() not in multivalue_dict:
return []
else:
my_list = multivalue_dict[prop.name()]
if prop.direction() == entity_pb.Index_Property.DESCENDING:
for index, item in enumerate(my_list):
my_list[index] = helper_functions.reverse_lex(item)
lists_of_prop_list.append(my_list)
# Get all combinations of the composite indexes.
all_combinations = []
if len(lists_of_prop_list) == 1:
for item in lists_of_prop_list[0]:
all_combinations.append([item])
elif len(lists_of_prop_list) > 1:
all_combinations = list(itertools.product(*lists_of_prop_list))
# We should throw an exception if the number of combinations is
# more than 20000. We currently do not.
# https://developers.google.com/appengine/docs/python/datastore/
# #Python_Quotas_and_limits
all_keys = []
for combo in all_combinations:
index_value = ""
for prop_value in combo:
index_value += str(prop_value) + dbconstants.KEY_DELIMITER
# We append the ent key to have unique keys if entities happen
# to share the same index values (and ancestor).
if definition.ancestor() == 1:
for ancestor in ancestor_list:
pre_comp_key = pre_comp_index_key + "{0}{1}".format(
ancestor, dbconstants.KEY_DELIMITER)
composite_key = "{0}{1}{2}".format(pre_comp_key, index_value,
ent_key)
all_keys.append(composite_key)
else:
composite_key = "{0}{1}{2}".format(pre_comp_index_key, index_value,
ent_key)
all_keys.append(composite_key)
return all_keys
