def get_index_kv_from_tuple(tuple_list, reverse=False): """ Returns keys/value of indexes for a set of entities. Args: tuple_list: A list of tuples of prefix and pb entities reverse: if these keys are for the descending table Returns: A list of keys and values of indexes """ all_rows = [] for prefix, entity in tuple_list: # Give some entities a property that makes it easy to sample keys. scatter_prop = get_scatter_prop(entity.key().path().element_list()) if scatter_prop is not None: # Prevent the original entity from being modified. prop_list = [prop for prop in entity.property_list()] + [scatter_prop] else: prop_list = entity.property_list() for prop in prop_list: val = str(encode_index_pb(prop.value())) if reverse: val = helper_functions.reverse_lex(val) params = [ prefix, get_entity_kind(entity), prop.name(), val, str(encode_index_pb(entity.key().path())) ] index_key = get_index_key_from_params(params) p_vals = [index_key, buffer(prefix + dbconstants.KEY_DELIMITER) + \ encode_index_pb(entity.key().path())] all_rows.append(p_vals) return tuple(all_rows)
def get_index_kv_from_tuple(tuple_list, reverse=False): """ Returns keys/value of indexes for a set of entities. Args: tuple_list: A list of tuples of prefix and pb entities reverse: if these keys are for the descending table Returns: A list of keys and values of indexes """ all_rows = [] for prefix, entity in tuple_list: # Give some entities a property that makes it easy to sample keys. scatter_prop = get_scatter_prop(entity.key().path().element_list()) if scatter_prop is not None: # Prevent the original entity from being modified. prop_list = [prop for prop in entity.property_list()] + [scatter_prop] else: prop_list = entity.property_list() for prop in prop_list: val = str(encode_index_pb(prop.value())) if reverse: val = helper_functions.reverse_lex(val) params = [prefix, get_entity_kind(entity), prop.name(), val, str(encode_index_pb(entity.key().path()))] index_key = get_index_key_from_params(params) p_vals = [index_key, buffer(prefix + dbconstants.KEY_DELIMITER) + \ encode_index_pb(entity.key().path())] all_rows.append(p_vals) return tuple(all_rows)
def get_composite_index_keys(index, entity): """ Creates keys to the composite index table for a given entity. Keys are built as such: app_id/ns/composite_id/ancestor/valuevaluevalue..../entity_key Components explained: ns: The namespace of the entity. composite_id: The composite ID assigned to this index upon creation. ancestor: The root ancestor path (only if the query this index is for has an ancestor) value(s): The string representation of mulitiple properties. entity_key: The entity key (full path) used as a means of having a unique identifier. This prevents two entities with the same values from colliding. Args: index: A datastore_pb.CompositeIndex. entity: A entity_pb.EntityProto. Returns: A list of strings representing keys to the composite table. """ composite_id = index.id() definition = index.definition() app_id = clean_app_id(entity.key().app()) name_space = entity.key().name_space() ent_key = encode_index_pb(entity.key().path()) pre_comp_index_key = "{0}{1}{2}{4}{3}{4}".format(app_id, dbconstants.KEY_DELIMITER, name_space, composite_id, dbconstants.KEY_DELIMITER) if definition.ancestor() == 1: ancestor_list = get_ancestor_paths_from_ent_key(ent_key) property_list_names = [prop.name() for prop in entity.property_list()] multivalue_dict = {} for prop in entity.property_list(): if prop.name() not in property_list_names: continue value = str(encode_index_pb(prop.value())) if prop.name() in multivalue_dict: multivalue_dict[prop.name()].append(value) else: multivalue_dict[prop.name()] = [value] # Build lists for which we'll get all combinations of indexes. lists_of_prop_list = [] for prop in definition.property_list(): # Check to make sure the entity has the required items. If not then we # do not create an index for the composite index. # The definition can also have a key as a part of the index, but this # is not repeated. if prop.name() == "__key__": value = str(encode_index_pb(entity.key().path())) if prop.direction() == entity_pb.Index_Property.DESCENDING: value = helper_functions.reverse_lex(value) lists_of_prop_list.append([value]) elif prop.name() not in multivalue_dict: return [] else: my_list = multivalue_dict[prop.name()] if prop.direction() == entity_pb.Index_Property.DESCENDING: for index, item in enumerate(my_list): my_list[index] = helper_functions.reverse_lex(item) lists_of_prop_list.append(my_list) # Get all combinations of the composite indexes. all_combinations = [] if len(lists_of_prop_list) == 1: for item in lists_of_prop_list[0]: all_combinations.append([item]) elif len(lists_of_prop_list) > 1: all_combinations = list(itertools.product(*lists_of_prop_list)) # We should throw an exception if the number of combinations is # more than 20000. We currently do not. # https://developers.google.com/appengine/docs/python/datastore/ # #Python_Quotas_and_limits all_keys = [] for combo in all_combinations: index_value = "" for prop_value in combo: index_value += str(prop_value) + dbconstants.KEY_DELIMITER # We append the ent key to have unique keys if entities happen # to share the same index values (and ancestor). if definition.ancestor() == 1: for ancestor in ancestor_list: pre_comp_key = pre_comp_index_key + "{0}{1}".format( ancestor, dbconstants.KEY_DELIMITER) composite_key = "{0}{1}{2}".format(pre_comp_key, index_value, ent_key) all_keys.append(composite_key) else: composite_key = "{0}{1}{2}".format(pre_comp_index_key, index_value, ent_key) all_keys.append(composite_key) return all_keys
def get_composite_index_keys(index, entity): """ Creates keys to the composite index table for a given entity. Keys are built as such: app_id/ns/composite_id/ancestor/valuevaluevalue..../entity_key Components explained: ns: The namespace of the entity. composite_id: The composite ID assigned to this index upon creation. ancestor: The root ancestor path (only if the query this index is for has an ancestor) value(s): The string representation of mulitiple properties. entity_key: The entity key (full path) used as a means of having a unique identifier. This prevents two entities with the same values from colliding. Args: index: A datastore_pb.CompositeIndex. entity: A entity_pb.EntityProto. Returns: A list of strings representing keys to the composite table. """ composite_id = index.id() definition = index.definition() app_id = clean_app_id(entity.key().app()) name_space = entity.key().name_space() ent_key = encode_index_pb(entity.key().path()) pre_comp_index_key = "{0}{1}{2}{4}{3}{4}".format(app_id, dbconstants.KEY_DELIMITER, name_space, composite_id, dbconstants.KEY_DELIMITER) if definition.ancestor() == 1: ancestor_list = get_ancestor_paths_from_ent_key(ent_key) property_list_names = [prop.name() for prop in entity.property_list()] multivalue_dict = {} for prop in entity.property_list(): if prop.name() not in property_list_names: continue value = str(encode_index_pb(prop.value())) if prop.name() in multivalue_dict: multivalue_dict[prop.name()].append(value) else: multivalue_dict[prop.name()] = [value] # Build lists for which we'll get all combinations of indexes. lists_of_prop_list = [] for prop in definition.property_list(): # Check to make sure the entity has the required items. If not then we # do not create an index for the composite index. # The definition can also have a key as a part of the index, but this # is not repeated. if prop.name() == "__key__": value = str(encode_index_pb(entity.key().path())) if prop.direction() == entity_pb.Index_Property.DESCENDING: value = helper_functions.reverse_lex(value) lists_of_prop_list.append([value]) elif prop.name() not in multivalue_dict: return [] else: my_list = multivalue_dict[prop.name()] if prop.direction() == entity_pb.Index_Property.DESCENDING: for index, item in enumerate(my_list): my_list[index] = helper_functions.reverse_lex(item) lists_of_prop_list.append(my_list) # Get all combinations of the composite indexes. all_combinations = [] if len(lists_of_prop_list) == 1: for item in lists_of_prop_list[0]: all_combinations.append([item]) elif len(lists_of_prop_list) > 1: all_combinations = list(itertools.product(*lists_of_prop_list)) # We should throw an exception if the number of combinations is # more than 20000. We currently do not. # https://developers.google.com/appengine/docs/python/datastore/ # #Python_Quotas_and_limits all_keys = [] for combo in all_combinations: index_value = "" for prop_value in combo: index_value += str(prop_value) + dbconstants.KEY_DELIMITER # We append the ent key to have unique keys if entities happen # to share the same index values (and ancestor). if definition.ancestor() == 1: for ancestor in ancestor_list: pre_comp_key = pre_comp_index_key + "{0}{1}".format( ancestor, dbconstants.KEY_DELIMITER) composite_key = "{0}{1}{2}".format(pre_comp_key, index_value, ent_key) all_keys.append(composite_key) else: composite_key = "{0}{1}{2}".format(pre_comp_index_key, index_value, ent_key) all_keys.append(composite_key) return all_keys