def test_composite_ancestor_index_no_properties(self): ci = datastore_pb.CompositeIndex() ci.set_app_id(datastore_types.ResolveAppId(None)) ci.set_id(0) ci.set_state(ci.WRITE_ONLY) index = ci.mutable_definition() index.set_ancestor(1) index.set_entity_type('Yar') stub = apiproxy_stub_map.apiproxy.GetStub('datastore_v3') stub.CreateIndex(ci) self.assertEquals(1, len(datastore.GetIndexes())) entity = datastore.Entity('Yar', id=123, _app=self.app_id) # 2 writes. entity['this'] = [None, None] # 2 writes for the entity. # 4 writes for the indexed properties. # 1 writes for the composite index. self.assertEquals( 7, datastore_viewer.DatastoreRequestHandler._get_write_ops(entity)) # Now use the same entity but give it an ancestor parent_entity = datastore.Entity('parent', id=123, _app=self.app_id) entity = datastore.Entity('Yar', parent=parent_entity.key(), id=123, _app=self.app_id) # 2 writes. entity['this'] = [None, None] # 2 writes for the entity. # 4 writes for the indexed properties. # 2 writes for the composite indices. self.assertEquals( 8, datastore_viewer.DatastoreRequestHandler._get_write_ops(entity))
def GetIndices(_app=None): """Fetches all composite indices in the datastore for this app. Returns: list of entity_pb.CompositeIndex """ resolved_app_id = datastore_types.ResolveAppId(_app) if hasattr(datastore_pb, 'GetIndicesRequest'): req = datastore_pb.GetIndicesRequest() req.set_app_id(resolved_app_id) else: req = api_base_pb.StringProto() req.app_id = MethodType(_StringProtoAppIdGet, req) req.set_app_id = MethodType(_StringProtoAppIdSet, req) req.set_app_id(resolved_app_id) resp = datastore_pb.CompositeIndices() resp = _Call('GetIndices', req, resp) return resp.index_list()
def GetSchema(_app=None, properties=True, start_kind=None, end_kind=None): """Infers an app's schema from the entities in the datastore. Note that the PropertyValue PBs in the returned EntityProtos are empty placeholders, so they may cause problems if you try to convert them to python values with e.g. datastore_types. In particular, user values will throw UserNotFoundError because their email and auth domain fields will be empty. Args: properties: boolean, whether to include property names and types start_kind, end_kind: optional range endpoints for the kinds to return, compared lexicographically Returns: list of entity_pb.EntityProto, with kind and property names and types """ req = datastore_pb.GetSchemaRequest() req.set_app(datastore_types.ResolveAppId(_app)) req.set_properties(properties) if start_kind is not None: req.set_start_kind(start_kind) if end_kind is not None: req.set_end_kind(end_kind) resp = datastore_pb.Schema() resp = _Call('GetSchema', req, resp) return resp.kind_list()
def test_composite_index_no_properties(self): ci = datastore_pb.CompositeIndex() ci.set_app_id(datastore_types.ResolveAppId(None)) ci.set_id(0) ci.set_state(ci.WRITE_ONLY) index = ci.mutable_definition() index.set_ancestor(0) index.set_entity_type('Yar') stub = apiproxy_stub_map.apiproxy.GetStub('datastore_v3') stub.CreateIndex(ci) self.assertEquals(1, len(datastore.GetIndexes())) # no properties, and composite index with no properties. entity = datastore.Entity('Yar', id=123, _app=self.app_id) # 2 writes. # We have the 2 built-in index writes, and one for the entity key in the # composite index despite the fact that there are no proerties defined in # the index. self.assertEquals( 3, datastore_viewer.DatastoreRequestHandler._get_write_ops(entity)) # now with a repeated property entity = datastore.Entity('Yar', id=123, _app=self.app_id) # 2 writes. entity['this'] = [1, 2, 3] # 2 for the entity # 6 for the indexed properties # 1 for the composite index self.assertEquals( 9, datastore_viewer.DatastoreRequestHandler._get_write_ops(entity))
def __init__(self, _app=None): self.app_id = datastore_types.ResolveAppId(_app) self.whole_app_stats = {} self.namespace_stats = {} self.found_non_empty_namespace = False self.old_stat_keys = [] self.timestamp = datetime.datetime.utcnow()
def GetIndices(_app=None): """Fetches all composite indices in the datastore for this app. Returns: list of entity_pb.CompositeIndex """ req = api_base_pb.StringProto() req.set_value(datastore_types.ResolveAppId(_app)) resp = datastore_pb.CompositeIndices() resp = _Call('GetIndices', req, resp) return resp.index_list()
def __init__(self, app=None, namespace=None, kind=None, ancestor=None, filter_predicate=None, order=None): """Constructor. Args: app: Optional app to query, derived from the environment if not specified. namespace: Optional namespace to query, derived from the environment if not specified. kind: Optional kind to query. ancestor: Optional ancestor to query. filter_predicate: Optional FilterPredicate by which to restrict the query. order: Optional Order in which to return results. Raises: datastore_errors.BadArgumentError if any argument is invalid. """ if kind is not None: datastore_types.ValidateString(kind, 'kind', datastore_errors.BadArgumentError) if ancestor is not None and not isinstance(ancestor, entity_pb.Reference): raise datastore_errors.BadArgumentError( 'ancestor argument should be entity_pb.Reference (%r)' % (ancestor, )) if filter_predicate is not None and not isinstance( filter_predicate, FilterPredicate): raise datastore_errors.BadArgumentError( 'filter_predicate should be datastore_query.FilterPredicate (%r)' % (ancestor, )) super(Query, self).__init__() if isinstance(order, CompositeOrder): if order.size() == 0: order = None elif isinstance(order, Order): order = CompositeOrder([order]) elif order is not None: raise datastore_errors.BadArgumentError( 'order should be Order (%r)' % (order, )) self.__app = datastore_types.ResolveAppId(app) self.__namespace = datastore_types.ResolveNamespace(namespace) self.__kind = kind self.__ancestor = ancestor self.__order = order self.__filter_predicate = filter_predicate
def GetIndices(_app=None): """Fetches all composite indices in the datastore for this app. Returns: list of entity_pb.CompositeIndex """ req = api_base_pb.StringProto() req.set_value(datastore_types.ResolveAppId(_app)) resp = datastore_pb.CompositeIndices() try: apiproxy_stub_map.MakeSyncCall('datastore_v3', 'GetIndices', req, resp) except apiproxy_errors.ApplicationError, err: raise datastore._ToDatastoreError(err)
def _Dynamic_TransactionQuery(self, request, response): if not request.has_ancestor(): raise apiproxy_errors.ApplicationError( datastore_pb.Error.BAD_REQUEST, 'No ancestor in transactional query.') app_id = datastore_types.ResolveAppId(None) if (datastore_rpc._GetDatastoreType(app_id) != datastore_rpc.BaseConnection.HIGH_REPLICATION_DATASTORE): raise apiproxy_errors.ApplicationError( datastore_pb.Error.BAD_REQUEST, 'remote_api supports transactional queries only in the ' 'high-replication datastore.') entity_group_key = entity_pb.Reference() entity_group_key.CopyFrom(request.ancestor()) group_path = entity_group_key.mutable_path() root = entity_pb.Path_Element() root.MergeFrom(group_path.element(0)) group_path.clear_element() group_path.add_element().CopyFrom(root) eg_element = group_path.add_element() eg_element.set_type(metadata.EntityGroup.KIND_NAME) eg_element.set_id(metadata.EntityGroup.ID) begin_request = datastore_pb.BeginTransactionRequest() begin_request.set_app(app_id) tx = datastore_pb.Transaction() self.__call('datastore_v3', 'BeginTransaction', begin_request, tx) request.mutable_transaction().CopyFrom(tx) self.__call('datastore_v3', 'RunQuery', request, response.mutable_result()) get_request = datastore_pb.GetRequest() get_request.mutable_transaction().CopyFrom(tx) get_request.add_key().CopyFrom(entity_group_key) get_response = datastore_pb.GetResponse() self.__call('datastore_v3', 'Get', get_request, get_response) entity_group = get_response.entity(0) response.mutable_entity_group_key().CopyFrom(entity_group_key) if entity_group.has_entity(): response.mutable_entity_group().CopyFrom(entity_group.entity()) self.__call('datastore_v3', 'Commit', tx, datastore_pb.CommitResponse())
def GetIndices(_app=None): """Fetches all composite indices in the datastore for this app. Returns: list of entity_pb.CompositeIndex """ resolved_app_id = datastore_types.ResolveAppId(_app) req = datastore_pb.GetIndicesRequest() req.app_id = resolved_app_id resp = datastore_pb.CompositeIndices() resp = _Call('GetIndices', req, resp) return resp.index
def _Call(call, req, resp): """Generic method for making a datastore API call. Args: call: string, the name of the RPC call req: the request PB. if the app_id field is not set, it defaults to the local app. resp: the response PB """ if hasattr(req, 'app_id'): req.set_app_id( datastore_types.ResolveAppId(req.app_id(), 'req.app_id()')) try: apiproxy_stub_map.MakeSyncCall('datastore_v3', call, req, resp) except apiproxy_errors.ApplicationError, err: raise datastore._ToDatastoreError(err)
def GetSchema(_app=None): """Infers an app's schema from the entities in the datastore. Note that the PropertyValue PBs in the returned EntityProtos are empty placeholders, so they may cause problems if you try to convert them to python values with e.g. datastore_types. In particular, user values will throw UserNotFoundError because their email and auth domain fields will be empty. Returns: list of entity_pb.EntityProto, with kind and property names and types """ req = api_base_pb.StringProto() req.set_value(datastore_types.ResolveAppId(_app)) resp = datastore_pb.Schema() _Call('GetSchema', req, resp) return resp.kind_list()
def test_composite_ancestor_index(self): ci = datastore_pb.CompositeIndex() ci.set_app_id(datastore_types.ResolveAppId(None)) ci.set_id(0) ci.set_state(ci.WRITE_ONLY) index = ci.mutable_definition() index.set_ancestor(1) index.set_entity_type('Yar') prop = index.add_property() prop.set_name('this') prop.set_direction(prop.ASCENDING) prop = index.add_property() prop.set_name('that') prop.set_direction(prop.DESCENDING) stub = apiproxy_stub_map.apiproxy.GetStub('datastore_v3') stub.CreateIndex(ci) self.assertEquals(1, len(datastore.GetIndexes())) entity = datastore.Entity('Yar', id=123, _app=self.app_id) # 2 writes. entity['this'] = 4 entity['that'] = 4 # 2 for the entity # 4 for the indexed properties # 1 for the composite index self.assertEquals( 7, datastore_viewer.DatastoreRequestHandler._get_write_ops(entity)) # Now use the same entity but give it an ancestor parent_entity = datastore.Entity('parent', id=123, _app=self.app_id) entity = datastore.Entity('Yar', parent=parent_entity.key(), id=123, _app=self.app_id) # 2 writes. entity['this'] = 4 entity['that'] = 4 # 2 writes for the entity. # 4 writes for the indexed properties. # 2 writes for the composite indices. self.assertEquals( 8, datastore_viewer.DatastoreRequestHandler._get_write_ops(entity)) # Now use the same entity but give it 2 ancestors. grandparent_entity = datastore.Entity('grandparent', id=123, _app=self.app_id) parent_entity = datastore.Entity('parent', parent=grandparent_entity.key(), id=123, _app=self.app_id) entity = datastore.Entity('Yar', parent=parent_entity.key(), id=123, _app=self.app_id) # 2 writes. entity['this'] = 4 entity['that'] = 4 # 2 writes for the entity. # 4 writes for the indexed properties. # 3 writes for the composite indices. self.assertEquals( 9, datastore_viewer.DatastoreRequestHandler._get_write_ops(entity)) # Now try it with a multi-value prop entity['this'] = [None, None, None] # 2 writes for the entity. # 8 writes for the indexed properties. # 9 writes for the composite indices. self.assertEquals( 19, datastore_viewer.DatastoreRequestHandler._get_write_ops(entity)) # Now try it with 2 multi-value props. entity['that'] = [None, None] # 2 writes for the entity. # 10 writes for the indexed properties. # 18 writes for the composite indices. self.assertEquals( 30, datastore_viewer.DatastoreRequestHandler._get_write_ops(entity))
def test_composite_index(self): ci = datastore_pb.CompositeIndex() ci.set_app_id(datastore_types.ResolveAppId(None)) ci.set_id(0) ci.set_state(ci.WRITE_ONLY) index = ci.mutable_definition() index.set_ancestor(0) index.set_entity_type('Yar') prop = index.add_property() prop.set_name('this') prop.set_direction(prop.ASCENDING) prop = index.add_property() prop.set_name('that') prop.set_direction(prop.DESCENDING) stub = apiproxy_stub_map.apiproxy.GetStub('datastore_v3') stub.CreateIndex(ci) self.assertEquals(1, len(datastore.GetIndexes())) # no properties, no composite indices. entity = datastore.Entity('Yar', id=123, _app=self.app_id) # 2 writes. # We only have the 2 built-in index writes because the entity doesn't have # property values for any of the index properties. self.assertEquals( 2, datastore_viewer.DatastoreRequestHandler._get_write_ops(entity)) entity['this'] = 4 # Unindexed property so no additional writes entity.set_unindexed_properties(('this', )) self.assertEquals( 2, datastore_viewer.DatastoreRequestHandler._get_write_ops(entity)) entity['that'] = 4 # Unindexed property so no additional writes entity.set_unindexed_properties(('this', 'that')) self.assertEquals( 2, datastore_viewer.DatastoreRequestHandler._get_write_ops(entity)) # no indexed property value on 'that' entity.set_unindexed_properties(('that', )) # 2 writes for the entity. # 2 writes for the single indexed property. self.assertEquals( 4, datastore_viewer.DatastoreRequestHandler._get_write_ops(entity)) # indexed property value on both 'this' and 'that' entity.set_unindexed_properties(()) # 2 writes for the entity # 4 writes for the indexed properties # 1 writes for the composite index self.assertEquals( 7, datastore_viewer.DatastoreRequestHandler._get_write_ops(entity)) # now run tests with null property values entity = datastore.Entity('Yar', id=123, _app=self.app_id) # 2 writes. entity['this'] = None # 2 for the entity # 2 for the single indexed property self.assertEquals( 4, datastore_viewer.DatastoreRequestHandler._get_write_ops(entity)) entity['that'] = None # 2 for the entity # 4 for the indexed properties # 1 for the composite index self.assertEquals( 7, datastore_viewer.DatastoreRequestHandler._get_write_ops(entity)) # now run tests with a repeated property entity = datastore.Entity('Yar', id=123, _app=self.app_id) # 2 writes. entity['this'] = [1, 2, 3] # 2 for the entity # 6 for the indexed properties self.assertEquals( 8, datastore_viewer.DatastoreRequestHandler._get_write_ops(entity)) entity['that'] = None # 2 for the entity # 8 for the indexed properties # 3 for the Composite index self.assertEquals( 13, datastore_viewer.DatastoreRequestHandler._get_write_ops(entity)) entity['that'] = [4, 5] # 2 for the entity # 10 for the indexed properties # 6 for the Composite index self.assertEquals( 18, datastore_viewer.DatastoreRequestHandler._get_write_ops(entity))