def test_composite_ancestor_index_no_properties(self):
ci = datastore_pb.CompositeIndex()
ci.set_app_id(datastore_types.ResolveAppId(None))
ci.set_id(0)
ci.set_state(ci.WRITE_ONLY)
index = ci.mutable_definition()
index.set_ancestor(1)
index.set_entity_type('Yar')
stub = apiproxy_stub_map.apiproxy.GetStub('datastore_v3')
stub.CreateIndex(ci)
self.assertEquals(1, len(datastore.GetIndexes()))
entity = datastore.Entity('Yar', id=123, _app=self.app_id) # 2 writes.
entity['this'] = [None, None]
# 2 writes for the entity.
# 4 writes for the indexed properties.
# 1 writes for the composite index.
self.assertEquals(
7, datastore_viewer.DatastoreRequestHandler._get_write_ops(entity))
# Now use the same entity but give it an ancestor
parent_entity = datastore.Entity('parent', id=123, _app=self.app_id)
entity = datastore.Entity('Yar',
parent=parent_entity.key(),
id=123,
_app=self.app_id) # 2 writes.
entity['this'] = [None, None]
# 2 writes for the entity.
# 4 writes for the indexed properties.
# 2 writes for the composite indices.
self.assertEquals(
8, datastore_viewer.DatastoreRequestHandler._get_write_ops(entity))
def GetIndices(_app=None):
"""Fetches all composite indices in the datastore for this app.
Returns:
list of entity_pb.CompositeIndex
"""
resolved_app_id = datastore_types.ResolveAppId(_app)
if hasattr(datastore_pb, 'GetIndicesRequest'):
req = datastore_pb.GetIndicesRequest()
req.set_app_id(resolved_app_id)
else:
req = api_base_pb.StringProto()
req.app_id = MethodType(_StringProtoAppIdGet, req)
req.set_app_id = MethodType(_StringProtoAppIdSet, req)
req.set_app_id(resolved_app_id)
resp = datastore_pb.CompositeIndices()
resp = _Call('GetIndices', req, resp)
return resp.index_list()
def GetSchema(_app=None, properties=True, start_kind=None, end_kind=None):
"""Infers an app's schema from the entities in the datastore.
Note that the PropertyValue PBs in the returned EntityProtos are empty
placeholders, so they may cause problems if you try to convert them to
python values with e.g. datastore_types. In particular, user values will
throw UserNotFoundError because their email and auth domain fields will be
empty.
Args:
properties: boolean, whether to include property names and types
start_kind, end_kind: optional range endpoints for the kinds to return,
compared lexicographically
Returns:
list of entity_pb.EntityProto, with kind and property names and types
"""
req = datastore_pb.GetSchemaRequest()
req.set_app(datastore_types.ResolveAppId(_app))
req.set_properties(properties)
if start_kind is not None:
req.set_start_kind(start_kind)
if end_kind is not None:
req.set_end_kind(end_kind)
resp = datastore_pb.Schema()
resp = _Call('GetSchema', req, resp)
return resp.kind_list()
def test_composite_index_no_properties(self):
ci = datastore_pb.CompositeIndex()
ci.set_app_id(datastore_types.ResolveAppId(None))
ci.set_id(0)
ci.set_state(ci.WRITE_ONLY)
index = ci.mutable_definition()
index.set_ancestor(0)
index.set_entity_type('Yar')
stub = apiproxy_stub_map.apiproxy.GetStub('datastore_v3')
stub.CreateIndex(ci)
self.assertEquals(1, len(datastore.GetIndexes()))
# no properties, and composite index with no properties.
entity = datastore.Entity('Yar', id=123, _app=self.app_id) # 2 writes.
# We have the 2 built-in index writes, and one for the entity key in the
# composite index despite the fact that there are no proerties defined in
# the index.
self.assertEquals(
3, datastore_viewer.DatastoreRequestHandler._get_write_ops(entity))
# now with a repeated property
entity = datastore.Entity('Yar', id=123, _app=self.app_id) # 2 writes.
entity['this'] = [1, 2, 3]
# 2 for the entity
# 6 for the indexed properties
# 1 for the composite index
self.assertEquals(
9, datastore_viewer.DatastoreRequestHandler._get_write_ops(entity))
def __init__(self, _app=None):
self.app_id = datastore_types.ResolveAppId(_app)
self.whole_app_stats = {}
self.namespace_stats = {}
self.found_non_empty_namespace = False
self.old_stat_keys = []
self.timestamp = datetime.datetime.utcnow()
def GetIndices(_app=None):
"""Fetches all composite indices in the datastore for this app.
Returns:
list of entity_pb.CompositeIndex
"""
req = api_base_pb.StringProto()
req.set_value(datastore_types.ResolveAppId(_app))
resp = datastore_pb.CompositeIndices()
resp = _Call('GetIndices', req, resp)
return resp.index_list()
def __init__(self,
app=None,
namespace=None,
kind=None,
ancestor=None,
filter_predicate=None,
order=None):
"""Constructor.
Args:
app: Optional app to query, derived from the environment if not specified.
namespace: Optional namespace to query, derived from the environment if
not specified.
kind: Optional kind to query.
ancestor: Optional ancestor to query.
filter_predicate: Optional FilterPredicate by which to restrict the query.
order: Optional Order in which to return results.
Raises:
datastore_errors.BadArgumentError if any argument is invalid.
"""
if kind is not None:
datastore_types.ValidateString(kind, 'kind',
datastore_errors.BadArgumentError)
if ancestor is not None and not isinstance(ancestor,
entity_pb.Reference):
raise datastore_errors.BadArgumentError(
'ancestor argument should be entity_pb.Reference (%r)' %
(ancestor, ))
if filter_predicate is not None and not isinstance(
filter_predicate, FilterPredicate):
raise datastore_errors.BadArgumentError(
'filter_predicate should be datastore_query.FilterPredicate (%r)'
% (ancestor, ))
super(Query, self).__init__()
if isinstance(order, CompositeOrder):
if order.size() == 0:
order = None
elif isinstance(order, Order):
order = CompositeOrder([order])
elif order is not None:
raise datastore_errors.BadArgumentError(
'order should be Order (%r)' % (order, ))
self.__app = datastore_types.ResolveAppId(app)
self.__namespace = datastore_types.ResolveNamespace(namespace)
self.__kind = kind
self.__ancestor = ancestor
self.__order = order
self.__filter_predicate = filter_predicate
def GetIndices(_app=None):
"""Fetches all composite indices in the datastore for this app.
Returns:
list of entity_pb.CompositeIndex
"""
req = api_base_pb.StringProto()
req.set_value(datastore_types.ResolveAppId(_app))
resp = datastore_pb.CompositeIndices()
try:
apiproxy_stub_map.MakeSyncCall('datastore_v3', 'GetIndices', req, resp)
except apiproxy_errors.ApplicationError, err:
raise datastore._ToDatastoreError(err)
def _Dynamic_TransactionQuery(self, request, response):
if not request.has_ancestor():
raise apiproxy_errors.ApplicationError(
datastore_pb.Error.BAD_REQUEST,
'No ancestor in transactional query.')
app_id = datastore_types.ResolveAppId(None)
if (datastore_rpc._GetDatastoreType(app_id) !=
datastore_rpc.BaseConnection.HIGH_REPLICATION_DATASTORE):
raise apiproxy_errors.ApplicationError(
datastore_pb.Error.BAD_REQUEST,
'remote_api supports transactional queries only in the '
'high-replication datastore.')
entity_group_key = entity_pb.Reference()
entity_group_key.CopyFrom(request.ancestor())
group_path = entity_group_key.mutable_path()
root = entity_pb.Path_Element()
root.MergeFrom(group_path.element(0))
group_path.clear_element()
group_path.add_element().CopyFrom(root)
eg_element = group_path.add_element()
eg_element.set_type(metadata.EntityGroup.KIND_NAME)
eg_element.set_id(metadata.EntityGroup.ID)
begin_request = datastore_pb.BeginTransactionRequest()
begin_request.set_app(app_id)
tx = datastore_pb.Transaction()
self.__call('datastore_v3', 'BeginTransaction', begin_request, tx)
request.mutable_transaction().CopyFrom(tx)
self.__call('datastore_v3', 'RunQuery', request, response.mutable_result())
get_request = datastore_pb.GetRequest()
get_request.mutable_transaction().CopyFrom(tx)
get_request.add_key().CopyFrom(entity_group_key)
get_response = datastore_pb.GetResponse()
self.__call('datastore_v3', 'Get', get_request, get_response)
entity_group = get_response.entity(0)
response.mutable_entity_group_key().CopyFrom(entity_group_key)
if entity_group.has_entity():
response.mutable_entity_group().CopyFrom(entity_group.entity())
self.__call('datastore_v3', 'Commit', tx, datastore_pb.CommitResponse())
def GetIndices(_app=None):
"""Fetches all composite indices in the datastore for this app.
Returns:
list of entity_pb.CompositeIndex
"""
resolved_app_id = datastore_types.ResolveAppId(_app)
req = datastore_pb.GetIndicesRequest()
req.app_id = resolved_app_id
resp = datastore_pb.CompositeIndices()
resp = _Call('GetIndices', req, resp)
return resp.index
def _Call(call, req, resp):
"""Generic method for making a datastore API call.
Args:
call: string, the name of the RPC call
req: the request PB. if the app_id field is not set, it defaults to the
local app.
resp: the response PB
"""
if hasattr(req, 'app_id'):
req.set_app_id(
datastore_types.ResolveAppId(req.app_id(), 'req.app_id()'))
try:
apiproxy_stub_map.MakeSyncCall('datastore_v3', call, req, resp)
except apiproxy_errors.ApplicationError, err:
raise datastore._ToDatastoreError(err)
def GetSchema(_app=None):
"""Infers an app's schema from the entities in the datastore.
Note that the PropertyValue PBs in the returned EntityProtos are empty
placeholders, so they may cause problems if you try to convert them to
python values with e.g. datastore_types. In particular, user values will
throw UserNotFoundError because their email and auth domain fields will be
empty.
Returns:
list of entity_pb.EntityProto, with kind and property names and types
"""
req = api_base_pb.StringProto()
req.set_value(datastore_types.ResolveAppId(_app))
resp = datastore_pb.Schema()
_Call('GetSchema', req, resp)
return resp.kind_list()
def test_composite_ancestor_index(self):
ci = datastore_pb.CompositeIndex()
ci.set_app_id(datastore_types.ResolveAppId(None))
ci.set_id(0)
ci.set_state(ci.WRITE_ONLY)
index = ci.mutable_definition()
index.set_ancestor(1)
index.set_entity_type('Yar')
prop = index.add_property()
prop.set_name('this')
prop.set_direction(prop.ASCENDING)
prop = index.add_property()
prop.set_name('that')
prop.set_direction(prop.DESCENDING)
stub = apiproxy_stub_map.apiproxy.GetStub('datastore_v3')
stub.CreateIndex(ci)
self.assertEquals(1, len(datastore.GetIndexes()))
entity = datastore.Entity('Yar', id=123, _app=self.app_id) # 2 writes.
entity['this'] = 4
entity['that'] = 4
# 2 for the entity
# 4 for the indexed properties
# 1 for the composite index
self.assertEquals(
7, datastore_viewer.DatastoreRequestHandler._get_write_ops(entity))
# Now use the same entity but give it an ancestor
parent_entity = datastore.Entity('parent', id=123, _app=self.app_id)
entity = datastore.Entity('Yar',
parent=parent_entity.key(),
id=123,
_app=self.app_id) # 2 writes.
entity['this'] = 4
entity['that'] = 4
# 2 writes for the entity.
# 4 writes for the indexed properties.
# 2 writes for the composite indices.
self.assertEquals(
8, datastore_viewer.DatastoreRequestHandler._get_write_ops(entity))
# Now use the same entity but give it 2 ancestors.
grandparent_entity = datastore.Entity('grandparent',
id=123,
_app=self.app_id)
parent_entity = datastore.Entity('parent',
parent=grandparent_entity.key(),
id=123,
_app=self.app_id)
entity = datastore.Entity('Yar',
parent=parent_entity.key(),
id=123,
_app=self.app_id) # 2 writes.
entity['this'] = 4
entity['that'] = 4
# 2 writes for the entity.
# 4 writes for the indexed properties.
# 3 writes for the composite indices.
self.assertEquals(
9, datastore_viewer.DatastoreRequestHandler._get_write_ops(entity))
# Now try it with a multi-value prop
entity['this'] = [None, None, None]
# 2 writes for the entity.
# 8 writes for the indexed properties.
# 9 writes for the composite indices.
self.assertEquals(
19,
datastore_viewer.DatastoreRequestHandler._get_write_ops(entity))
# Now try it with 2 multi-value props.
entity['that'] = [None, None]
# 2 writes for the entity.
# 10 writes for the indexed properties.
# 18 writes for the composite indices.
self.assertEquals(
30,
datastore_viewer.DatastoreRequestHandler._get_write_ops(entity))
def test_composite_index(self):
ci = datastore_pb.CompositeIndex()
ci.set_app_id(datastore_types.ResolveAppId(None))
ci.set_id(0)
ci.set_state(ci.WRITE_ONLY)
index = ci.mutable_definition()
index.set_ancestor(0)
index.set_entity_type('Yar')
prop = index.add_property()
prop.set_name('this')
prop.set_direction(prop.ASCENDING)
prop = index.add_property()
prop.set_name('that')
prop.set_direction(prop.DESCENDING)
stub = apiproxy_stub_map.apiproxy.GetStub('datastore_v3')
stub.CreateIndex(ci)
self.assertEquals(1, len(datastore.GetIndexes()))
# no properties, no composite indices.
entity = datastore.Entity('Yar', id=123, _app=self.app_id) # 2 writes.
# We only have the 2 built-in index writes because the entity doesn't have
# property values for any of the index properties.
self.assertEquals(
2, datastore_viewer.DatastoreRequestHandler._get_write_ops(entity))
entity['this'] = 4
# Unindexed property so no additional writes
entity.set_unindexed_properties(('this', ))
self.assertEquals(
2, datastore_viewer.DatastoreRequestHandler._get_write_ops(entity))
entity['that'] = 4
# Unindexed property so no additional writes
entity.set_unindexed_properties(('this', 'that'))
self.assertEquals(
2, datastore_viewer.DatastoreRequestHandler._get_write_ops(entity))
# no indexed property value on 'that'
entity.set_unindexed_properties(('that', ))
# 2 writes for the entity.
# 2 writes for the single indexed property.
self.assertEquals(
4, datastore_viewer.DatastoreRequestHandler._get_write_ops(entity))
# indexed property value on both 'this' and 'that'
entity.set_unindexed_properties(())
# 2 writes for the entity
# 4 writes for the indexed properties
# 1 writes for the composite index
self.assertEquals(
7, datastore_viewer.DatastoreRequestHandler._get_write_ops(entity))
# now run tests with null property values
entity = datastore.Entity('Yar', id=123, _app=self.app_id) # 2 writes.
entity['this'] = None
# 2 for the entity
# 2 for the single indexed property
self.assertEquals(
4, datastore_viewer.DatastoreRequestHandler._get_write_ops(entity))
entity['that'] = None
# 2 for the entity
# 4 for the indexed properties
# 1 for the composite index
self.assertEquals(
7, datastore_viewer.DatastoreRequestHandler._get_write_ops(entity))
# now run tests with a repeated property
entity = datastore.Entity('Yar', id=123, _app=self.app_id) # 2 writes.
entity['this'] = [1, 2, 3]
# 2 for the entity
# 6 for the indexed properties
self.assertEquals(
8, datastore_viewer.DatastoreRequestHandler._get_write_ops(entity))
entity['that'] = None
# 2 for the entity
# 8 for the indexed properties
# 3 for the Composite index
self.assertEquals(
13,
datastore_viewer.DatastoreRequestHandler._get_write_ops(entity))
entity['that'] = [4, 5]
# 2 for the entity
# 10 for the indexed properties
# 6 for the Composite index
self.assertEquals(
18,
datastore_viewer.DatastoreRequestHandler._get_write_ops(entity))
