def test_nested_entity_no_key(self):
from gcloud.datastore._generated import entity_pb2
from gcloud.datastore.helpers import _new_value_pb
PROJECT = 's~FOO'
KIND = 'KIND'
INSIDE_NAME = 'IFOO'
OUTSIDE_NAME = 'OBAR'
INSIDE_VALUE = 1337
entity_inside = entity_pb2.Entity()
inside_val_pb = _new_value_pb(entity_inside, INSIDE_NAME)
inside_val_pb.integer_value = INSIDE_VALUE
entity_pb = entity_pb2.Entity()
entity_pb.key.partition_id.dataset_id = PROJECT
element = entity_pb.key.path_element.add()
element.kind = KIND
outside_val_pb = _new_value_pb(entity_pb, OUTSIDE_NAME)
outside_val_pb.entity_value.CopyFrom(entity_inside)
entity = self._callFUT(entity_pb)
self.assertEqual(entity.key.project, PROJECT)
self.assertEqual(entity.key.flat_path, (KIND,))
self.assertEqual(len(entity), 1)
inside_entity = entity[OUTSIDE_NAME]
self.assertEqual(inside_entity.key, None)
self.assertEqual(len(inside_entity), 1)
self.assertEqual(inside_entity[INSIDE_NAME], INSIDE_VALUE)
def test_inverts_to_protobuf(self):
from gcloud.datastore._generated import entity_pb2
from gcloud.datastore.helpers import _new_value_pb
from gcloud.datastore.helpers import entity_from_protobuf
original_pb = entity_pb2.Entity()
# Add a key.
original_pb.key.partition_id.dataset_id = project = 'PROJECT'
elem1 = original_pb.key.path_element.add()
elem1.kind = 'Family'
elem1.id = 1234
elem2 = original_pb.key.path_element.add()
elem2.kind = 'King'
elem2.name = 'Spades'
# Add an integer property.
val_pb1 = _new_value_pb(original_pb, 'foo')
val_pb1.integer_value = 1337
val_pb1.indexed = False
# Add a string property.
val_pb2 = _new_value_pb(original_pb, 'bar')
val_pb2.string_value = u'hello'
# Add a nested (entity) property.
val_pb3 = _new_value_pb(original_pb, 'entity-baz')
sub_pb = entity_pb2.Entity()
sub_val_pb1 = _new_value_pb(sub_pb, 'x')
sub_val_pb1.double_value = 3.14
sub_val_pb2 = _new_value_pb(sub_pb, 'y')
sub_val_pb2.double_value = 2.718281828
val_pb3.meaning = 9
val_pb3.entity_value.CopyFrom(sub_pb)
# Add a list property.
val_pb4 = _new_value_pb(original_pb, 'list-quux')
list_val1 = val_pb4.list_value.add()
list_val1.indexed = False
list_val1.meaning = meaning = 22
list_val1.blob_value = b'\xe2\x98\x83'
list_val2 = val_pb4.list_value.add()
list_val2.indexed = False
list_val2.meaning = meaning
list_val2.blob_value = b'\xe2\x98\x85'
# Convert to the user-space Entity.
entity = entity_from_protobuf(original_pb)
# Convert the user-space Entity back to a protobuf.
new_pb = self._callFUT(entity)
# NOTE: entity_to_protobuf() strips the project so we "cheat".
new_pb.key.partition_id.dataset_id = project
self._compareEntityProto(original_pb, new_pb)
def test_get_multi_w_deferred_from_backend_but_not_passed(self):
from gcloud.datastore._generated import entity_pb2
from gcloud.datastore.entity import Entity
from gcloud.datastore.key import Key
key1 = Key('Kind', project=self.PROJECT)
key1_pb = key1.to_protobuf()
key2 = Key('Kind', 2345, project=self.PROJECT)
key2_pb = key2.to_protobuf()
entity1_pb = entity_pb2.Entity()
entity1_pb.key.CopyFrom(key1_pb)
entity2_pb = entity_pb2.Entity()
entity2_pb.key.CopyFrom(key2_pb)
creds = object()
client = self._makeOne(credentials=creds)
# mock up two separate requests
client.connection._add_lookup_result([entity1_pb], deferred=[key2_pb])
client.connection._add_lookup_result([entity2_pb])
missing = []
found = client.get_multi([key1, key2], missing=missing)
self.assertEqual(len(found), 2)
self.assertEqual(len(missing), 0)
# Check the actual contents on the response.
self.assertTrue(isinstance(found[0], Entity))
self.assertEqual(found[0].key.path, key1.path)
self.assertEqual(found[0].key.project, key1.project)
self.assertTrue(isinstance(found[1], Entity))
self.assertEqual(found[1].key.path, key2.path)
self.assertEqual(found[1].key.project, key2.project)
cw = client.connection._lookup_cw
self.assertEqual(len(cw), 2)
ds_id, k_pbs, eventual, tid = cw[0]
self.assertEqual(ds_id, self.PROJECT)
self.assertEqual(len(k_pbs), 2)
self.assertEqual(key1_pb, k_pbs[0])
self.assertEqual(key2_pb, k_pbs[1])
self.assertFalse(eventual)
self.assertTrue(tid is None)
ds_id, k_pbs, eventual, tid = cw[1]
self.assertEqual(ds_id, self.PROJECT)
self.assertEqual(len(k_pbs), 1)
self.assertEqual(key2_pb, k_pbs[0])
self.assertFalse(eventual)
self.assertTrue(tid is None)
def test_get_multi_miss_w_missing(self):
from gcloud.datastore._generated import entity_pb2
from gcloud.datastore.key import Key
KIND = 'Kind'
ID = 1234
# Make a missing entity pb to be returned from mock backend.
missed = entity_pb2.Entity()
missed.key.partition_id.dataset_id = self.PROJECT
path_element = missed.key.path_element.add()
path_element.kind = KIND
path_element.id = ID
creds = object()
client = self._makeOne(credentials=creds)
# Set missing entity on mock connection.
client.connection._add_lookup_result(missing=[missed])
key = Key(KIND, ID, project=self.PROJECT)
missing = []
entities = client.get_multi([key], missing=missing)
self.assertEqual(entities, [])
self.assertEqual([missed.key.to_protobuf() for missed in missing],
[key.to_protobuf()])
def test_run_query_w_namespace_nonempty_result(self):
from gcloud.datastore._generated import datastore_pb2
from gcloud.datastore._generated import entity_pb2
PROJECT = 'PROJECT'
KIND = 'Kind'
entity = entity_pb2.Entity()
q_pb = self._make_query_pb(KIND)
rsp_pb = datastore_pb2.RunQueryResponse()
rsp_pb.batch.entity_results.add(entity=entity)
rsp_pb.batch.entity_result_type = 1 # FULL
rsp_pb.batch.more_results = 3 # NO_MORE_RESULTS
conn = self._makeOne()
URI = '/'.join([
conn.api_base_url,
conn.API_VERSION,
'projects',
PROJECT + ':runQuery',
])
http = conn._http = Http({'status': '200'}, rsp_pb.SerializeToString())
pbs = conn.run_query(PROJECT, q_pb, 'NS')[0]
self.assertEqual(len(pbs), 1)
cw = http._called_with
self._verifyProtobufCall(cw, URI, conn)
rq_class = datastore_pb2.RunQueryRequest
request = rq_class()
request.ParseFromString(cw['body'])
self.assertEqual(request.partition_id.namespace_id, 'NS')
self.assertEqual(request.query, q_pb)
def test_lookup_single_key_nonempty_response(self):
from gcloud.datastore._generated import datastore_pb2
from gcloud.datastore._generated import entity_pb2
PROJECT = 'PROJECT'
key_pb = self._make_key_pb(PROJECT)
rsp_pb = datastore_pb2.LookupResponse()
entity = entity_pb2.Entity()
entity.key.CopyFrom(key_pb)
rsp_pb.found.add(entity=entity)
conn = self._makeOne()
URI = '/'.join([
conn.api_base_url,
conn.API_VERSION,
'projects',
PROJECT + ':lookup',
])
http = conn._http = Http({'status': '200'}, rsp_pb.SerializeToString())
(found, ), missing, deferred = conn.lookup(PROJECT, [key_pb])
self.assertEqual(len(missing), 0)
self.assertEqual(len(deferred), 0)
self.assertEqual(found.key.path[0].kind, 'Kind')
self.assertEqual(found.key.path[0].id, 1234)
cw = http._called_with
self._verifyProtobufCall(cw, URI, conn)
rq_class = datastore_pb2.LookupRequest
request = rq_class()
request.ParseFromString(cw['body'])
keys = list(request.keys)
self.assertEqual(len(keys), 1)
self.assertEqual(key_pb, keys[0])
def test_empty(self):
from gcloud.datastore._generated import entity_pb2
from gcloud.datastore.entity import Entity
entity = Entity()
entity_pb = self._callFUT(entity)
self._compareEntityProto(entity_pb, entity_pb2.Entity())
def test_variable_meanings(self):
from gcloud.datastore._generated import entity_pb2
from gcloud.datastore.entity import Entity
from gcloud.datastore.helpers import _new_value_pb
entity = Entity()
name = 'quux'
entity[name] = values = [1, 20, 300]
meaning = 9
entity._meanings[name] = ([None, meaning, None], values)
entity_pb = self._callFUT(entity)
# Construct the expected protobuf.
expected_pb = entity_pb2.Entity()
value_pb = _new_value_pb(expected_pb, name)
value0 = value_pb.array_value.values.add()
value0.integer_value = values[0]
# The only array entry with a meaning is the middle one.
value1 = value_pb.array_value.values.add()
value1.integer_value = values[1]
value1.meaning = meaning
value2 = value_pb.array_value.values.add()
value2.integer_value = values[2]
self._compareEntityProto(entity_pb, expected_pb)
def _addQueryResults(self,
connection,
cursor=_END,
more=False,
skipped_results=None,
no_entity=False):
from gcloud.datastore._generated import entity_pb2
from gcloud.datastore._generated import query_pb2
from gcloud.datastore.helpers import _new_value_pb
if more:
more_enum = query_pb2.QueryResultBatch.NOT_FINISHED
else:
more_enum = query_pb2.QueryResultBatch.MORE_RESULTS_AFTER_LIMIT
_ID = 123
if no_entity:
entities = []
else:
entity_pb = entity_pb2.Entity()
entity_pb.key.partition_id.project_id = self._PROJECT
path_element = entity_pb.key.path.add()
path_element.kind = self._KIND
path_element.id = _ID
value_pb = _new_value_pb(entity_pb, 'foo')
value_pb.string_value = u'Foo'
entities = [entity_pb]
connection._results.append(
(entities, cursor, more_enum, skipped_results))
def test_entity_no_key(self):
from gcloud.datastore._generated import entity_pb2
entity_pb = entity_pb2.Entity()
entity = self._callFUT(entity_pb)
self.assertEqual(entity.key, None)
self.assertEqual(dict(entity), {})
def test_it(self):
from gcloud.datastore._generated import entity_pb2
entity_pb = entity_pb2.Entity()
name = 'foo'
result = self._callFUT(entity_pb, name)
self.assertTrue(isinstance(result, entity_pb2.Value))
self.assertEqual(len(entity_pb.properties), 1)
self.assertEqual(entity_pb.properties[name], result)
def test_it(self):
from gcloud.datastore._generated import entity_pb2
from gcloud.datastore.helpers import _new_value_pb
_PROJECT = 'PROJECT'
_KIND = 'KIND'
_ID = 1234
entity_pb = entity_pb2.Entity()
entity_pb.key.partition_id.dataset_id = _PROJECT
entity_pb.key.path_element.add(kind=_KIND, id=_ID)
value_pb = _new_value_pb(entity_pb, 'foo')
value_pb.string_value = 'Foo'
unindexed_val_pb = _new_value_pb(entity_pb, 'bar')
unindexed_val_pb.integer_value = 10
unindexed_val_pb.indexed = False
list_val_pb1 = _new_value_pb(entity_pb, 'baz')
list_pb1 = list_val_pb1.list_value
unindexed_list_val_pb = list_pb1.add()
unindexed_list_val_pb.integer_value = 11
unindexed_list_val_pb.indexed = False
list_val_pb2 = _new_value_pb(entity_pb, 'qux')
list_pb2 = list_val_pb2.list_value
indexed_list_val_pb = list_pb2.add()
indexed_list_val_pb.integer_value = 12
indexed_list_val_pb.indexed = True
entity = self._callFUT(entity_pb)
self.assertEqual(entity.kind, _KIND)
self.assertEqual(entity.exclude_from_indexes, frozenset(['bar',
'baz']))
entity_props = dict(entity)
self.assertEqual(entity_props, {
'foo': 'Foo',
'bar': 10,
'baz': [11],
'qux': [12]
})
# Also check the key.
key = entity.key
self.assertEqual(key.project, _PROJECT)
self.assertEqual(key.namespace, None)
self.assertEqual(key.kind, _KIND)
self.assertEqual(key.id, _ID)
def _make_entity_pb(project, kind, integer_id, name=None, str_val=None):
from gcloud.datastore._generated import entity_pb2
from gcloud.datastore.helpers import _new_value_pb
entity_pb = entity_pb2.Entity()
entity_pb.key.partition_id.dataset_id = project
path_element = entity_pb.key.path_element.add()
path_element.kind = kind
path_element.id = integer_id
if name is not None and str_val is not None:
value_pb = _new_value_pb(entity_pb, name)
value_pb.string_value = str_val
return entity_pb
def test_it(self):
import types
from gcloud.datastore._generated import entity_pb2
from gcloud.datastore.helpers import _new_value_pb
entity_pb = entity_pb2.Entity()
name1 = 'foo'
name2 = 'bar'
val_pb1 = _new_value_pb(entity_pb, name1)
val_pb2 = _new_value_pb(entity_pb, name2)
result = self._callFUT(entity_pb)
self.assertTrue(isinstance(result, types.GeneratorType))
self.assertEqual(list(result), [(name1, val_pb1), (name2, val_pb2)])
def test_entity_with_meaning(self):
from gcloud.datastore._generated import entity_pb2
from gcloud.datastore.helpers import _new_value_pb
entity_pb = entity_pb2.Entity()
name = 'hello'
value_pb = _new_value_pb(entity_pb, name)
value_pb.meaning = meaning = 9
value_pb.string_value = val = u'something'
entity = self._callFUT(entity_pb)
self.assertEqual(entity.key, None)
self.assertEqual(dict(entity), {name: val})
self.assertEqual(entity._meanings, {name: (meaning, val)})
def _addQueryResults(self, connection, cursor=_END, more=False):
from gcloud.datastore._generated import entity_pb2
from gcloud.datastore._generated import query_pb2
from gcloud.datastore.helpers import _new_value_pb
MORE = query_pb2.QueryResultBatch.NOT_FINISHED
NO_MORE = query_pb2.QueryResultBatch.MORE_RESULTS_AFTER_LIMIT
_ID = 123
entity_pb = entity_pb2.Entity()
entity_pb.key.partition_id.dataset_id = self._PROJECT
path_element = entity_pb.key.path_element.add()
path_element.kind = self._KIND
path_element.id = _ID
value_pb = _new_value_pb(entity_pb, 'foo')
value_pb.string_value = u'Foo'
connection._results.append(
([entity_pb], cursor, MORE if more else NO_MORE))
def test_meaning_with_change(self):
from gcloud.datastore._generated import entity_pb2
from gcloud.datastore.entity import Entity
from gcloud.datastore.helpers import _new_value_pb
entity = Entity()
name = 'foo'
entity[name] = value = 42
entity._meanings[name] = (9, 1337)
entity_pb = self._callFUT(entity)
expected_pb = entity_pb2.Entity()
value_pb = _new_value_pb(expected_pb, name)
value_pb.integer_value = value
# NOTE: No meaning is used since the value differs from the
# value stored.
self._compareEntityProto(entity_pb, expected_pb)
def entity_to_protobuf(entity):
"""Converts an entity into a protobuf.
:type entity: :class:`gcloud.datastore.entity.Entity`
:param entity: The entity to be turned into a protobuf.
:rtype: :class:`gcloud.datastore._generated.entity_pb2.Entity`
:returns: The protobuf representing the entity.
"""
entity_pb = _entity_pb2.Entity()
if entity.key is not None:
key_pb = entity.key.to_protobuf()
entity_pb.key.CopyFrom(key_pb)
for name, value in entity.items():
value_is_list = isinstance(value, list)
if value_is_list and len(value) == 0:
continue
value_pb = _new_value_pb(entity_pb, name)
# Set the appropriate value.
_set_protobuf_value(value_pb, value)
# Add index information to protobuf.
if name in entity.exclude_from_indexes:
if not value_is_list:
value_pb.indexed = False
for sub_value in value_pb.list_value:
sub_value.indexed = False
# Add meaning information to protobuf.
if name in entity._meanings:
meaning, orig_value = entity._meanings[name]
# Only add the meaning back to the protobuf if the value is
# unchanged from when it was originally read from the API.
if orig_value is value:
# For lists, we set meaning on each sub-element.
if value_is_list:
for sub_value_pb in value_pb.list_value:
sub_value_pb.meaning = meaning
else:
value_pb.meaning = meaning
return entity_pb
def test_key_only(self):
from gcloud.datastore._generated import entity_pb2
from gcloud.datastore.entity import Entity
from gcloud.datastore.key import Key
kind, name = 'PATH', 'NAME'
project = 'PROJECT'
key = Key(kind, name, project=project)
entity = Entity(key=key)
entity_pb = self._callFUT(entity)
expected_pb = entity_pb2.Entity()
expected_pb.key.partition_id.dataset_id = project
path_elt = expected_pb.key.path_element.add()
path_elt.kind = kind
path_elt.name = name
self._compareEntityProto(entity_pb, expected_pb)
def test_simple_fields(self):
from gcloud.datastore._generated import entity_pb2
from gcloud.datastore.entity import Entity
from gcloud.datastore.helpers import _new_value_pb
entity = Entity()
name1 = 'foo'
entity[name1] = value1 = 42
name2 = 'bar'
entity[name2] = value2 = u'some-string'
entity_pb = self._callFUT(entity)
expected_pb = entity_pb2.Entity()
val_pb1 = _new_value_pb(expected_pb, name1)
val_pb1.integer_value = value1
val_pb2 = _new_value_pb(expected_pb, name2)
val_pb2.string_value = value2
self._compareEntityProto(entity_pb, expected_pb)
def lookup(self, project, key_pbs):
from gcloud.datastore._generated import entity_pb2
# Store the arguments called with.
self._called_project = project
self._called_key_pbs = key_pbs
key_pb, = key_pbs
response = entity_pb2.Entity()
response.key.CopyFrom(key_pb)
response.key.partition_id.dataset_id = self.prefix + project
missing = []
deferred = []
if self.from_missing:
missing[:] = [response]
self._lookup_result = []
else:
self._lookup_result = [response]
return self._lookup_result, missing, deferred
def entity_to_protobuf(entity):
"""Converts an entity into a protobuf.
:type entity: :class:`gcloud.datastore.entity.Entity`
:param entity: The entity to be turned into a protobuf.
:rtype: :class:`gcloud.datastore._generated.entity_pb2.Entity`
:returns: The protobuf representing the entity.
"""
entity_pb = _entity_pb2.Entity()
if entity.key is not None:
key_pb = entity.key.to_protobuf()
entity_pb.key.CopyFrom(key_pb)
for name, value in entity.items():
value_is_list = isinstance(value, list)
if value_is_list and len(value) == 0:
continue
value_pb = _new_value_pb(entity_pb, name)
# Set the appropriate value.
_set_protobuf_value(value_pb, value)
# Add index information to protobuf.
if name in entity.exclude_from_indexes:
if not value_is_list:
value_pb.exclude_from_indexes = True
for sub_value in value_pb.array_value.values:
sub_value.exclude_from_indexes = True
# Add meaning information to protobuf.
_set_pb_meaning_from_entity(entity,
name,
value,
value_pb,
is_list=value_is_list)
return entity_pb
def test_mismatched_value_indexed(self):
from gcloud.datastore._generated import entity_pb2
from gcloud.datastore.helpers import _new_value_pb
_PROJECT = 'PROJECT'
_KIND = 'KIND'
_ID = 1234
entity_pb = entity_pb2.Entity()
entity_pb.key.partition_id.project_id = _PROJECT
entity_pb.key.path.add(kind=_KIND, id=_ID)
array_val_pb = _new_value_pb(entity_pb, 'baz')
array_pb = array_val_pb.array_value.values
unindexed_value_pb1 = array_pb.add()
unindexed_value_pb1.integer_value = 10
unindexed_value_pb1.exclude_from_indexes = True
unindexed_value_pb2 = array_pb.add()
unindexed_value_pb2.integer_value = 11
with self.assertRaises(ValueError):
self._callFUT(entity_pb)
def test_mismatched_value_indexed(self):
from gcloud.datastore._generated import entity_pb2
from gcloud.datastore.helpers import _new_value_pb
_PROJECT = 'PROJECT'
_KIND = 'KIND'
_ID = 1234
entity_pb = entity_pb2.Entity()
entity_pb.key.partition_id.dataset_id = _PROJECT
entity_pb.key.path_element.add(kind=_KIND, id=_ID)
list_val_pb = _new_value_pb(entity_pb, 'baz')
list_pb = list_val_pb.list_value
unindexed_value_pb1 = list_pb.add()
unindexed_value_pb1.integer_value = 10
unindexed_value_pb1.indexed = False
unindexed_value_pb2 = list_pb.add()
unindexed_value_pb2.integer_value = 11
unindexed_value_pb2.indexed = True
with self.assertRaises(ValueError):
self._callFUT(entity_pb)