def test_nested_entity_no_key(self):
from google.cloud.proto.datastore.v1 import entity_pb2
from google.cloud.datastore.helpers import _new_value_pb
PROJECT = 'FOO'
KIND = 'KIND'
INSIDE_NAME = 'IFOO'
OUTSIDE_NAME = 'OBAR'
INSIDE_VALUE = 1337
entity_inside = entity_pb2.Entity()
inside_val_pb = _new_value_pb(entity_inside, INSIDE_NAME)
inside_val_pb.integer_value = INSIDE_VALUE
entity_pb = entity_pb2.Entity()
entity_pb.key.partition_id.project_id = PROJECT
element = entity_pb.key.path.add()
element.kind = KIND
outside_val_pb = _new_value_pb(entity_pb, OUTSIDE_NAME)
outside_val_pb.entity_value.CopyFrom(entity_inside)
entity = self._call_fut(entity_pb)
self.assertEqual(entity.key.project, PROJECT)
self.assertEqual(entity.key.flat_path, (KIND,))
self.assertEqual(len(entity), 1)
inside_entity = entity[OUTSIDE_NAME]
self.assertIsNone(inside_entity.key)
self.assertEqual(len(inside_entity), 1)
self.assertEqual(inside_entity[INSIDE_NAME], INSIDE_VALUE)
def test_get_multi_w_deferred_from_backend_but_not_passed(self):
from google.cloud.proto.datastore.v1 import entity_pb2
from google.cloud.datastore.entity import Entity
from google.cloud.datastore.key import Key
key1 = Key('Kind', project=self.PROJECT)
key1_pb = key1.to_protobuf()
key2 = Key('Kind', 2345, project=self.PROJECT)
key2_pb = key2.to_protobuf()
entity1_pb = entity_pb2.Entity()
entity1_pb.key.CopyFrom(key1_pb)
entity2_pb = entity_pb2.Entity()
entity2_pb.key.CopyFrom(key2_pb)
creds = _make_credentials()
client = self._make_one(credentials=creds)
# mock up two separate requests
client._connection._add_lookup_result([entity1_pb], deferred=[key2_pb])
client._connection._add_lookup_result([entity2_pb])
missing = []
found = client.get_multi([key1, key2], missing=missing)
self.assertEqual(len(found), 2)
self.assertEqual(len(missing), 0)
# Check the actual contents on the response.
self.assertIsInstance(found[0], Entity)
self.assertEqual(found[0].key.path, key1.path)
self.assertEqual(found[0].key.project, key1.project)
self.assertIsInstance(found[1], Entity)
self.assertEqual(found[1].key.path, key2.path)
self.assertEqual(found[1].key.project, key2.project)
cw = client._connection._lookup_cw
self.assertEqual(len(cw), 2)
ds_id, k_pbs, eventual, tid = cw[0]
self.assertEqual(ds_id, self.PROJECT)
self.assertEqual(len(k_pbs), 2)
self.assertEqual(key1_pb, k_pbs[0])
self.assertEqual(key2_pb, k_pbs[1])
self.assertFalse(eventual)
self.assertIsNone(tid)
ds_id, k_pbs, eventual, tid = cw[1]
self.assertEqual(ds_id, self.PROJECT)
self.assertEqual(len(k_pbs), 1)
self.assertEqual(key2_pb, k_pbs[0])
self.assertFalse(eventual)
self.assertIsNone(tid)
def test_inverts_to_protobuf(self):
from google.cloud.proto.datastore.v1 import entity_pb2
from google.cloud.datastore.helpers import _new_value_pb
from google.cloud.datastore.helpers import entity_from_protobuf
original_pb = entity_pb2.Entity()
# Add a key.
original_pb.key.partition_id.project_id = project = 'PROJECT'
elem1 = original_pb.key.path.add()
elem1.kind = 'Family'
elem1.id = 1234
elem2 = original_pb.key.path.add()
elem2.kind = 'King'
elem2.name = 'Spades'
# Add an integer property.
val_pb1 = _new_value_pb(original_pb, 'foo')
val_pb1.integer_value = 1337
val_pb1.exclude_from_indexes = True
# Add a string property.
val_pb2 = _new_value_pb(original_pb, 'bar')
val_pb2.string_value = u'hello'
# Add a nested (entity) property.
val_pb3 = _new_value_pb(original_pb, 'entity-baz')
sub_pb = entity_pb2.Entity()
sub_val_pb1 = _new_value_pb(sub_pb, 'x')
sub_val_pb1.double_value = 3.14
sub_val_pb2 = _new_value_pb(sub_pb, 'y')
sub_val_pb2.double_value = 2.718281828
val_pb3.meaning = 9
val_pb3.entity_value.CopyFrom(sub_pb)
# Add a list property.
val_pb4 = _new_value_pb(original_pb, 'list-quux')
array_val1 = val_pb4.array_value.values.add()
array_val1.exclude_from_indexes = False
array_val1.meaning = meaning = 22
array_val1.blob_value = b'\xe2\x98\x83'
array_val2 = val_pb4.array_value.values.add()
array_val2.exclude_from_indexes = False
array_val2.meaning = meaning
array_val2.blob_value = b'\xe2\x98\x85'
# Convert to the user-space Entity.
entity = entity_from_protobuf(original_pb)
# Convert the user-space Entity back to a protobuf.
new_pb = self._call_fut(entity)
# NOTE: entity_to_protobuf() strips the project so we "cheat".
new_pb.key.partition_id.project_id = project
self._compareEntityProto(original_pb, new_pb)
def test_get_multi_miss_w_missing(self):
from google.cloud.proto.datastore.v1 import entity_pb2
from google.cloud.proto.datastore.v1 import datastore_pb2
from google.cloud.datastore.key import Key
KIND = 'Kind'
ID = 1234
# Make a missing entity pb to be returned from mock backend.
missed = entity_pb2.Entity()
missed.key.partition_id.project_id = self.PROJECT
path_element = missed.key.path.add()
path_element.kind = KIND
path_element.id = ID
creds = _make_credentials()
client = self._make_one(credentials=creds)
# Set missing entity on mock connection.
lookup_response = _make_lookup_response(missing=[missed])
ds_api = _make_datastore_api(lookup_response=lookup_response)
client._datastore_api_internal = ds_api
key = Key(KIND, ID, project=self.PROJECT)
missing = []
entities = client.get_multi([key], missing=missing)
self.assertEqual(entities, [])
key_pb = key.to_protobuf()
self.assertEqual([missed.key.to_protobuf() for missed in missing],
[key_pb])
read_options = datastore_pb2.ReadOptions()
ds_api.lookup.assert_called_once_with(self.PROJECT, read_options,
[key_pb])
def test_empty(self):
from google.cloud.proto.datastore.v1 import entity_pb2
from google.cloud.datastore.entity import Entity
entity = Entity()
entity_pb = self._call_fut(entity)
self._compareEntityProto(entity_pb, entity_pb2.Entity())
def test_variable_meanings(self):
from google.cloud.proto.datastore.v1 import entity_pb2
from google.cloud.datastore.entity import Entity
from google.cloud.datastore.helpers import _new_value_pb
entity = Entity()
name = 'quux'
entity[name] = values = [1, 20, 300]
meaning = 9
entity._meanings[name] = ([None, meaning, None], values)
entity_pb = self._call_fut(entity)
# Construct the expected protobuf.
expected_pb = entity_pb2.Entity()
value_pb = _new_value_pb(expected_pb, name)
value0 = value_pb.array_value.values.add()
value0.integer_value = values[0]
# The only array entry with a meaning is the middle one.
value1 = value_pb.array_value.values.add()
value1.integer_value = values[1]
value1.meaning = meaning
value2 = value_pb.array_value.values.add()
value2.integer_value = values[2]
self._compareEntityProto(entity_pb, expected_pb)
def to_entity(line):
entity = entity_pb2.Entity()
fields = line.split(
',') #id,president,startYear,endYear,party,homeState,dateOfBirth
id = fields[0]
president = fields[1]
names = president.split(' ')
firstName = names[0]
lastName = names[1]
startYear = fields[2]
endYear = fields[3]
party = fields[4]
homeState = fields[5]
dateOfBirth = fields[6]
googledatastore.helper.add_key_path(entity.key, kind, str(id))
googledatastore.helper.add_properties(
entity, {
'firstName': unicode(firstName),
'lastName': unicode(lastName),
'startYear': int(startYear),
'endYear': int(endYear),
'party': unicode(party),
'homeState': unicode(homeState),
'dateOfBirth': datetime.strptime(dateOfBirth, '%Y-%m-%d')
})
return entity
def create_ds_entity(element):
entity = entity_pb2.Entity()
kind = 'FileMetaData'
datastore_helper.add_key_path(entity.key, kind, str(uuid.uuid4()))
datastore_helper.add_properties(entity, element)
return entity
def make_entity(self, content):
"""Create entity and set properties."""
entity = entity_pb2.Entity()
# We use uuid for keys.
datastore_helper.add_key_path(entity.key, self._kind, str(uuid.uuid4()))
datastore_helper.add_properties(entity, content)
return entity
def test_run_query_w_namespace_nonempty_result(self):
from google.cloud.proto.datastore.v1 import datastore_pb2
from google.cloud.proto.datastore.v1 import entity_pb2
from google.cloud.proto.datastore.v1 import query_pb2
project = 'PROJECT'
kind = 'Kind'
entity = entity_pb2.Entity()
q_pb = self._make_query_pb(kind)
rsp_pb = datastore_pb2.RunQueryResponse()
rsp_pb.batch.entity_results.add(entity=entity)
rsp_pb.batch.entity_result_type = query_pb2.EntityResult.FULL
rsp_pb.batch.more_results = query_pb2.QueryResultBatch.NO_MORE_RESULTS
# Create mock HTTP and client with response.
http = Http({'status': '200'}, rsp_pb.SerializeToString())
client = mock.Mock(_http=http,
_base_url='test.invalid',
spec=['_http', '_base_url'])
# Make request.
conn = self._make_one(client)
namespace = 'NS'
response = conn.run_query(project, q_pb, namespace=namespace)
# Check the result and verify the callers.
self.assertEqual(response, rsp_pb)
cw = http._called_with
uri = _build_expected_url(conn.api_base_url, project, 'runQuery')
_verify_protobuf_call(self, cw, uri)
request = datastore_pb2.RunQueryRequest()
request.ParseFromString(cw['body'])
self.assertEqual(request.partition_id.namespace_id, namespace)
self.assertEqual(request.query, q_pb)
def test_lookup_single_key_nonempty_response(self):
from google.cloud.proto.datastore.v1 import datastore_pb2
from google.cloud.proto.datastore.v1 import entity_pb2
PROJECT = 'PROJECT'
key_pb = self._make_key_pb(PROJECT)
rsp_pb = datastore_pb2.LookupResponse()
entity = entity_pb2.Entity()
entity.key.CopyFrom(key_pb)
rsp_pb.found.add(entity=entity)
http = Http({'status': '200'}, rsp_pb.SerializeToString())
client = mock.Mock(_http=http, spec=['_http'])
conn = self._make_one(client)
URI = '/'.join([
conn.api_base_url,
conn.API_VERSION,
'projects',
PROJECT + ':lookup',
])
(found, ), missing, deferred = conn.lookup(PROJECT, [key_pb])
self.assertEqual(len(missing), 0)
self.assertEqual(len(deferred), 0)
self.assertEqual(found.key.path[0].kind, 'Kind')
self.assertEqual(found.key.path[0].id, 1234)
cw = http._called_with
self._verify_protobuf_call(cw, URI, conn)
rq_class = datastore_pb2.LookupRequest
request = rq_class()
request.ParseFromString(cw['body'])
keys = list(request.keys)
self.assertEqual(len(keys), 1)
self.assertEqual(key_pb, keys[0])
def test_get_multi_miss_w_missing(self):
from google.cloud.proto.datastore.v1 import entity_pb2
from google.cloud.datastore.key import Key
KIND = 'Kind'
ID = 1234
# Make a missing entity pb to be returned from mock backend.
missed = entity_pb2.Entity()
missed.key.partition_id.project_id = self.PROJECT
path_element = missed.key.path.add()
path_element.kind = KIND
path_element.id = ID
creds = _make_credentials()
client = self._make_one(credentials=creds)
# Set missing entity on mock connection.
client._connection._add_lookup_result(missing=[missed])
key = Key(KIND, ID, project=self.PROJECT)
missing = []
entities = client.get_multi([key], missing=missing)
self.assertEqual(entities, [])
self.assertEqual([missed.key.to_protobuf() for missed in missing],
[key.to_protobuf()])
def build_user_entity(row):
entity = entity_pb2.Entity()
datastore_helper.add_key_path(entity.key, 'User', row['email'])
props = dict(row)
del props['email']
datastore_helper.add_properties(entity, props)
return entity
def to_entity(content):
entity = entity_pb2.Entity()
googledatastore.helper.add_key_path(entity.key, kind,
str(uuid.uuid4()))
googledatastore.helper.add_properties(entity,
{'content': unicode(content)})
return entity
def test_dict_to_entity_recursive(self):
from google.cloud.proto.datastore.v1 import entity_pb2
from google.cloud.datastore.entity import Entity
entity = Entity()
entity['a'] = {
'b': {
'c': {
'd': 1.25,
},
'e': True,
},
'f': 10,
}
entity_pb = self._call_fut(entity)
b_entity_pb = entity_pb2.Entity(
properties={
'c': entity_pb2.Value(
entity_value=entity_pb2.Entity(
properties={
'd': entity_pb2.Value(
double_value=1.25,
),
},
),
),
'e': entity_pb2.Value(boolean_value=True),
}
)
expected_pb = entity_pb2.Entity(
properties={
'a': entity_pb2.Value(
entity_value=entity_pb2.Entity(
properties={
'b': entity_pb2.Value(
entity_value=b_entity_pb,
),
'f': entity_pb2.Value(
integer_value=10,
),
},
),
),
},
)
self.assertEqual(entity_pb, expected_pb)
def test_get_multi_w_deferred_from_backend_but_not_passed(self):
from google.cloud.proto.datastore.v1 import datastore_pb2
from google.cloud.proto.datastore.v1 import entity_pb2
from google.cloud.datastore.entity import Entity
from google.cloud.datastore.key import Key
key1 = Key('Kind', project=self.PROJECT)
key1_pb = key1.to_protobuf()
key2 = Key('Kind', 2345, project=self.PROJECT)
key2_pb = key2.to_protobuf()
entity1_pb = entity_pb2.Entity()
entity1_pb.key.CopyFrom(key1_pb)
entity2_pb = entity_pb2.Entity()
entity2_pb.key.CopyFrom(key2_pb)
creds = _make_credentials()
client = self._make_one(credentials=creds)
# Mock up two separate requests. Using an iterable as side_effect
# allows multiple return values.
lookup_response1 = _make_lookup_response(results=[entity1_pb],
deferred=[key2_pb])
lookup_response2 = _make_lookup_response(results=[entity2_pb])
ds_api = _make_datastore_api()
ds_api.lookup = mock.Mock(
side_effect=[lookup_response1, lookup_response2], spec=[])
client._datastore_api_internal = ds_api
missing = []
found = client.get_multi([key1, key2], missing=missing)
self.assertEqual(len(found), 2)
self.assertEqual(len(missing), 0)
# Check the actual contents on the response.
self.assertIsInstance(found[0], Entity)
self.assertEqual(found[0].key.path, key1.path)
self.assertEqual(found[0].key.project, key1.project)
self.assertIsInstance(found[1], Entity)
self.assertEqual(found[1].key.path, key2.path)
self.assertEqual(found[1].key.project, key2.project)
self.assertEqual(ds_api.lookup.call_count, 2)
read_options = datastore_pb2.ReadOptions()
ds_api.lookup.assert_any_call(self.PROJECT, read_options, [key2_pb])
ds_api.lookup.assert_any_call(self.PROJECT, read_options,
[key1_pb, key2_pb])
def make_entity(self, content):
entity = entity_pb2.Entity()
if self._namespace is not None:
entity.key.partition_id.namespace_id = self._namespace
helper.add_key_path(entity.key, self._kind, str(uuid.uuid4()))
helper.add_properties(entity, content)
return entity
def test_entity_no_key(self):
from google.cloud.proto.datastore.v1 import entity_pb2
entity_pb = entity_pb2.Entity()
entity = self._call_fut(entity_pb)
self.assertIsNone(entity.key)
self.assertEqual(dict(entity), {})
def _make_entity(kind, id_, project):
from google.cloud.proto.datastore.v1 import entity_pb2
key = entity_pb2.Key()
key.partition_id.project_id = project
elem = key.path.add()
elem.kind = kind
elem.id = id_
return entity_pb2.Entity(key=key)
def test_with_empty_list(self):
from google.cloud.proto.datastore.v1 import entity_pb2
from google.cloud.datastore.entity import Entity
entity = Entity()
entity['foo'] = []
entity_pb = self._call_fut(entity)
self._compare_entity_proto(entity_pb, entity_pb2.Entity())
def build_company_entity(data):
entity = entity_pb2.Entity()
datastore_helper.add_key_path(entity.key, 'Company', data[0])
props = {}
props['reg_date'] = data[1]
props['boxes_bought'] = 0
props['boxes_prov'] = 0
datastore_helper.add_properties(entity, props)
return entity
def make_entity(self, prefix, candidates):
entity = entity_pb2.Entity()
datastore_helper.add_key_path(entity.key, "prefix", prefix, "id",
str(uuid.uuid4()), "candidates",
unicode(candidates))
# All entities created will have the same ancestor
#datastore_helper.add_properties(entity, {"wordlist": unicode(candidates)})
#datastore_helper.add_properties(entity, {"candidates": unicode(candidates)})
return entity
def test_it(self):
from google.cloud.proto.datastore.v1 import entity_pb2
entity_pb = entity_pb2.Entity()
name = 'foo'
result = self._call_fut(entity_pb, name)
self.assertIsInstance(result, entity_pb2.Value)
self.assertEqual(len(entity_pb.properties), 1)
self.assertEqual(entity_pb.properties[name], result)
def make_entity(self, content):
entity = entity_pb2.Entity()
if self._namespace is not None:
entity.key.partition_id.namespace_id = self._namespace
# All entities created will have the same ancestor
datastore_helper.add_key_path(entity.key, self._kind, self._ancestor,
self._kind, str(uuid.uuid4()))
datastore_helper.add_properties(entity, {"content": unicode(content)})
return entity
def make_entity(self, content):
"""Create entity from given string."""
entity = entity_pb2.Entity()
if self._namespace is not None:
entity.key.partition_id.namespace_id = self._namespace
# All entities created will have the same ancestor
datastore_helper.add_key_path(entity.key, self._kind, self._ancestor,
self._kind, hashlib.sha1(content).hexdigest())
datastore_helper.add_properties(entity, {'content': str(content)})
return entity
def test_dict_to_entity(self):
from google.cloud.proto.datastore.v1 import entity_pb2
from google.cloud.datastore.entity import Entity
entity = Entity()
entity['a'] = {'b': u'c'}
entity_pb = self._call_fut(entity)
expected_pb = entity_pb2.Entity(
properties={
'a': entity_pb2.Value(
entity_value=entity_pb2.Entity(
properties={
'b': entity_pb2.Value(
string_value='c',
),
},
),
),
},
)
self.assertEqual(entity_pb, expected_pb)
def process(self, idx):
entity = entity_pb2.Entity()
embed = entity_pb2.Entity() # embedded document
ss = {
'ss': unicode('X'), # string
'bb': False, # boolean
'll': [1, 2, 3], # list of integers
'ii': 123 # integer
}
datastore_helper.add_properties(
embed, ss) # setting properties for embedded document
element = dict()
element['s'] = unicode('String') # string
element['b'] = True # boolean
element['l'] = [unicode('s'), unicode('a'),
unicode('b')] # list of strings
element['i'] = 1 # integer
element['e'] = embed # embedded document
datastore_helper.add_key_path(entity.key, 'ds',
idx) # setting id for document
datastore_helper.add_properties(
entity, element) # setting properties for document
return [entity]
def test_entity_with_meaning(self):
from google.cloud.proto.datastore.v1 import entity_pb2
from google.cloud.datastore.helpers import _new_value_pb
entity_pb = entity_pb2.Entity()
name = 'hello'
value_pb = _new_value_pb(entity_pb, name)
value_pb.meaning = meaning = 9
value_pb.string_value = val = u'something'
entity = self._call_fut(entity_pb)
self.assertIsNone(entity.key)
self.assertEqual(dict(entity), {name: val})
self.assertEqual(entity._meanings, {name: (meaning, val)})
def _make_entity_pb(project, kind, integer_id, name=None, str_val=None):
from google.cloud.proto.datastore.v1 import entity_pb2
from google.cloud.datastore.helpers import _new_value_pb
entity_pb = entity_pb2.Entity()
entity_pb.key.partition_id.project_id = project
path_element = entity_pb.key.path.add()
path_element.kind = kind
path_element.id = integer_id
if name is not None and str_val is not None:
value_pb = _new_value_pb(entity_pb, name)
value_pb.string_value = str_val
return entity_pb
def test_it(self):
import types
from google.cloud.proto.datastore.v1 import entity_pb2
from google.cloud.datastore.helpers import _new_value_pb
entity_pb = entity_pb2.Entity()
name1 = 'foo'
name2 = 'bar'
val_pb1 = _new_value_pb(entity_pb, name1)
val_pb2 = _new_value_pb(entity_pb, name2)
result = self._call_fut(entity_pb)
self.assertIsInstance(result, types.GeneratorType)
self.assertEqual(sorted(result),
sorted([(name1, val_pb1), (name2, val_pb2)]))