def test_fetch_explicit_limit(self):
from gcloud.datastore.datastore_v1_pb2 import Entity
_CURSOR = 'CURSOR'
_DATASET = 'DATASET'
_KIND = 'KIND'
_ID = 123
_NAMESPACE = 'NAMESPACE'
entity_pb = Entity()
path_element = entity_pb.key.path_element.add()
path_element.kind = _KIND
path_element.id = _ID
prop = entity_pb.property.add()
prop.name = 'foo'
prop.value.string_value = u'Foo'
connection = _Connection(entity_pb)
connection._cursor = _CURSOR
dataset = _Dataset(_DATASET, connection)
query = self._makeOne(_KIND, dataset, _NAMESPACE)
limited = query.limit(13)
entities = query.fetch(13)
self.assertEqual(query._cursor, _CURSOR)
self.assertEqual(len(entities), 1)
self.assertEqual(entities[0].key().path(), [{
'kind': _KIND,
'id': _ID
}])
expected_called_with = {
'dataset_id': _DATASET,
'query_pb': limited.to_protobuf(),
'namespace': _NAMESPACE,
}
self.assertEqual(connection._called_with, expected_called_with)
def test_put_multi_w_single_empty_entity(self):
# https://github.com/GoogleCloudPlatform/gcloud-python/issues/649
from gcloud.datastore.entity import Entity
creds = object()
client = self._makeOne(credentials=creds)
self.assertRaises(ValueError, client.put_multi, Entity())
def test_variable_meanings(self):
from gcloud.datastore._generated import entity_pb2
from gcloud.datastore.entity import Entity
from gcloud.datastore.helpers import _new_value_pb
entity = Entity()
name = 'quux'
entity[name] = values = [1, 20, 300]
meaning = 9
entity._meanings[name] = ([None, meaning, None], values)
entity_pb = self._callFUT(entity)
# Construct the expected protobuf.
expected_pb = entity_pb2.Entity()
value_pb = _new_value_pb(expected_pb, name)
value0 = value_pb.array_value.values.add()
value0.integer_value = values[0]
# The only array entry with a meaning is the middle one.
value1 = value_pb.array_value.values.add()
value1.integer_value = values[1]
value1.meaning = meaning
value2 = value_pb.array_value.values.add()
value2.integer_value = values[2]
self._compareEntityProto(entity_pb, expected_pb)
def test_fetch_default_limit(self):
from gcloud.datastore.datastore_v1_pb2 import Entity
_DATASET = 'DATASET'
_KIND = 'KIND'
_ID = 123
entity_pb = Entity()
path_element = entity_pb.key.path_element.add()
path_element.kind = _KIND
path_element.id = _ID
prop = entity_pb.property.add()
prop.name = 'foo'
prop.value.string_value = u'Foo'
connection = _Connection(entity_pb)
dataset = _Dataset(_DATASET, connection)
query = self._makeOne(_KIND, dataset)
entities = query.fetch()
self.assertEqual(len(entities), 1)
self.assertEqual(entities[0].key().path(), [{
'kind': _KIND,
'id': _ID
}])
expected_called_with = {
'dataset_id': _DATASET,
'query_pb': query.to_protobuf(),
'namespace': None,
}
self.assertEqual(connection._called_with, expected_called_with)
def test_empty(self):
from gcloud.datastore._generated import entity_pb2
from gcloud.datastore.entity import Entity
entity = Entity()
entity_pb = self._callFUT(entity)
self._compareEntityProto(entity_pb, entity_pb2.Entity())
def test_add_filter_w_known_operator_and_entity(self):
from gcloud.datastore.entity import Entity
query = self._makeOne(self._makeClient())
other = Entity()
other['firstname'] = u'John'
other['lastname'] = u'Smith'
query.add_filter('other', '=', other)
self.assertEqual(query.filters, [('other', '=', other)])
def test_entity_empty_wo_key(self):
from gcloud.datastore.entity import Entity
pb = self._makePB()
entity = Entity()
self._callFUT(pb, entity)
value = pb.entity_value
self.assertEqual(value.key.SerializeToString(), '')
props = list(value.property)
self.assertEqual(len(props), 0)
def test_entity_empty_wo_key(self):
from gcloud.datastore.entity import Entity
from gcloud.datastore.helpers import _property_tuples
pb = self._makePB()
entity = Entity()
self._callFUT(pb, entity)
value = pb.entity_value
self.assertEqual(value.key.SerializeToString(), b'')
self.assertEqual(len(list(_property_tuples(value))), 0)
def _get_report(employee_id, report_id, create=True):
key = Key('Employee', employee_id, 'Expense Report', report_id)
reports = datastore.get([key])
if len(reports) == 0:
if not create:
return None
report = Entity(key)
datastore.put([report])
else:
report, = reports
return report
def _get_employee(employee_id, create=True):
key = Key('Employee', employee_id)
employees = datastore.get([key])
if len(employees) == 0:
if not create:
return None
employee = Entity(key)
employee['created'] = employee['updated'] = datetime.datetime.utcnow()
datastore.put([employee])
else:
employee, = employees
return employee
def entity_from_protobuf(pb):
"""Factory method for creating an entity based on a protobuf.
The protobuf should be one returned from the Cloud Datastore
Protobuf API.
:type pb: :class:`gcloud.datastore._generated.entity_pb2.Entity`
:param pb: The Protobuf representing the entity.
:rtype: :class:`gcloud.datastore.entity.Entity`
:returns: The entity derived from the protobuf.
"""
key = None
if pb.HasField('key'): # Message field (Key)
key = key_from_protobuf(pb.key)
entity_props = {}
entity_meanings = {}
exclude_from_indexes = []
for prop_name, value_pb in _property_tuples(pb):
value = _get_value_from_value_pb(value_pb)
entity_props[prop_name] = value
# Check if the property has an associated meaning.
is_list = isinstance(value, list)
meaning = _get_meaning(value_pb, is_list=is_list)
if meaning is not None:
entity_meanings[prop_name] = (meaning, value)
# Check if ``value_pb`` was excluded from index. Lists need to be
# special-cased and we require all ``exclude_from_indexes`` values
# in a list agree.
if is_list:
exclude_values = set(value_pb.exclude_from_indexes
for value_pb in value_pb.array_value.values)
if len(exclude_values) != 1:
raise ValueError('For an array_value, subvalues must either '
'all be indexed or all excluded from '
'indexes.')
if exclude_values.pop():
exclude_from_indexes.append(prop_name)
else:
if value_pb.exclude_from_indexes:
exclude_from_indexes.append(prop_name)
entity = Entity(key=key, exclude_from_indexes=exclude_from_indexes)
entity.update(entity_props)
entity._meanings.update(entity_meanings)
return entity
def _fetch_page_helper(self,
cursor=b'\x00',
limit=None,
more_results=False,
_more_pb=None,
use_fetch=False):
import base64
from gcloud.datastore.datastore_v1_pb2 import Entity
_CURSOR_FOR_USER = (None
if cursor is None else base64.b64encode(cursor))
_MORE_RESULTS = more_results
_DATASET = 'DATASET'
_KIND = 'KIND'
_ID = 123
_NAMESPACE = 'NAMESPACE'
entity_pb = Entity()
path_element = entity_pb.key.path_element.add()
path_element.kind = _KIND
path_element.id = _ID
prop = entity_pb.property.add()
prop.name = 'foo'
prop.value.string_value = u'Foo'
if _more_pb is None:
connection = _Connection(entity_pb)
else:
connection = _Connection(entity_pb, _more=_more_pb)
connection._cursor = cursor
dataset = _Dataset(_DATASET, connection)
query = self._makeOne(_KIND, dataset, _NAMESPACE)
if use_fetch:
entities = query.fetch(limit)
else:
entities, cursor, more_results = query.fetch_page(limit)
self.assertEqual(cursor, _CURSOR_FOR_USER)
self.assertEqual(more_results, _MORE_RESULTS)
self.assertEqual(len(entities), 1)
self.assertEqual(entities[0].key().path(), [{
'kind': _KIND,
'id': _ID
}])
limited_query = query
if limit is not None:
limited_query = query.limit(limit)
expected_called_with = {
'dataset_id': _DATASET,
'query_pb': limited_query.to_protobuf(),
'namespace': _NAMESPACE,
}
self.assertEqual(connection._called_with, expected_called_with)
def _upsert_report(employee_id, report_id, rows):
_get_employee(employee_id) # force existence
report = _get_report(employee_id, report_id)
_purge_report_items(report)
# Add items based on rows.
report_path = list(report.key.flat_path)
for i, row in enumerate(rows):
path = report_path + ['Expense Item', i + 1]
key = Key(*path)
item = Entity(key)
for k, v in row.items():
item[k] = v
datastore.put([item])
return report
def test_entity_w_key(self):
from gcloud.datastore.entity import Entity
from gcloud.datastore.key import Key
pb = self._makePB()
key = Key('KIND', 123, dataset_id='DATASET')
entity = Entity(key=key)
entity['foo'] = u'Foo'
self._callFUT(pb, entity)
value = pb.entity_value
self.assertEqual(value.key, key.to_protobuf())
props = list(value.property)
self.assertEqual(len(props), 1)
self.assertEqual(props[0].name, 'foo')
self.assertEqual(props[0].value.string_value, u'Foo')
def entity(self, kind, exclude_from_indexes=()):
"""Create an entity bound to this dataset.
:type kind: string
:param kind: the "kind" of the new entity (see
https://cloud.google.com/datastore/docs/concepts/entities#Datastore_Kinds_and_identifiers)
:param exclude_from_indexes: names of fields whose values are not to
be indexed.
:rtype: :class:`gcloud.datastore.entity.Entity`
:returns: a new Entity instance, bound to this dataset.
"""
return Entity(dataset=self,
kind=kind,
exclude_from_indexes=exclude_from_indexes)
def test_meaning_with_change(self):
from gcloud.datastore._generated import entity_pb2
from gcloud.datastore.entity import Entity
from gcloud.datastore.helpers import _new_value_pb
entity = Entity()
name = 'foo'
entity[name] = value = 42
entity._meanings[name] = (9, 1337)
entity_pb = self._callFUT(entity)
expected_pb = entity_pb2.Entity()
value_pb = _new_value_pb(expected_pb, name)
value_pb.integer_value = value
# NOTE: No meaning is used since the value differs from the
# value stored.
self._compareEntityProto(entity_pb, expected_pb)
def test_key_only(self):
from gcloud.datastore._generated import entity_pb2
from gcloud.datastore.entity import Entity
from gcloud.datastore.key import Key
kind, name = 'PATH', 'NAME'
project = 'PROJECT'
key = Key(kind, name, project=project)
entity = Entity(key=key)
entity_pb = self._callFUT(entity)
expected_pb = entity_pb2.Entity()
expected_pb.key.partition_id.dataset_id = project
path_elt = expected_pb.key.path_element.add()
path_elt.kind = kind
path_elt.name = name
self._compareEntityProto(entity_pb, expected_pb)
def test_none(self):
from gcloud.datastore.entity import Entity
entity = Entity()
pb = self._makePB()
self._callFUT(pb, False)
self._callFUT(pb, 3.1415926)
self._callFUT(pb, 42)
self._callFUT(pb, (1 << 63) - 1)
self._callFUT(pb, 'str')
self._callFUT(pb, b'str')
self._callFUT(pb, u'str')
self._callFUT(pb, entity)
self._callFUT(pb, [u'a', 0, 3.14])
self._callFUT(pb, None)
self.assertEqual(len(pb.ListFields()), 0)
def test_entity_w_key(self):
from gcloud.datastore.entity import Entity
from gcloud.datastore.helpers import _property_tuples
from gcloud.datastore.key import Key
name = 'foo'
value = u'Foo'
pb = self._makePB()
key = Key('KIND', 123, project='PROJECT')
entity = Entity(key=key)
entity[name] = value
self._callFUT(pb, entity)
entity_pb = pb.entity_value
self.assertEqual(entity_pb.key, key.to_protobuf())
prop_dict = dict(_property_tuples(entity_pb))
self.assertEqual(len(prop_dict), 1)
self.assertEqual(list(prop_dict.keys()), [name])
self.assertEqual(prop_dict[name].string_value, value)
def test_simple_fields(self):
from gcloud.datastore._generated import entity_pb2
from gcloud.datastore.entity import Entity
from gcloud.datastore.helpers import _new_value_pb
entity = Entity()
name1 = 'foo'
entity[name1] = value1 = 42
name2 = 'bar'
entity[name2] = value2 = u'some-string'
entity_pb = self._callFUT(entity)
expected_pb = entity_pb2.Entity()
val_pb1 = _new_value_pb(expected_pb, name1)
val_pb1.integer_value = value1
val_pb2 = _new_value_pb(expected_pb, name2)
val_pb2.string_value = value2
self._compareEntityProto(entity_pb, expected_pb)
def entity_from_protobuf(pb):
"""Factory method for creating an entity based on a protobuf.
The protobuf should be one returned from the Cloud Datastore
Protobuf API.
:type pb: :class:`gcloud.datastore._datastore_v1_pb2.Entity`
:param pb: The Protobuf representing the entity.
:rtype: :class:`gcloud.datastore.entity.Entity`
:returns: The entity derived from the protobuf.
"""
key = None
if pb.HasField('key'):
key = key_from_protobuf(pb.key)
entity_props = {}
exclude_from_indexes = []
for property_pb in pb.property:
value = _get_value_from_property_pb(property_pb)
entity_props[property_pb.name] = value
# Check if property_pb.value was indexed. Lists need to be
# special-cased and we require all `indexed` values in a list agree.
if isinstance(value, list):
indexed_values = set(value_pb.indexed
for value_pb in property_pb.value.list_value)
if len(indexed_values) != 1:
raise ValueError('For a list_value, subvalues must either all '
'be indexed or all excluded from indexes.')
if not indexed_values.pop():
exclude_from_indexes.append(property_pb.name)
else:
if not property_pb.value.indexed:
exclude_from_indexes.append(property_pb.name)
entity = Entity(key=key, exclude_from_indexes=exclude_from_indexes)
entity.update(entity_props)
return entity
def test_filter_w_known_operator_and_entity(self):
import operator
from gcloud.datastore.entity import Entity
query = self._makeOne()
other = Entity()
other['firstname'] = u'John'
other['lastname'] = u'Smith'
after = query.filter('other', '=', other)
self.assertFalse(after is query)
self.assertTrue(isinstance(after, self._getTargetClass()))
q_pb = after.to_protobuf()
self.assertEqual(q_pb.filter.composite_filter.operator, 1) # AND
f_pb, = list(q_pb.filter.composite_filter.filter)
p_pb = f_pb.property_filter
self.assertEqual(p_pb.property.name, 'other')
other_pb = p_pb.value.entity_value
props = sorted(other_pb.property, key=operator.attrgetter('name'))
self.assertEqual(len(props), 2)
self.assertEqual(props[0].name, 'firstname')
self.assertEqual(props[0].value.string_value, u'John')
self.assertEqual(props[1].name, 'lastname')
self.assertEqual(props[1].value.string_value, u'Smith')
def test_save_entity_w_transaction_nested_entity(self):
from gcloud.datastore.connection import datastore_pb
from gcloud.datastore.entity import Entity
from gcloud.datastore.key import Key
mutation = datastore_pb.Mutation()
class Xact(object):
def mutation(self):
return mutation
DATASET_ID = 'DATASET'
nested = Entity()
nested['bar'] = u'Bar'
key_pb = Key(path=[{'kind': 'Kind', 'id': 1234}]).to_protobuf()
rsp_pb = datastore_pb.CommitResponse()
conn = self._makeOne()
conn.transaction(Xact())
http = conn._http = Http({'status': '200'}, rsp_pb.SerializeToString())
result = conn.save_entity(DATASET_ID, key_pb, {'foo': nested})
self.assertEqual(result, True)
self.assertEqual(http._called_with, None)
mutation = conn.mutation()
self.assertEqual(len(mutation.upsert), 1)
def test_w_single_empty_entity(self):
# https://github.com/GoogleCloudPlatform/gcloud-python/issues/649
from gcloud.datastore.entity import Entity
self.assertRaises(ValueError, self._callFUT, Entity())
def test_entity(self):
from gcloud.datastore.entity import Entity
entity = Entity()
name, value = self._callFUT(entity)
self.assertEqual(name, 'entity_value')
self.assertTrue(value is entity)
def test_key(self):
dataset = Dataset(id='test-dataset')
entity = Entity(dataset, 'TestKind')
self.assertIsInstance(entity.key(), Key)
def test_init_sets_proper_values(self):
dataset = Dataset(id='test-dataset')
entity = Entity(dataset, 'TestKind')
self.assertEqual('test-dataset', entity.dataset().id())
self.assertEqual('TestKind', entity.kind())
def entity(self, kind): from gcloud.datastore.entity import Entity return Entity(dataset=self, kind=kind)
