def test_variable_meanings(self):
from gcloud.datastore._generated import entity_pb2
from gcloud.datastore.entity import Entity
from gcloud.datastore.helpers import _new_value_pb
entity = Entity()
name = 'quux'
entity[name] = values = [1, 20, 300]
meaning = 9
entity._meanings[name] = ([None, meaning, None], values)
entity_pb = self._callFUT(entity)
# Construct the expected protobuf.
expected_pb = entity_pb2.Entity()
value_pb = _new_value_pb(expected_pb, name)
value0 = value_pb.array_value.values.add()
value0.integer_value = values[0]
# The only array entry with a meaning is the middle one.
value1 = value_pb.array_value.values.add()
value1.integer_value = values[1]
value1.meaning = meaning
value2 = value_pb.array_value.values.add()
value2.integer_value = values[2]
self._compareEntityProto(entity_pb, expected_pb)
def entity_from_protobuf(pb):
"""Factory method for creating an entity based on a protobuf.
The protobuf should be one returned from the Cloud Datastore
Protobuf API.
:type pb: :class:`gcloud.datastore._generated.entity_pb2.Entity`
:param pb: The Protobuf representing the entity.
:rtype: :class:`gcloud.datastore.entity.Entity`
:returns: The entity derived from the protobuf.
"""
key = None
if pb.HasField('key'): # Message field (Key)
key = key_from_protobuf(pb.key)
entity_props = {}
entity_meanings = {}
exclude_from_indexes = []
for prop_name, value_pb in _property_tuples(pb):
value = _get_value_from_value_pb(value_pb)
entity_props[prop_name] = value
# Check if the property has an associated meaning.
is_list = isinstance(value, list)
meaning = _get_meaning(value_pb, is_list=is_list)
if meaning is not None:
entity_meanings[prop_name] = (meaning, value)
# Check if ``value_pb`` was excluded from index. Lists need to be
# special-cased and we require all ``exclude_from_indexes`` values
# in a list agree.
if is_list:
exclude_values = set(value_pb.exclude_from_indexes
for value_pb in value_pb.array_value.values)
if len(exclude_values) != 1:
raise ValueError('For an array_value, subvalues must either '
'all be indexed or all excluded from '
'indexes.')
if exclude_values.pop():
exclude_from_indexes.append(prop_name)
else:
if value_pb.exclude_from_indexes:
exclude_from_indexes.append(prop_name)
entity = Entity(key=key, exclude_from_indexes=exclude_from_indexes)
entity.update(entity_props)
entity._meanings.update(entity_meanings)
return entity
def test_empty(self):
from gcloud.datastore._generated import entity_pb2
from gcloud.datastore.entity import Entity
entity = Entity()
entity_pb = self._callFUT(entity)
self._compareEntityProto(entity_pb, entity_pb2.Entity())
def test_from_key(self):
key = Key(dataset=Dataset('test-dataset')).kind('TestKind').id(1234)
entity = Entity.from_key(key)
self.assertEqual('test-dataset', entity.dataset().id())
self.assertEqual('TestKind', entity.key().kind())
self.assertEqual(entity.key().kind(), entity.kind())
self.assertEqual(1234, entity.key().id())
def test_put_multi_w_single_empty_entity(self):
# https://github.com/GoogleCloudPlatform/gcloud-python/issues/649
from gcloud.datastore.entity import Entity
creds = object()
client = self._makeOne(credentials=creds)
self.assertRaises(ValueError, client.put_multi, Entity())
def test_add_filter_w_known_operator_and_entity(self):
from gcloud.datastore.entity import Entity
query = self._makeOne(self._makeClient())
other = Entity()
other['firstname'] = u'John'
other['lastname'] = u'Smith'
query.add_filter('other', '=', other)
self.assertEqual(query.filters, [('other', '=', other)])
def test_meaning_with_change(self):
from gcloud.datastore._generated import entity_pb2
from gcloud.datastore.entity import Entity
from gcloud.datastore.helpers import _new_value_pb
entity = Entity()
name = 'foo'
entity[name] = value = 42
entity._meanings[name] = (9, 1337)
entity_pb = self._callFUT(entity)
expected_pb = entity_pb2.Entity()
value_pb = _new_value_pb(expected_pb, name)
value_pb.integer_value = value
# NOTE: No meaning is used since the value differs from the
# value stored.
self._compareEntityProto(entity_pb, expected_pb)
def test_meaning_with_change(self):
from gcloud.datastore._generated import entity_pb2
from gcloud.datastore.entity import Entity
from gcloud.datastore.helpers import _new_value_pb
entity = Entity()
name = 'foo'
entity[name] = value = 42
entity._meanings[name] = (9, 1337)
entity_pb = self._callFUT(entity)
expected_pb = entity_pb2.Entity()
value_pb = _new_value_pb(expected_pb, name)
value_pb.integer_value = value
# NOTE: No meaning is used since the value differs from the
# value stored.
self._compareEntityProto(entity_pb, expected_pb)
def test_entity_empty_wo_key(self):
from gcloud.datastore.entity import Entity
pb = self._makePB()
entity = Entity()
self._callFUT(pb, entity)
value = pb.entity_value
self.assertEqual(value.key.SerializeToString(), '')
props = list(value.property)
self.assertEqual(len(props), 0)
def test_entity_empty_wo_key(self):
from gcloud.datastore.entity import Entity
from gcloud.datastore.helpers import _property_tuples
pb = self._makePB()
entity = Entity()
self._callFUT(pb, entity)
value = pb.entity_value
self.assertEqual(value.key.SerializeToString(), b'')
self.assertEqual(len(list(_property_tuples(value))), 0)
def _get_report(employee_id, report_id, create=True):
key = Key('Employee', employee_id, 'Expense Report', report_id)
reports = datastore.get([key])
if len(reports) == 0:
if not create:
return None
report = Entity(key)
datastore.put([report])
else:
report, = reports
return report
def get_entities(self, keys):
# This import is here to avoid circular references.
from gcloud.datastore.entity import Entity
entity_pbs = self.connection().lookup(dataset_id=self.id(),
key_pbs=[k.to_protobuf() for k in keys])
entities = []
for entity_pb in entity_pbs:
entities.append(Entity.from_protobuf(entity_pb, dataset=self))
return entities
def entity_from_protobuf(pb):
"""Factory method for creating an entity based on a protobuf.
The protobuf should be one returned from the Cloud Datastore
Protobuf API.
:type pb: :class:`gcloud.datastore._datastore_v1_pb2.Entity`
:param pb: The Protobuf representing the entity.
:rtype: :class:`gcloud.datastore.entity.Entity`
:returns: The entity derived from the protobuf.
"""
key = None
if pb.HasField('key'):
key = key_from_protobuf(pb.key)
entity_props = {}
exclude_from_indexes = []
for property_pb in pb.property:
value = _get_value_from_property_pb(property_pb)
entity_props[property_pb.name] = value
# Check if property_pb.value was indexed. Lists need to be
# special-cased and we require all `indexed` values in a list agree.
if isinstance(value, list):
indexed_values = set(value_pb.indexed
for value_pb in property_pb.value.list_value)
if len(indexed_values) != 1:
raise ValueError('For a list_value, subvalues must either all '
'be indexed or all excluded from indexes.')
if not indexed_values.pop():
exclude_from_indexes.append(property_pb.name)
else:
if not property_pb.value.indexed:
exclude_from_indexes.append(property_pb.name)
entity = Entity(key=key, exclude_from_indexes=exclude_from_indexes)
entity.update(entity_props)
return entity
def entity_from_protobuf(pb):
"""Factory method for creating an entity based on a protobuf.
The protobuf should be one returned from the Cloud Datastore
Protobuf API.
:type pb: :class:`gcloud.datastore._datastore_v1_pb2.Entity`
:param pb: The Protobuf representing the entity.
:rtype: :class:`gcloud.datastore.entity.Entity`
:returns: The entity derived from the protobuf.
"""
key = None
if pb.HasField('key'):
key = key_from_protobuf(pb.key)
entity_props = {}
exclude_from_indexes = []
for property_pb in pb.property:
value = _get_value_from_property_pb(property_pb)
entity_props[property_pb.name] = value
# Check if property_pb.value was indexed. Lists need to be
# special-cased and we require all `indexed` values in a list agree.
if isinstance(value, list):
indexed_values = set(value_pb.indexed
for value_pb in property_pb.value.list_value)
if len(indexed_values) != 1:
raise ValueError('For a list_value, subvalues must either all '
'be indexed or all excluded from indexes.')
if not indexed_values.pop():
exclude_from_indexes.append(property_pb.name)
else:
if not property_pb.value.indexed:
exclude_from_indexes.append(property_pb.name)
entity = Entity(key=key, exclude_from_indexes=exclude_from_indexes)
entity.update(entity_props)
return entity
def _get_employee(employee_id, create=True):
key = Key('Employee', employee_id)
employees = datastore.get([key])
if len(employees) == 0:
if not create:
return None
employee = Entity(key)
employee['created'] = employee['updated'] = datetime.datetime.utcnow()
datastore.put([employee])
else:
employee, = employees
return employee
def _fetch_page_helper(self,
cursor=b'\x00',
limit=None,
more_results=False,
_more_pb=None,
use_fetch=False):
import base64
from gcloud.datastore.datastore_v1_pb2 import Entity
_CURSOR_FOR_USER = (None
if cursor is None else base64.b64encode(cursor))
_MORE_RESULTS = more_results
_DATASET = 'DATASET'
_KIND = 'KIND'
_ID = 123
_NAMESPACE = 'NAMESPACE'
entity_pb = Entity()
path_element = entity_pb.key.path_element.add()
path_element.kind = _KIND
path_element.id = _ID
prop = entity_pb.property.add()
prop.name = 'foo'
prop.value.string_value = u'Foo'
if _more_pb is None:
connection = _Connection(entity_pb)
else:
connection = _Connection(entity_pb, _more=_more_pb)
connection._cursor = cursor
dataset = _Dataset(_DATASET, connection)
query = self._makeOne(_KIND, dataset, _NAMESPACE)
if use_fetch:
entities = query.fetch(limit)
else:
entities, cursor, more_results = query.fetch_page(limit)
self.assertEqual(cursor, _CURSOR_FOR_USER)
self.assertEqual(more_results, _MORE_RESULTS)
self.assertEqual(len(entities), 1)
self.assertEqual(entities[0].key().path(), [{
'kind': _KIND,
'id': _ID
}])
limited_query = query
if limit is not None:
limited_query = query.limit(limit)
expected_called_with = {
'dataset_id': _DATASET,
'query_pb': limited_query.to_protobuf(),
'namespace': _NAMESPACE,
}
self.assertEqual(connection._called_with, expected_called_with)
def _upsert_report(employee_id, report_id, rows):
_get_employee(employee_id) # force existence
report = _get_report(employee_id, report_id)
_purge_report_items(report)
# Add items based on rows.
report_path = list(report.key.flat_path)
for i, row in enumerate(rows):
path = report_path + ['Expense Item', i + 1]
key = Key(*path)
item = Entity(key)
for k, v in row.items():
item[k] = v
datastore.put([item])
return report
def test_entity_w_key(self):
from gcloud.datastore.entity import Entity
from gcloud.datastore.key import Key
pb = self._makePB()
key = Key('KIND', 123, dataset_id='DATASET')
entity = Entity(key=key)
entity['foo'] = u'Foo'
self._callFUT(pb, entity)
value = pb.entity_value
self.assertEqual(value.key, key.to_protobuf())
props = list(value.property)
self.assertEqual(len(props), 1)
self.assertEqual(props[0].name, 'foo')
self.assertEqual(props[0].value.string_value, u'Foo')
def entity(self, kind, exclude_from_indexes=()):
"""Create an entity bound to this dataset.
:type kind: string
:param kind: the "kind" of the new entity (see
https://cloud.google.com/datastore/docs/concepts/entities#Datastore_Kinds_and_identifiers)
:param exclude_from_indexes: names of fields whose values are not to
be indexed.
:rtype: :class:`gcloud.datastore.entity.Entity`
:returns: a new Entity instance, bound to this dataset.
"""
return Entity(dataset=self,
kind=kind,
exclude_from_indexes=exclude_from_indexes)
def test_key_only(self):
from gcloud.datastore._generated import entity_pb2
from gcloud.datastore.entity import Entity
from gcloud.datastore.key import Key
kind, name = 'PATH', 'NAME'
project = 'PROJECT'
key = Key(kind, name, project=project)
entity = Entity(key=key)
entity_pb = self._callFUT(entity)
expected_pb = entity_pb2.Entity()
expected_pb.key.partition_id.dataset_id = project
path_elt = expected_pb.key.path_element.add()
path_elt.kind = kind
path_elt.name = name
self._compareEntityProto(entity_pb, expected_pb)
def test_none(self):
from gcloud.datastore.entity import Entity
entity = Entity()
pb = self._makePB()
self._callFUT(pb, False)
self._callFUT(pb, 3.1415926)
self._callFUT(pb, 42)
self._callFUT(pb, (1 << 63) - 1)
self._callFUT(pb, 'str')
self._callFUT(pb, b'str')
self._callFUT(pb, u'str')
self._callFUT(pb, entity)
self._callFUT(pb, [u'a', 0, 3.14])
self._callFUT(pb, None)
self.assertEqual(len(pb.ListFields()), 0)
def test_simple_fields(self):
from gcloud.datastore._generated import entity_pb2
from gcloud.datastore.entity import Entity
from gcloud.datastore.helpers import _new_value_pb
entity = Entity()
name1 = 'foo'
entity[name1] = value1 = 42
name2 = 'bar'
entity[name2] = value2 = u'some-string'
entity_pb = self._callFUT(entity)
expected_pb = entity_pb2.Entity()
val_pb1 = _new_value_pb(expected_pb, name1)
val_pb1.integer_value = value1
val_pb2 = _new_value_pb(expected_pb, name2)
val_pb2.string_value = value2
self._compareEntityProto(entity_pb, expected_pb)
def test_entity_w_key(self):
from gcloud.datastore.entity import Entity
from gcloud.datastore.helpers import _property_tuples
from gcloud.datastore.key import Key
name = 'foo'
value = u'Foo'
pb = self._makePB()
key = Key('KIND', 123, project='PROJECT')
entity = Entity(key=key)
entity[name] = value
self._callFUT(pb, entity)
entity_pb = pb.entity_value
self.assertEqual(entity_pb.key, key.to_protobuf())
prop_dict = dict(_property_tuples(entity_pb))
self.assertEqual(len(prop_dict), 1)
self.assertEqual(list(prop_dict.keys()), [name])
self.assertEqual(prop_dict[name].string_value, value)
def fetch(self, limit=None):
"""Executes the Query and returns all matching entities.
This makes an API call to the Cloud Datastore, sends the Query as a
protobuf, parses the responses to Entity protobufs, and then converts
them to :class:`gcloud.datastore.entity.Entity` objects.
For example::
>>> from gcloud import datastore
>>> dataset = datastore.get_dataset('dataset-id', email, key_path)
>>> query = dataset.query('Person').filter('name =', 'Sally')
>>> query.fetch()
[<Entity object>, <Entity object>, ...]
>>> query.fetch(1)
[<Entity object>]
>>> query.limit()
None
:type limit: integer
:param limit: An optional limit to apply temporarily to this query.
That is, the Query itself won't be altered,
but the limit will be applied to the query
before it is executed.
:rtype: list of :class:`gcloud.datastore.entity.Entity`'s
:returns: The list of entities matching this query's criteria.
"""
clone = self
if limit:
clone = self.limit(limit)
(entity_pbs,
end_cursor,
more_results,
skipped_results) = self.dataset().connection().run_query(
query_pb=clone.to_protobuf(), dataset_id=self.dataset().id())
self._cursor = end_cursor
return [Entity.from_protobuf(entity, dataset=self.dataset())
for entity in entity_pbs]
def _get_value_from_protobuf(pb):
"""Given a protobuf for a Property, get the correct value.
The Cloud Datastore Protobuf API returns a Property Protobuf
which has one value set and the rest blank.
This function retrieves the the one value provided.
Some work is done to coerce the return value into a more useful type
(particularly in the case of a timestamp value, or a key value).
:type pb: :class:`gcloud.datastore.datastore_v1_pb2.Property`
:param pb: The Property Protobuf.
:returns: The value provided by the Protobuf.
"""
if pb.value.HasField('timestamp_microseconds_value'):
microseconds = pb.value.timestamp_microseconds_value
naive = (datetime.utcfromtimestamp(0) +
timedelta(microseconds=microseconds))
return naive.replace(tzinfo=pytz.utc)
elif pb.value.HasField('key_value'):
return Key.from_protobuf(pb.value.key_value)
elif pb.value.HasField('boolean_value'):
return pb.value.boolean_value
elif pb.value.HasField('double_value'):
return pb.value.double_value
elif pb.value.HasField('integer_value'):
return pb.value.integer_value
elif pb.value.HasField('string_value'):
return pb.value.string_value
elif pb.value.HasField('entity_value'):
return Entity.from_protobuf(pb.value.entity_value)
else:
return None
def entity_from_protobuf(pb, dataset=None):
"""Factory method for creating an entity based on a protobuf.
The protobuf should be one returned from the Cloud Datastore
Protobuf API.
:type pb: :class:`gcloud.datastore.datastore_v1_pb2.Entity`
:param pb: The Protobuf representing the entity.
:rtype: :class:`gcloud.datastore.entity.Entity`
:returns: The entity derived from the protobuf.
"""
key = key_from_protobuf(pb.key)
entity = Entity.from_key(key, dataset)
for property_pb in pb.property:
value = _get_value_from_property_pb(property_pb)
entity[property_pb.name] = value
return entity
def fetch(self, limit=None):
"""Executes the Query and returns all matching entities.
This makes an API call to the Cloud Datastore,
sends the Query as a protobuf,
parses the responses to Entity protobufs,
and then converts them to :class:`gcloud.datastore.entity.Entity` objects.
For example::
>>> from gcloud import datastore
>>> dataset = datastore.get_dataset('dataset-id', email, key_path)
>>> query = dataset.query('Person').filter('name =', 'Sally')
>>> query.fetch()
[<Entity object>, <Entity object>, ...]
>>> query.fetch(1)
[<Entity object>]
>>> query.limit()
None
:type limit: integer
:param limit: An optional limit to apply temporarily to this query.
That is, the Query itself won't be altered,
but the limit will be applied to the query
before it is executed.
:rtype: list of :class:`gcloud.datastore.entity.Entity`'s
:returns: The list of entities matching this query's criteria.
"""
clone = self
if limit:
clone = self.limit(limit)
entity_pbs = self.dataset().connection().run_query(
query_pb=clone.to_protobuf(), dataset_id=self.dataset().id())
return [
Entity.from_protobuf(entity, dataset=self.dataset())
for entity in entity_pbs
]
def test_filter_w_known_operator_and_entity(self):
import operator
from gcloud.datastore.entity import Entity
query = self._makeOne()
other = Entity()
other['firstname'] = u'John'
other['lastname'] = u'Smith'
after = query.filter('other', '=', other)
self.assertFalse(after is query)
self.assertTrue(isinstance(after, self._getTargetClass()))
q_pb = after.to_protobuf()
self.assertEqual(q_pb.filter.composite_filter.operator, 1) # AND
f_pb, = list(q_pb.filter.composite_filter.filter)
p_pb = f_pb.property_filter
self.assertEqual(p_pb.property.name, 'other')
other_pb = p_pb.value.entity_value
props = sorted(other_pb.property, key=operator.attrgetter('name'))
self.assertEqual(len(props), 2)
self.assertEqual(props[0].name, 'firstname')
self.assertEqual(props[0].value.string_value, u'John')
self.assertEqual(props[1].name, 'lastname')
self.assertEqual(props[1].value.string_value, u'Smith')
def test_save_entity_w_transaction_nested_entity(self):
from gcloud.datastore.connection import datastore_pb
from gcloud.datastore.entity import Entity
from gcloud.datastore.key import Key
mutation = datastore_pb.Mutation()
class Xact(object):
def mutation(self):
return mutation
DATASET_ID = 'DATASET'
nested = Entity()
nested['bar'] = u'Bar'
key_pb = Key(path=[{'kind': 'Kind', 'id': 1234}]).to_protobuf()
rsp_pb = datastore_pb.CommitResponse()
conn = self._makeOne()
conn.transaction(Xact())
http = conn._http = Http({'status': '200'}, rsp_pb.SerializeToString())
result = conn.save_entity(DATASET_ID, key_pb, {'foo': nested})
self.assertEqual(result, True)
self.assertEqual(http._called_with, None)
mutation = conn.mutation()
self.assertEqual(len(mutation.upsert), 1)
def test_key(self): dataset = Dataset(id='test-dataset') entity = Entity(dataset, 'TestKind') self.assertIsInstance(entity.key(), Key)
def test_w_single_empty_entity(self):
# https://github.com/GoogleCloudPlatform/gcloud-python/issues/649
from gcloud.datastore.entity import Entity
self.assertRaises(ValueError, self._callFUT, Entity())
def test_init_sets_proper_values(self):
dataset = Dataset(id='test-dataset')
entity = Entity(dataset, 'TestKind')
self.assertEqual('test-dataset', entity.dataset().id())
self.assertEqual('TestKind', entity.kind())
def test_key(self):
dataset = Dataset(id='test-dataset')
entity = Entity(dataset, 'TestKind')
self.assertIsInstance(entity.key(), Key)
def test_init_sets_proper_values(self):
dataset = Dataset(id='test-dataset')
entity = Entity(dataset, 'TestKind')
self.assertEqual('test-dataset', entity.dataset().id())
self.assertEqual('TestKind', entity.kind())
def test_entity(self):
from gcloud.datastore.entity import Entity
entity = Entity()
name, value = self._callFUT(entity)
self.assertEqual(name, 'entity_value')
self.assertTrue(value is entity)
def entity(self, kind): from gcloud.datastore.entity import Entity return Entity(dataset=self, kind=kind)
