def test_nested_entity_no_key(self): from google.cloud.proto.datastore.v1 import entity_pb2 from google.cloud.datastore.helpers import _new_value_pb PROJECT = 'FOO' KIND = 'KIND' INSIDE_NAME = 'IFOO' OUTSIDE_NAME = 'OBAR' INSIDE_VALUE = 1337 entity_inside = entity_pb2.Entity() inside_val_pb = _new_value_pb(entity_inside, INSIDE_NAME) inside_val_pb.integer_value = INSIDE_VALUE entity_pb = entity_pb2.Entity() entity_pb.key.partition_id.project_id = PROJECT element = entity_pb.key.path.add() element.kind = KIND outside_val_pb = _new_value_pb(entity_pb, OUTSIDE_NAME) outside_val_pb.entity_value.CopyFrom(entity_inside) entity = self._call_fut(entity_pb) self.assertEqual(entity.key.project, PROJECT) self.assertEqual(entity.key.flat_path, (KIND,)) self.assertEqual(len(entity), 1) inside_entity = entity[OUTSIDE_NAME] self.assertIsNone(inside_entity.key) self.assertEqual(len(inside_entity), 1) self.assertEqual(inside_entity[INSIDE_NAME], INSIDE_VALUE)
def test_get_multi_w_deferred_from_backend_but_not_passed(self): from google.cloud.proto.datastore.v1 import entity_pb2 from google.cloud.datastore.entity import Entity from google.cloud.datastore.key import Key key1 = Key('Kind', project=self.PROJECT) key1_pb = key1.to_protobuf() key2 = Key('Kind', 2345, project=self.PROJECT) key2_pb = key2.to_protobuf() entity1_pb = entity_pb2.Entity() entity1_pb.key.CopyFrom(key1_pb) entity2_pb = entity_pb2.Entity() entity2_pb.key.CopyFrom(key2_pb) creds = _make_credentials() client = self._make_one(credentials=creds) # mock up two separate requests client._connection._add_lookup_result([entity1_pb], deferred=[key2_pb]) client._connection._add_lookup_result([entity2_pb]) missing = [] found = client.get_multi([key1, key2], missing=missing) self.assertEqual(len(found), 2) self.assertEqual(len(missing), 0) # Check the actual contents on the response. self.assertIsInstance(found[0], Entity) self.assertEqual(found[0].key.path, key1.path) self.assertEqual(found[0].key.project, key1.project) self.assertIsInstance(found[1], Entity) self.assertEqual(found[1].key.path, key2.path) self.assertEqual(found[1].key.project, key2.project) cw = client._connection._lookup_cw self.assertEqual(len(cw), 2) ds_id, k_pbs, eventual, tid = cw[0] self.assertEqual(ds_id, self.PROJECT) self.assertEqual(len(k_pbs), 2) self.assertEqual(key1_pb, k_pbs[0]) self.assertEqual(key2_pb, k_pbs[1]) self.assertFalse(eventual) self.assertIsNone(tid) ds_id, k_pbs, eventual, tid = cw[1] self.assertEqual(ds_id, self.PROJECT) self.assertEqual(len(k_pbs), 1) self.assertEqual(key2_pb, k_pbs[0]) self.assertFalse(eventual) self.assertIsNone(tid)
def test_inverts_to_protobuf(self): from google.cloud.proto.datastore.v1 import entity_pb2 from google.cloud.datastore.helpers import _new_value_pb from google.cloud.datastore.helpers import entity_from_protobuf original_pb = entity_pb2.Entity() # Add a key. original_pb.key.partition_id.project_id = project = 'PROJECT' elem1 = original_pb.key.path.add() elem1.kind = 'Family' elem1.id = 1234 elem2 = original_pb.key.path.add() elem2.kind = 'King' elem2.name = 'Spades' # Add an integer property. val_pb1 = _new_value_pb(original_pb, 'foo') val_pb1.integer_value = 1337 val_pb1.exclude_from_indexes = True # Add a string property. val_pb2 = _new_value_pb(original_pb, 'bar') val_pb2.string_value = u'hello' # Add a nested (entity) property. val_pb3 = _new_value_pb(original_pb, 'entity-baz') sub_pb = entity_pb2.Entity() sub_val_pb1 = _new_value_pb(sub_pb, 'x') sub_val_pb1.double_value = 3.14 sub_val_pb2 = _new_value_pb(sub_pb, 'y') sub_val_pb2.double_value = 2.718281828 val_pb3.meaning = 9 val_pb3.entity_value.CopyFrom(sub_pb) # Add a list property. val_pb4 = _new_value_pb(original_pb, 'list-quux') array_val1 = val_pb4.array_value.values.add() array_val1.exclude_from_indexes = False array_val1.meaning = meaning = 22 array_val1.blob_value = b'\xe2\x98\x83' array_val2 = val_pb4.array_value.values.add() array_val2.exclude_from_indexes = False array_val2.meaning = meaning array_val2.blob_value = b'\xe2\x98\x85' # Convert to the user-space Entity. entity = entity_from_protobuf(original_pb) # Convert the user-space Entity back to a protobuf. new_pb = self._call_fut(entity) # NOTE: entity_to_protobuf() strips the project so we "cheat". new_pb.key.partition_id.project_id = project self._compareEntityProto(original_pb, new_pb)
def test_get_multi_miss_w_missing(self): from google.cloud.proto.datastore.v1 import entity_pb2 from google.cloud.proto.datastore.v1 import datastore_pb2 from google.cloud.datastore.key import Key KIND = 'Kind' ID = 1234 # Make a missing entity pb to be returned from mock backend. missed = entity_pb2.Entity() missed.key.partition_id.project_id = self.PROJECT path_element = missed.key.path.add() path_element.kind = KIND path_element.id = ID creds = _make_credentials() client = self._make_one(credentials=creds) # Set missing entity on mock connection. lookup_response = _make_lookup_response(missing=[missed]) ds_api = _make_datastore_api(lookup_response=lookup_response) client._datastore_api_internal = ds_api key = Key(KIND, ID, project=self.PROJECT) missing = [] entities = client.get_multi([key], missing=missing) self.assertEqual(entities, []) key_pb = key.to_protobuf() self.assertEqual([missed.key.to_protobuf() for missed in missing], [key_pb]) read_options = datastore_pb2.ReadOptions() ds_api.lookup.assert_called_once_with(self.PROJECT, read_options, [key_pb])
def test_empty(self): from google.cloud.proto.datastore.v1 import entity_pb2 from google.cloud.datastore.entity import Entity entity = Entity() entity_pb = self._call_fut(entity) self._compareEntityProto(entity_pb, entity_pb2.Entity())
def test_variable_meanings(self): from google.cloud.proto.datastore.v1 import entity_pb2 from google.cloud.datastore.entity import Entity from google.cloud.datastore.helpers import _new_value_pb entity = Entity() name = 'quux' entity[name] = values = [1, 20, 300] meaning = 9 entity._meanings[name] = ([None, meaning, None], values) entity_pb = self._call_fut(entity) # Construct the expected protobuf. expected_pb = entity_pb2.Entity() value_pb = _new_value_pb(expected_pb, name) value0 = value_pb.array_value.values.add() value0.integer_value = values[0] # The only array entry with a meaning is the middle one. value1 = value_pb.array_value.values.add() value1.integer_value = values[1] value1.meaning = meaning value2 = value_pb.array_value.values.add() value2.integer_value = values[2] self._compareEntityProto(entity_pb, expected_pb)
def to_entity(line): entity = entity_pb2.Entity() fields = line.split( ',') #id,president,startYear,endYear,party,homeState,dateOfBirth id = fields[0] president = fields[1] names = president.split(' ') firstName = names[0] lastName = names[1] startYear = fields[2] endYear = fields[3] party = fields[4] homeState = fields[5] dateOfBirth = fields[6] googledatastore.helper.add_key_path(entity.key, kind, str(id)) googledatastore.helper.add_properties( entity, { 'firstName': unicode(firstName), 'lastName': unicode(lastName), 'startYear': int(startYear), 'endYear': int(endYear), 'party': unicode(party), 'homeState': unicode(homeState), 'dateOfBirth': datetime.strptime(dateOfBirth, '%Y-%m-%d') }) return entity
def create_ds_entity(element): entity = entity_pb2.Entity() kind = 'FileMetaData' datastore_helper.add_key_path(entity.key, kind, str(uuid.uuid4())) datastore_helper.add_properties(entity, element) return entity
def make_entity(self, content): """Create entity and set properties.""" entity = entity_pb2.Entity() # We use uuid for keys. datastore_helper.add_key_path(entity.key, self._kind, str(uuid.uuid4())) datastore_helper.add_properties(entity, content) return entity
def test_run_query_w_namespace_nonempty_result(self): from google.cloud.proto.datastore.v1 import datastore_pb2 from google.cloud.proto.datastore.v1 import entity_pb2 from google.cloud.proto.datastore.v1 import query_pb2 project = 'PROJECT' kind = 'Kind' entity = entity_pb2.Entity() q_pb = self._make_query_pb(kind) rsp_pb = datastore_pb2.RunQueryResponse() rsp_pb.batch.entity_results.add(entity=entity) rsp_pb.batch.entity_result_type = query_pb2.EntityResult.FULL rsp_pb.batch.more_results = query_pb2.QueryResultBatch.NO_MORE_RESULTS # Create mock HTTP and client with response. http = Http({'status': '200'}, rsp_pb.SerializeToString()) client = mock.Mock(_http=http, _base_url='test.invalid', spec=['_http', '_base_url']) # Make request. conn = self._make_one(client) namespace = 'NS' response = conn.run_query(project, q_pb, namespace=namespace) # Check the result and verify the callers. self.assertEqual(response, rsp_pb) cw = http._called_with uri = _build_expected_url(conn.api_base_url, project, 'runQuery') _verify_protobuf_call(self, cw, uri) request = datastore_pb2.RunQueryRequest() request.ParseFromString(cw['body']) self.assertEqual(request.partition_id.namespace_id, namespace) self.assertEqual(request.query, q_pb)
def test_lookup_single_key_nonempty_response(self): from google.cloud.proto.datastore.v1 import datastore_pb2 from google.cloud.proto.datastore.v1 import entity_pb2 PROJECT = 'PROJECT' key_pb = self._make_key_pb(PROJECT) rsp_pb = datastore_pb2.LookupResponse() entity = entity_pb2.Entity() entity.key.CopyFrom(key_pb) rsp_pb.found.add(entity=entity) http = Http({'status': '200'}, rsp_pb.SerializeToString()) client = mock.Mock(_http=http, spec=['_http']) conn = self._make_one(client) URI = '/'.join([ conn.api_base_url, conn.API_VERSION, 'projects', PROJECT + ':lookup', ]) (found, ), missing, deferred = conn.lookup(PROJECT, [key_pb]) self.assertEqual(len(missing), 0) self.assertEqual(len(deferred), 0) self.assertEqual(found.key.path[0].kind, 'Kind') self.assertEqual(found.key.path[0].id, 1234) cw = http._called_with self._verify_protobuf_call(cw, URI, conn) rq_class = datastore_pb2.LookupRequest request = rq_class() request.ParseFromString(cw['body']) keys = list(request.keys) self.assertEqual(len(keys), 1) self.assertEqual(key_pb, keys[0])
def test_get_multi_miss_w_missing(self): from google.cloud.proto.datastore.v1 import entity_pb2 from google.cloud.datastore.key import Key KIND = 'Kind' ID = 1234 # Make a missing entity pb to be returned from mock backend. missed = entity_pb2.Entity() missed.key.partition_id.project_id = self.PROJECT path_element = missed.key.path.add() path_element.kind = KIND path_element.id = ID creds = _make_credentials() client = self._make_one(credentials=creds) # Set missing entity on mock connection. client._connection._add_lookup_result(missing=[missed]) key = Key(KIND, ID, project=self.PROJECT) missing = [] entities = client.get_multi([key], missing=missing) self.assertEqual(entities, []) self.assertEqual([missed.key.to_protobuf() for missed in missing], [key.to_protobuf()])
def build_user_entity(row): entity = entity_pb2.Entity() datastore_helper.add_key_path(entity.key, 'User', row['email']) props = dict(row) del props['email'] datastore_helper.add_properties(entity, props) return entity
def to_entity(content): entity = entity_pb2.Entity() googledatastore.helper.add_key_path(entity.key, kind, str(uuid.uuid4())) googledatastore.helper.add_properties(entity, {'content': unicode(content)}) return entity
def test_dict_to_entity_recursive(self): from google.cloud.proto.datastore.v1 import entity_pb2 from google.cloud.datastore.entity import Entity entity = Entity() entity['a'] = { 'b': { 'c': { 'd': 1.25, }, 'e': True, }, 'f': 10, } entity_pb = self._call_fut(entity) b_entity_pb = entity_pb2.Entity( properties={ 'c': entity_pb2.Value( entity_value=entity_pb2.Entity( properties={ 'd': entity_pb2.Value( double_value=1.25, ), }, ), ), 'e': entity_pb2.Value(boolean_value=True), } ) expected_pb = entity_pb2.Entity( properties={ 'a': entity_pb2.Value( entity_value=entity_pb2.Entity( properties={ 'b': entity_pb2.Value( entity_value=b_entity_pb, ), 'f': entity_pb2.Value( integer_value=10, ), }, ), ), }, ) self.assertEqual(entity_pb, expected_pb)
def test_get_multi_w_deferred_from_backend_but_not_passed(self): from google.cloud.proto.datastore.v1 import datastore_pb2 from google.cloud.proto.datastore.v1 import entity_pb2 from google.cloud.datastore.entity import Entity from google.cloud.datastore.key import Key key1 = Key('Kind', project=self.PROJECT) key1_pb = key1.to_protobuf() key2 = Key('Kind', 2345, project=self.PROJECT) key2_pb = key2.to_protobuf() entity1_pb = entity_pb2.Entity() entity1_pb.key.CopyFrom(key1_pb) entity2_pb = entity_pb2.Entity() entity2_pb.key.CopyFrom(key2_pb) creds = _make_credentials() client = self._make_one(credentials=creds) # Mock up two separate requests. Using an iterable as side_effect # allows multiple return values. lookup_response1 = _make_lookup_response(results=[entity1_pb], deferred=[key2_pb]) lookup_response2 = _make_lookup_response(results=[entity2_pb]) ds_api = _make_datastore_api() ds_api.lookup = mock.Mock( side_effect=[lookup_response1, lookup_response2], spec=[]) client._datastore_api_internal = ds_api missing = [] found = client.get_multi([key1, key2], missing=missing) self.assertEqual(len(found), 2) self.assertEqual(len(missing), 0) # Check the actual contents on the response. self.assertIsInstance(found[0], Entity) self.assertEqual(found[0].key.path, key1.path) self.assertEqual(found[0].key.project, key1.project) self.assertIsInstance(found[1], Entity) self.assertEqual(found[1].key.path, key2.path) self.assertEqual(found[1].key.project, key2.project) self.assertEqual(ds_api.lookup.call_count, 2) read_options = datastore_pb2.ReadOptions() ds_api.lookup.assert_any_call(self.PROJECT, read_options, [key2_pb]) ds_api.lookup.assert_any_call(self.PROJECT, read_options, [key1_pb, key2_pb])
def make_entity(self, content): entity = entity_pb2.Entity() if self._namespace is not None: entity.key.partition_id.namespace_id = self._namespace helper.add_key_path(entity.key, self._kind, str(uuid.uuid4())) helper.add_properties(entity, content) return entity
def test_entity_no_key(self): from google.cloud.proto.datastore.v1 import entity_pb2 entity_pb = entity_pb2.Entity() entity = self._call_fut(entity_pb) self.assertIsNone(entity.key) self.assertEqual(dict(entity), {})
def _make_entity(kind, id_, project): from google.cloud.proto.datastore.v1 import entity_pb2 key = entity_pb2.Key() key.partition_id.project_id = project elem = key.path.add() elem.kind = kind elem.id = id_ return entity_pb2.Entity(key=key)
def test_with_empty_list(self): from google.cloud.proto.datastore.v1 import entity_pb2 from google.cloud.datastore.entity import Entity entity = Entity() entity['foo'] = [] entity_pb = self._call_fut(entity) self._compare_entity_proto(entity_pb, entity_pb2.Entity())
def build_company_entity(data): entity = entity_pb2.Entity() datastore_helper.add_key_path(entity.key, 'Company', data[0]) props = {} props['reg_date'] = data[1] props['boxes_bought'] = 0 props['boxes_prov'] = 0 datastore_helper.add_properties(entity, props) return entity
def make_entity(self, prefix, candidates): entity = entity_pb2.Entity() datastore_helper.add_key_path(entity.key, "prefix", prefix, "id", str(uuid.uuid4()), "candidates", unicode(candidates)) # All entities created will have the same ancestor #datastore_helper.add_properties(entity, {"wordlist": unicode(candidates)}) #datastore_helper.add_properties(entity, {"candidates": unicode(candidates)}) return entity
def test_it(self): from google.cloud.proto.datastore.v1 import entity_pb2 entity_pb = entity_pb2.Entity() name = 'foo' result = self._call_fut(entity_pb, name) self.assertIsInstance(result, entity_pb2.Value) self.assertEqual(len(entity_pb.properties), 1) self.assertEqual(entity_pb.properties[name], result)
def make_entity(self, content): entity = entity_pb2.Entity() if self._namespace is not None: entity.key.partition_id.namespace_id = self._namespace # All entities created will have the same ancestor datastore_helper.add_key_path(entity.key, self._kind, self._ancestor, self._kind, str(uuid.uuid4())) datastore_helper.add_properties(entity, {"content": unicode(content)}) return entity
def make_entity(self, content): """Create entity from given string.""" entity = entity_pb2.Entity() if self._namespace is not None: entity.key.partition_id.namespace_id = self._namespace # All entities created will have the same ancestor datastore_helper.add_key_path(entity.key, self._kind, self._ancestor, self._kind, hashlib.sha1(content).hexdigest()) datastore_helper.add_properties(entity, {'content': str(content)}) return entity
def test_dict_to_entity(self): from google.cloud.proto.datastore.v1 import entity_pb2 from google.cloud.datastore.entity import Entity entity = Entity() entity['a'] = {'b': u'c'} entity_pb = self._call_fut(entity) expected_pb = entity_pb2.Entity( properties={ 'a': entity_pb2.Value( entity_value=entity_pb2.Entity( properties={ 'b': entity_pb2.Value( string_value='c', ), }, ), ), }, ) self.assertEqual(entity_pb, expected_pb)
def process(self, idx): entity = entity_pb2.Entity() embed = entity_pb2.Entity() # embedded document ss = { 'ss': unicode('X'), # string 'bb': False, # boolean 'll': [1, 2, 3], # list of integers 'ii': 123 # integer } datastore_helper.add_properties( embed, ss) # setting properties for embedded document element = dict() element['s'] = unicode('String') # string element['b'] = True # boolean element['l'] = [unicode('s'), unicode('a'), unicode('b')] # list of strings element['i'] = 1 # integer element['e'] = embed # embedded document datastore_helper.add_key_path(entity.key, 'ds', idx) # setting id for document datastore_helper.add_properties( entity, element) # setting properties for document return [entity]
def test_entity_with_meaning(self): from google.cloud.proto.datastore.v1 import entity_pb2 from google.cloud.datastore.helpers import _new_value_pb entity_pb = entity_pb2.Entity() name = 'hello' value_pb = _new_value_pb(entity_pb, name) value_pb.meaning = meaning = 9 value_pb.string_value = val = u'something' entity = self._call_fut(entity_pb) self.assertIsNone(entity.key) self.assertEqual(dict(entity), {name: val}) self.assertEqual(entity._meanings, {name: (meaning, val)})
def _make_entity_pb(project, kind, integer_id, name=None, str_val=None): from google.cloud.proto.datastore.v1 import entity_pb2 from google.cloud.datastore.helpers import _new_value_pb entity_pb = entity_pb2.Entity() entity_pb.key.partition_id.project_id = project path_element = entity_pb.key.path.add() path_element.kind = kind path_element.id = integer_id if name is not None and str_val is not None: value_pb = _new_value_pb(entity_pb, name) value_pb.string_value = str_val return entity_pb
def test_it(self): import types from google.cloud.proto.datastore.v1 import entity_pb2 from google.cloud.datastore.helpers import _new_value_pb entity_pb = entity_pb2.Entity() name1 = 'foo' name2 = 'bar' val_pb1 = _new_value_pb(entity_pb, name1) val_pb2 = _new_value_pb(entity_pb, name2) result = self._call_fut(entity_pb) self.assertIsInstance(result, types.GeneratorType) self.assertEqual(sorted(result), sorted([(name1, val_pb1), (name2, val_pb2)]))