def group_for_key(key):
""" Extract the root path for a given key.
Args:
key: An encoded or decoded Reference object.
Returns:
A Reference object containing the root key.
"""
if not isinstance(key, entity_pb.Reference):
key = entity_pb.Reference(key)
first_element = key.path().element(0)
# Avoid modifying the original object.
key_copy = entity_pb.Reference()
key_copy.CopyFrom(key)
# Groups without a namespace should match groups with an empty namespace.
if not key_copy.name_space():
key_copy.set_name_space('')
key_copy.path().clear_element()
element = key_copy.path().add_element()
element.MergeFrom(first_element)
return key_copy
def __init__(self, encoded=None):
"""Constructor. Creates a Key from a string.
Args:
# a base64-encoded primary key, generated by Key.__str__
encoded: str
"""
if encoded is not None:
if not isinstance(encoded, basestring):
try:
repr_encoded = repr(encoded)
except:
repr_encoded = "<couldn't encode>"
raise datastore_errors.BadArgumentError(
'Key() expects a string; received %s (a %s).' %
(repr_encoded, typename(encoded)))
try:
modulo = len(encoded) % 4
if modulo != 0:
encoded += ('=' * (4 - modulo))
encoded_pb = base64.urlsafe_b64decode(str(encoded))
self.__reference = entity_pb.Reference(encoded_pb)
assert self.__reference.IsInitialized()
except (AssertionError, TypeError), e:
raise datastore_errors.BadKeyError(
'Invalid string key %s. Details: %s' % (encoded, e))
except Exception, e:
if e.__class__.__name__ == 'ProtocolBufferDecodeError':
raise datastore_errors.BadKeyError('Invalid string key %s.' % encoded)
else:
raise
def testUnicode(self):
flat_input = [u'Kind\u1234', 1, 'Subkind', u'foobar\u4321']
flat = [
flat_input[0].encode('utf8'), flat_input[1], flat_input[2],
flat_input[3].encode('utf8')
]
pairs = [(flat[i], flat[i + 1]) for i in xrange(0, len(flat), 2)]
k = key.Key(flat=flat_input)
self.assertEqual(k.pairs(), pairs)
self.assertEqual(k.flat(), flat)
# TODO: test these more thoroughly
r = k.reference()
serialized = k.serialized()
urlsafe = k.urlsafe()
key.Key(urlsafe=urlsafe.decode('utf8'))
key.Key(serialized=serialized.decode('utf8'))
key.Key(reference=r)
# TODO: this may not make sense -- the protobuf utf8-encodes values
r = entity_pb.Reference()
r.set_app('_')
e = r.mutable_path().add_element()
e.set_type(flat_input[0])
e.set_name(flat_input[3])
k = key.Key(reference=r)
self.assertEqual(k.reference(), r)
def record_reads(self, app, txid, group_keys):
""" Keep track of which entity groups were read in a transaction.
Args:
app: A string specifying an application ID.
txid: An integer specifying a transaction ID.
group_keys: An iterable containing Reference objects.
"""
batch = BatchStatement(consistency_level=ConsistencyLevel.QUORUM,
retry_policy=BASIC_RETRIES)
insert = self.session.prepare("""
INSERT INTO transactions (txid_hash, operation, namespace, path)
VALUES (?, ?, ?, ?)
USING TTL {ttl}
""".format(ttl=dbconstants.MAX_TX_DURATION * 2))
for group_key in group_keys:
if not isinstance(group_key, entity_pb.Reference):
group_key = entity_pb.Reference(group_key)
args = (tx_partition(app,
txid), TxnActions.GET, group_key.name_space(),
bytearray(group_key.path().Encode()))
batch.add(insert, args)
try:
yield self.tornado_cassandra.execute(batch)
except dbconstants.TRANSIENT_CASSANDRA_ERRORS:
message = 'Exception while recording reads in a transaction'
logger.exception(message)
raise AppScaleDBConnectionError(message)
def key(self):
key = entity_pb.Reference()
key.set_app(self.project_id)
if self.namespace is not None:
key.set_name_space(self.namespace)
key.mutable_path().MergeFrom(Path.decode(self.path))
return key
def _ReferenceFromSerialized(serialized):
"""Construct a Reference from a serialized Reference."""
if not isinstance(serialized, basestring):
raise TypeError('serialized must be a string; received %r' %
serialized)
elif isinstance(serialized, unicode):
serialized = serialized.encode('utf8')
return entity_pb.Reference(serialized)
def ReferencePropertyToReference(refprop):
ref = entity_pb.Reference()
ref.set_app(refprop.app())
if refprop.has_name_space():
ref.set_name_space(refprop.name_space())
for pathelem in refprop.pathelement_list():
ref.mutable_path().add_element().CopyFrom(pathelem)
return ref
def test_dynamic_delete(self):
async_true = gen.Future()
async_true.set_result(True)
entity_lock = flexmock(EntityLock)
entity_lock.should_receive('acquire').and_return(async_true)
entity_lock.should_receive('release')
del_request = flexmock()
del_request.should_receive("key_list")
del_request.should_receive("has_transaction").never()
del_request.should_receive("transaction").never()
db_batch = flexmock()
db_batch.should_receive('valid_data_version_sync').and_return(True)
transaction_manager = flexmock(
create_transaction_id=lambda project, xg: 1,
delete_transaction_id=lambda project, txid: None,
set_groups=lambda project_id, txid, groups: None)
dd = DatastoreDistributed(db_batch, transaction_manager,
self.get_zookeeper())
dd.index_manager = flexmock(
projects={'guestbook': flexmock(indexes_pb=[])})
yield dd.dynamic_delete("appid", del_request)
fake_key = entity_pb.Reference()
fake_key.set_app('foo')
path = fake_key.mutable_path()
element = path.add_element()
element.set_type('bar')
element.set_id(1)
del_request = flexmock()
del_request.should_receive("key_list").and_return([fake_key])
del_request.should_receive("has_transaction").and_return(True)
transaction = flexmock()
transaction.should_receive("handle").and_return(1)
del_request.should_receive("transaction").and_return(transaction)
del_request.should_receive("has_mark_changes").and_return(False)
dd = DatastoreDistributed(db_batch, transaction_manager,
self.get_zookeeper())
dd.index_manager = flexmock(
projects={'appid': flexmock(indexes_pb=[])})
flexmock(utils).should_receive("get_entity_kind").and_return("kind")
db_batch.should_receive('delete_entities_tx').and_return(ASYNC_NONE)
yield dd.dynamic_delete("appid", del_request)
del_request = flexmock()
del_request.should_receive("key_list").and_return([fake_key])
del_request.should_receive("has_transaction").and_return(False)
del_request.should_receive("has_mark_changes").and_return(False)
dd = DatastoreDistributed(db_batch, transaction_manager,
self.get_zookeeper())
dd.index_manager = flexmock(
projects={'appid': flexmock(indexes_pb=[])})
flexmock(dd).should_receive("delete_entities").and_return(ASYNC_NONE).once()
yield dd.dynamic_delete("appid", del_request)
def _ReferenceFromPairs(pairs, reference=None, app=None, namespace=None):
"""Construct a Reference from a list of pairs.
If a Reference is passed in as the second argument, it is modified
in place. The app and namespace are set from the corresponding
keyword arguments, with the customary defaults.
"""
if reference is None:
reference = entity_pb.Reference()
path = reference.mutable_path()
last = False
for kind, idorname in pairs:
if last:
raise datastore_errors.BadArgumentError(
'Incomplete Key entry must be last')
if not isinstance(kind, basestring):
if isinstance(kind, type):
# Late import to avoid cycles.
from ndb.model import Model
modelclass = kind
assert issubclass(modelclass, Model), repr(modelclass)
kind = modelclass._get_kind()
assert isinstance(kind, basestring), (repr(modelclass), repr(kind))
if isinstance(kind, unicode):
kind = kind.encode('utf8')
assert 1 <= len(kind) <= 500
elem = path.add_element()
elem.set_type(kind)
if isinstance(idorname, (int, long)):
assert 1 <= idorname < 2**63
elem.set_id(idorname)
elif isinstance(idorname, basestring):
if isinstance(idorname, unicode):
idorname = idorname.encode('utf8')
assert 1 <= len(idorname) <= 500
elem.set_name(idorname)
elif idorname is None:
elem.set_id(0)
last = True
else:
assert False, 'bad idorname (%r)' % (idorname, )
# An empty app id means to use the default app id.
if not app:
app = _DefaultAppId()
# Always set the app id, since it is mandatory.
reference.set_app(app)
# An empty namespace overrides the default namespace.
if namespace is None:
namespace = _DefaultNamespace()
# Only set the namespace if it is not empty.
if namespace:
reference.set_name_space(namespace)
return reference
def __delete_v1_key(self, v1_key, v1_txn):
"""Deletes an entity from a v1 key in a transaction."""
v3_key = entity_pb.Reference()
self.__entity_converter.v1_to_v3_reference(v1_key, v3_key)
v3_txn = datastore_pb.Transaction()
self.__service_converter.v1_to_v3_txn(v1_txn, v3_txn)
v3_delete_req = datastore_pb.DeleteRequest()
v3_delete_req.mutable_transaction().CopyFrom(v3_txn)
v3_delete_req.add_key().CopyFrom(v3_key)
v3_delete_resp = datastore_pb.DeleteResponse()
self.__make_v3_call('Delete', v3_delete_req, v3_delete_resp)
def DbGetValue(self, v, p):
if not v.has_referencevalue():
return None
ref = entity_pb.Reference()
rv = v.referencevalue()
if rv.has_app():
ref.set_app(rv.app())
if rv.has_name_space():
ref.set_name_space(rv.name_space())
path = ref.mutable_path()
for elem in rv.pathelement_list():
path.add_element().CopyFrom(elem)
return Key(reference=ref)
def encode_entity_table_key(key):
""" Create a key that can be used for the entities table.
Args:
key: An encoded or decoded Reference object.
Returns:
A string containing an entities table key.
"""
if not isinstance(key, entity_pb.Reference):
key = entity_pb.Reference(key)
prefix = dbconstants.KEY_DELIMITER.join([key.app(), key.name_space()])
return get_entity_key(prefix, key.path())
def _Dynamic_TransactionQuery(self, request, response):
if not request.has_ancestor():
raise apiproxy_errors.ApplicationError(
datastore_pb.Error.BAD_REQUEST,
'No ancestor in transactional query.')
app_id = datastore_types.ResolveAppId(None)
if (datastore_rpc._GetDatastoreType(app_id) !=
datastore_rpc.BaseConnection.HIGH_REPLICATION_DATASTORE):
raise apiproxy_errors.ApplicationError(
datastore_pb.Error.BAD_REQUEST,
'remote_api supports transactional queries only in the '
'high-replication datastore.')
entity_group_key = entity_pb.Reference()
entity_group_key.CopyFrom(request.ancestor())
group_path = entity_group_key.mutable_path()
root = entity_pb.Path_Element()
root.MergeFrom(group_path.element(0))
group_path.clear_element()
group_path.add_element().CopyFrom(root)
eg_element = group_path.add_element()
eg_element.set_type(metadata.EntityGroup.KIND_NAME)
eg_element.set_id(metadata.EntityGroup.ID)
begin_request = datastore_pb.BeginTransactionRequest()
begin_request.set_app(app_id)
tx = datastore_pb.Transaction()
self.__call('datastore_v3', 'BeginTransaction', begin_request, tx)
request.mutable_transaction().CopyFrom(tx)
self.__call('datastore_v3', 'RunQuery', request, response.mutable_result())
get_request = datastore_pb.GetRequest()
get_request.mutable_transaction().CopyFrom(tx)
get_request.add_key().CopyFrom(entity_group_key)
get_response = datastore_pb.GetResponse()
self.__call('datastore_v3', 'Get', get_request, get_response)
entity_group = get_response.entity(0)
response.mutable_entity_group_key().CopyFrom(entity_group_key)
if entity_group.has_entity():
response.mutable_entity_group().CopyFrom(entity_group.entity())
self.__call('datastore_v3', 'Commit', tx, datastore_pb.CommitResponse())
def _unpack_keys(self, blob):
keys = []
pos = 0
while pos < len(blob):
namespace, pos = Text.decode(blob, pos)
path, pos = Path.unpack(blob, pos)
key = entity_pb.Reference()
key.set_app(self.project_id)
key.set_name_space(namespace)
key.mutable_path().MergeFrom(Path.decode(path))
keys.append(key)
return keys
def v4_to_v3_references(self, v4_keys):
"""Converts a list of v4 Keys to a list of v3 References.
Args:
v4_keys: a list of entity_v4_pb.Key objects
Returns:
a list of entity_pb.Reference objects
"""
v3_refs = []
for v4_key in v4_keys:
v3_ref = entity_pb.Reference()
self.v4_to_v3_reference(v4_key, v3_ref)
v3_refs.append(v3_ref)
return v3_refs
def group_for_key(key):
""" Extract the root path for a given key.
Args:
key: An encoded or decoded Reference object.
Returns:
A Reference object containing the root key.
"""
if not isinstance(key, entity_pb.Reference):
key = entity_pb.Reference(key)
first_element = key.path().element(0)
key.path().clear_element()
element = key.path().add_element()
element.MergeFrom(first_element)
return key
def v4_value_to_v3_property_value(self, v4_value, v3_value):
"""Converts a v4 Value to a v3 PropertyValue.
Args:
v4_value: an entity_v4_pb.Value
v3_value: an entity_pb.PropertyValue to populate
"""
v3_value.Clear()
if v4_value.has_boolean_value():
v3_value.set_booleanvalue(v4_value.boolean_value())
elif v4_value.has_integer_value():
v3_value.set_int64value(v4_value.integer_value())
elif v4_value.has_double_value():
v3_value.set_doublevalue(v4_value.double_value())
elif v4_value.has_timestamp_microseconds_value():
v3_value.set_int64value(v4_value.timestamp_microseconds_value())
elif v4_value.has_key_value():
v3_ref = entity_pb.Reference()
self.v4_to_v3_reference(v4_value.key_value(), v3_ref)
self.v3_reference_to_v3_property_value(v3_ref, v3_value)
elif v4_value.has_blob_key_value():
v3_value.set_stringvalue(v4_value.blob_key_value())
elif v4_value.has_string_value():
v3_value.set_stringvalue(v4_value.string_value())
elif v4_value.has_blob_value():
v3_value.set_stringvalue(v4_value.blob_value())
elif v4_value.has_entity_value():
v4_entity_value = v4_value.entity_value()
v4_meaning = v4_value.meaning()
if (v4_meaning == MEANING_GEORSS_POINT
or v4_meaning == MEANING_PREDEFINED_ENTITY_POINT):
self.__v4_to_v3_point_value(v4_entity_value,
v3_value.mutable_pointvalue())
elif v4_meaning == MEANING_PREDEFINED_ENTITY_USER:
self.v4_entity_to_v3_user_value(v4_entity_value,
v3_value.mutable_uservalue())
else:
v3_entity_value = entity_pb.EntityProto()
self.v4_to_v3_entity(v4_entity_value, v3_entity_value)
v3_value.set_stringvalue(
v3_entity_value.SerializePartialToString())
elif v4_value.has_geo_point_value():
point_value = v3_value.mutable_pointvalue()
point_value.set_x(v4_value.geo_point_value().latitude())
point_value.set_y(v4_value.geo_point_value().longitude())
else:
pass
def reference_property_to_reference(refprop):
""" Creates a Reference from a ReferenceProperty.
Args:
refprop: A entity_pb.ReferenceProperty object.
Returns:
A entity_pb.Reference object.
"""
ref = entity_pb.Reference()
app_id = clean_app_id(refprop.app())
ref.set_app(app_id)
if refprop.has_name_space():
ref.set_name_space(refprop.name_space())
for pathelem in refprop.pathelement_list():
ref.mutable_path().add_element().CopyFrom(pathelem)
return ref
def create_key(app, namespace, path):
""" Create Reference object from app, namespace, and path.
Args:
app: A string specifying an application ID.
namespace: A string specifying the namespace.
path: An encoded Path object.
Returns:
A Reference object.
"""
key = entity_pb.Reference()
key.set_app(app)
key.set_name_space(namespace)
key_path = key.mutable_path()
key_path.MergeFromString(path)
return key
def __apply_v1_mutation(self, v1_mutation, v1_txn):
"""Applies a v1 Mutation.
Args:
v1_mutation: a googledatastore.Mutation
v1_txn: an optional v1 transaction handle or None
Returns:
a tuple (googledatastore.MutationResult, number of index writes)
"""
v3_txn = None
v3_key = None
if v1_txn:
v3_txn = datastore_pb.Transaction()
self.__service_converter.v1_to_v3_txn(v1_txn, v3_txn)
if v1_mutation.op == googledatastore.Mutation.INSERT:
v3_entity = entity_pb.EntityProto()
self.__entity_converter.v1_to_v3_entity(v1_mutation.entity,
v3_entity)
index_writes, v3_key = self.__insert_v3_entity(v3_entity, v3_txn)
elif v1_mutation.op == googledatastore.Mutation.UPDATE:
v3_entity = entity_pb.EntityProto()
self.__entity_converter.v1_to_v3_entity(v1_mutation.entity,
v3_entity)
index_writes = self.__update_v3_entity(v3_entity, v3_txn)
elif v1_mutation.op == googledatastore.Mutation.UPSERT:
v3_entity = entity_pb.EntityProto()
self.__entity_converter.v1_to_v3_entity(v1_mutation.entity,
v3_entity)
index_writes, v3_key = self.__upsert_v3_entity(v3_entity, v3_txn)
elif v1_mutation.op == googledatastore.Mutation.DELETE:
v3_ref = entity_pb.Reference()
self.__entity_converter.v1_to_v3_reference(v1_mutation.key, v3_ref)
index_writes = self.__delete_v3_reference(v3_ref, v3_txn)
v1_mutation_result = googledatastore.MutationResult()
if v3_key and not datastore_pbs.is_complete_v1_key(
v1_mutation.entity.key):
self.__entity_converter.v3_to_v1_key(v3_key,
v1_mutation_result.key)
return v1_mutation_result, index_writes
def _FromPb(pb):
"""Static factory method. Creates a Key from an entity_pb.Reference.
Not intended to be used by application developers. Enforced by hiding the
entity_pb classes.
Args:
pb: entity_pb.Reference
"""
if not isinstance(pb, entity_pb.Reference):
raise datastore_errors.BadArgumentError(
'Key constructor takes an entity_pb.Reference; received %s (a %s).' %
(pb, typename(pb)))
key = Key()
key.__reference = entity_pb.Reference()
key.__reference.CopyFrom(pb)
return key
def testSerialized(self):
flat = ['Kind', 1, 'Subkind', 'foobar']
r = entity_pb.Reference()
r.set_app('_')
e = r.mutable_path().add_element()
e.set_type(flat[0])
e.set_id(flat[1])
e = r.mutable_path().add_element()
e.set_type(flat[2])
e.set_name(flat[3])
serialized = r.Encode()
urlsafe = base64.urlsafe_b64encode(r.Encode()).rstrip('=')
k = key.Key(flat=flat)
self.assertEqual(k.serialized(), serialized)
self.assertEqual(k.urlsafe(), urlsafe)
self.assertEqual(k.reference(), r)
k = key.Key(urlsafe=urlsafe)
self.assertEqual(k.serialized(), serialized)
self.assertEqual(k.urlsafe(), urlsafe)
self.assertEqual(k.reference(), r)
k = key.Key(serialized=serialized)
self.assertEqual(k.serialized(), serialized)
self.assertEqual(k.urlsafe(), urlsafe)
self.assertEqual(k.reference(), r)
k = key.Key(reference=r)
self.assertTrue(k._reference() is not r)
self.assertEqual(k.serialized(), serialized)
self.assertEqual(k.urlsafe(), urlsafe)
self.assertEqual(k.reference(), r)
k = key.Key(reference=r, app=r.app(), namespace='')
self.assertTrue(k._reference() is not r)
self.assertEqual(k.serialized(), serialized)
self.assertEqual(k.urlsafe(), urlsafe)
self.assertEqual(k.reference(), r)
k1 = key.Key('A', 1)
self.assertEqual(k1.urlsafe(), 'agFfcgcLEgFBGAEM')
k2 = key.Key(urlsafe=k1.urlsafe())
self.assertEqual(k1, k2)
def _ToPb(self):
"""Converts this Key to its protocol buffer representation.
Not intended to be used by application developers. Enforced by hiding the
entity_pb classes.
Returns:
# the Reference PB representation of this Key
entity_pb.Reference
"""
pb = entity_pb.Reference()
pb.CopyFrom(self.__reference)
if not self.has_id_or_name():
pb.mutable_path().element_list()[-1].set_id(0)
pb.app().decode('utf-8')
for pathelem in pb.path().element_list():
pathelem.type().decode('utf-8')
return pb
def test_dynamic_delete(self):
entity_lock = flexmock(EntityLock)
entity_lock.should_receive('acquire')
entity_lock.should_receive('release')
del_request = flexmock()
del_request.should_receive("key_list")
del_request.should_receive("has_transaction").never()
del_request.should_receive("transaction").never()
db_batch = flexmock()
db_batch.should_receive('valid_data_version').and_return(True)
dd = DatastoreDistributed(db_batch, self.get_zookeeper())
dd.dynamic_delete("appid", del_request)
fake_key = entity_pb.Reference()
fake_key.set_app('foo')
path = fake_key.mutable_path()
element = path.add_element()
element.set_type('bar')
element.set_id(1)
del_request = flexmock()
del_request.should_receive("key_list").and_return([fake_key])
del_request.should_receive("has_transaction").and_return(True)
transaction = flexmock()
transaction.should_receive("handle").and_return(1)
del_request.should_receive("transaction").and_return(transaction)
del_request.should_receive("has_mark_changes").and_return(False)
dd = DatastoreDistributed(db_batch, self.get_zookeeper())
flexmock(utils).should_receive("get_entity_kind").and_return("kind")
db_batch.should_receive('delete_entities_tx')
dd.dynamic_delete("appid", del_request)
del_request = flexmock()
del_request.should_receive("key_list").and_return([fake_key])
del_request.should_receive("has_transaction").and_return(False)
del_request.should_receive("has_mark_changes").and_return(False)
dd = DatastoreDistributed(db_batch, self.get_zookeeper())
flexmock(dd).should_receive("delete_entities").once()
dd.dynamic_delete("appid", del_request)
def v1_value_to_v3_property_value(self, v1_value, v3_value):
v3_value.Clear()
if v1_value.HasField('boolean_value'):
v3_value.set_booleanvalue(v1_value.boolean_value)
elif v1_value.HasField('integer_value'):
v3_value.set_int64value(v1_value.integer_value)
elif v1_value.HasField('double_value'):
v3_value.set_double_value(v1_value.double_value)
elif v1_value.HasField('timestamp_microseconds_value'):
v3_value.set_int64value(v1_value.timestamp_microseconds_value)
elif v1_value.HasField('key_value'):
v3_ref = entity_pb.Reference()
self.v1_to_v3_reference(v1_value.key_value, v3_ref)
self.v3_reference_to_v3_property_value(v3_ref, v3_value)
elif v1_value.HasField('blob_key_value'):
v3_value.set_stringvalue(v1_value.blob_key_value)
elif v1_value.HasField('string_value'):
v3_value.set_stringvalue(str(v1_value.string_value))
elif v1_value.HasField('blob_value'):
v3_value.set_stringvalue(v1_value.blob_value)
elif v1_value.HasField('entity_value'):
v1_entity_value = v1_value.entity_value
v1_meaning = v1_value.meaning
if (v1_meaning == datastore_pbs.MEANING_GEORSS_POINT or v1_meaning
== datastore_pbs.MEANING_PREDEFINED_ENTITY_POINT):
self.__v1_to_v3_point_value(v1_entity_value,
v3_value.mutable_pointvalue())
elif v1_meaning == datastore_pbs.MEANING_PREDEFINED_ENTITY_USER:
self.__v1_to_v3_user_value(v1_entity_value,
v3_value.mutable_uservalue())
else:
v3_entity_value = entity_pb.EntityProto()
self.v1_to_v3_entity(v1_entity_value, v3_entity_value)
v3_value.set_stringvalue(
v3_entity_value.SerializePartialToString())
else:
pass
def __get_v1_entity_version(self, v1_key, v1_txn):
"""Returns the version of an entity, or _NO_VERSION if it does not exist.
Args:
v1_key: the key of the entity to lookup.
v1_txn: the transaction to use when retrieving the entity.
Returns:
the version number of the entity if it was found, or _NO_VERSION
otherwise.
"""
v3_key = entity_pb.Reference()
self.__entity_converter.v1_to_v3_reference(v1_key, v3_key)
v3_txn = datastore_pb.Transaction()
self.__service_converter.v1_to_v3_txn(v1_txn, v3_txn)
v3_get_req = datastore_pb.GetRequest()
v3_get_req.mutable_transaction().CopyFrom(v3_txn)
v3_get_req.key_list().append(v3_key)
v3_get_resp = datastore_pb.GetResponse()
self.__make_v3_call('Get', v3_get_req, v3_get_resp)
if v3_get_resp.entity(0).has_entity():
return v3_get_resp.entity(0).version()
return _NO_VERSION
def _ReferenceFromReference(reference):
"""Copy a Reference."""
new_reference = entity_pb.Reference()
new_reference.CopyFrom(reference)
return new_reference
def _ReferenceFromSerialized(serialized):
"""Construct a Reference from a serialized Reference."""
assert isinstance(serialized, basestring)
if isinstance(serialized, unicode):
serialized = serialized.encode('utf8')
return entity_pb.Reference(serialized)
def v3_property_to_v4_value(self, v3_property, indexed, v4_value):
"""Converts a v3 Property to a v4 Value.
Args:
v3_property: an entity_pb.Property
indexed: whether the v3 property is indexed
v4_value: an entity_v4_pb.Value to populate
"""
v4_value.Clear()
v3_property_value = v3_property.value()
v3_meaning = v3_property.meaning()
v3_uri_meaning = None
if v3_property.meaning_uri():
v3_uri_meaning = v3_property.meaning_uri()
if not self.__is_v3_property_value_union_valid(v3_property_value):
v3_meaning = None
v3_uri_meaning = None
elif v3_meaning == entity_pb.Property.NO_MEANING:
v3_meaning = None
elif not self.__is_v3_property_value_meaning_valid(v3_property_value,
v3_meaning):
v3_meaning = None
is_zlib_value = False
if v3_uri_meaning:
if v3_uri_meaning == URI_MEANING_ZLIB:
if v3_property_value.has_stringvalue():
is_zlib_value = True
if v3_meaning != entity_pb.Property.BLOB:
v3_meaning = entity_pb.Property.BLOB
else:
pass
else:
pass
if v3_property_value.has_booleanvalue():
v4_value.set_boolean_value(v3_property_value.booleanvalue())
elif v3_property_value.has_int64value():
if v3_meaning == entity_pb.Property.GD_WHEN:
v4_value.set_timestamp_microseconds_value(
v3_property_value.int64value())
v3_meaning = None
else:
v4_value.set_integer_value(v3_property_value.int64value())
elif v3_property_value.has_doublevalue():
v4_value.set_double_value(v3_property_value.doublevalue())
elif v3_property_value.has_referencevalue():
v3_ref = entity_pb.Reference()
self.__v3_reference_value_to_v3_reference(
v3_property_value.referencevalue(), v3_ref)
self.v3_to_v4_key(v3_ref, v4_value.mutable_key_value())
elif v3_property_value.has_stringvalue():
if v3_meaning == entity_pb.Property.ENTITY_PROTO:
serialized_entity_v3 = v3_property_value.stringvalue()
v3_entity = entity_pb.EntityProto()
v3_entity.ParsePartialFromString(serialized_entity_v3)
self.v3_to_v4_entity(v3_entity, v4_value.mutable_entity_value())
v3_meaning = None
elif (v3_meaning == entity_pb.Property.BLOB
or v3_meaning == entity_pb.Property.BYTESTRING):
v4_value.set_blob_value(v3_property_value.stringvalue())
if indexed or v3_meaning == entity_pb.Property.BLOB:
v3_meaning = None
else:
string_value = v3_property_value.stringvalue()
if is_valid_utf8(string_value):
if v3_meaning == entity_pb.Property.BLOBKEY:
v4_value.set_blob_key_value(string_value)
v3_meaning = None
else:
v4_value.set_string_value(string_value)
else:
v4_value.set_blob_value(string_value)
if v3_meaning != entity_pb.Property.INDEX_VALUE:
v3_meaning = None
elif v3_property_value.has_pointvalue():
self.__v3_to_v4_point_entity(v3_property_value.pointvalue(),
v4_value.mutable_entity_value())
if v3_meaning != entity_pb.Property.GEORSS_POINT:
v4_value.set_meaning(MEANING_PREDEFINED_ENTITY_POINT)
v3_meaning = None
elif v3_property_value.has_uservalue():
self.__v3_to_v4_user_entity(v3_property_value.uservalue(),
v4_value.mutable_entity_value())
v4_value.set_meaning(MEANING_PREDEFINED_ENTITY_USER)
else:
pass
if is_zlib_value:
v4_value.set_meaning(MEANING_ZLIB)
elif v3_meaning:
v4_value.set_meaning(v3_meaning)
if indexed != v4_value.indexed():
v4_value.set_indexed(indexed)
def v3_property_to_v1_value(self, v3_property, indexed, v1_value):
v1_value.Clear()
# TODO
v3_property_value = v3_property.value()
v3_meaning = v3_property.meaning()
v3_uri_meaning = None
if v3_property.meaning_uri():
v3_uri_meaning = v3_property.meaning_uri()
if not self.__is_v3_property_value_union_valid(v3_property_value):
v3_meaning = None
v3_uri_meaning = None
elif v3_meaning == entity_pb.Property.NO_MEANING:
v3_meaning = None
elif not self.__is_v3_property_value_meaning_valid(
v3_property_value, v3_meaning):
v3_meaning = None
is_zlib_value = False
if v3_uri_meaning:
if v3_uri_meaning == datastore_pbs.URI_MEANING_ZLIB:
if v3_property_value.has_stringvalue():
is_zlib_value = True
if v3_meaning != entity_pb.Property.BLOB:
v3_meaning = entity_pb.Property.BLOB
else:
pass
else:
pass
if v3_property_value.has_booleanvalue():
v1_value.boolean_value = v3_property_value.booleanvalue()
elif v3_property_value.has_int64value():
if v3_meaning == entity_pb.Property.GD_WHEN:
v1_value.timestamp_microseconds_value = v3_property_value.int64value(
)
v3_meaning = None
else:
v1_value.integer_value = v3_property_value.int64value()
elif v3_property_value.has_doublevalue():
v1_value.double_value = v3_property_value.double_value()
elif v3_property_value.has_referencevalue():
v3_ref = entity_pb.Reference()
self.__v3_reference_value_to_v3_reference(
v3_property_value.referencevalue(), v3_ref)
self.v3_to_v1_key(v3_ref, v1_value.key)
elif v3_property_value.has_stringvalue():
if v3_meaning == entity_pb.Property.ENTITY_PROTO:
serialized_entity_v3 = v3_property_value.stringvalue()
v3_entity = entity_pb.EntityProto()
v3_entity.ParsePartialFromString(serialized_entity_v3)
self.v3_to_v1_entity(v3_entity, v1_value.entity_value)
v3_meaning = None
elif (v3_meaning == entity_pb.Property.BLOB
or v3_meaning == entity_pb.Property.BYTESTRING):
v1_value.blob_value = v3_property_value.stringvalue()
if indexed or v3_meaning == entity_pb.Property.BLOB:
v3_meaning = None
else:
string_value = v3_property_value.stringvalue()
if datastore_pbs.is_valid_utf8(string_value):
if v3_meaning == entity_pb.Property.BLOBKEY:
v1_value.blob_key_value = unicode(string_value, 'utf8')
v3_meaning = None
else:
v1_value.string_value = unicode(string_value, 'utf8')
else:
v1_value.blob_value = string_value
if v3_meaning != entity_pb.Property.INDEX_VALUE:
v3_meaning = None
elif v3_property_value.has_pointvalue():
self.__v3_to_v1_point_entity(v3_property_value.pointvalue(),
v1_value.entity_value)
if v3_meaning != entity_pb.Property.GEORESS_POINT:
v1_value.meaning = datastore_pbs.MEANING_PREDEFINED_ENTITY_POINT
v3_meaning = None
elif v3_property_value.has_uservalue():
self.__v3_to_v1_user_entity(v3_property_value.uservalue(),
v1_value.entity_value)
v1_value.meaning = datastore_pbs.MEANING_PREDEFINED_ENTITY_USER
else:
pass
if is_zlib_value:
v1_value.meaning = datastore_pbs.MEANING_ZLIB
elif v3_meaning:
v1_value.meaning = v3_meaning
if indexed != v1_value.indexed:
v1_value.indexed = indexed
