def get_result(self):
if self.cache_result is not None:
return self.cache_result
if len(self.exception) >= 1:
raise self.exception[0]
else:
return [db.class_for_kind(e.kind()).from_entity(e) if isinstance(e, Entity) else e for e in self.results]
def clear_datastore(appid=('a', ''),
host=('h', ''),
path=('p', ''),
kinds=('k', ''),
clear_memcache=('c', False),
secure=True):
"""Clear all the data on GAE environment using remote_api.
"""
if not appid:
appid = get_appid()
if not host:
host = "%s.appspot.com" % appid
if not path:
path = '/remote_api'
if not kinds:
models = None
else:
models_dict = get_all_models_as_dict()
models = []
for kind in kinds.split(','):
models.append(db.class_for_kind(kind))
remote_api_stub.ConfigureRemoteApi(appid,
path,
auth_func,
host,
secure=secure,
save_cookies=True)
remote_api_stub.MaybeInvokeAuthentication()
delete_all_entities(models)
if clear_memcache:
from google.appengine.api import memcache
memcache.flush_all()
sys.stderr.write("Flushed memcache.\n")
def get_result(self):
if self.cache_result is not None:
return self.cache_result
if len(self.exception) >= 1:
raise self.exception[0]
else:
return [db.class_for_kind(e.kind()).from_entity(e) if isinstance(e, Entity) else e for e in self.results]
def post(self):
console.log(self.body)
console.log(self.client)
if not self.client is None:
if 'updates' in self.body:
keys = []
for key in self.body['updates']:
try:
keys.append(db.Key(key))
except:
pass
entities = Entity.get(keys)
for entity in entities:
entity.execute(self.body['updates'][str(entity.key())])
db.put(entities)
for entity in Entity.gql('updated > :updated', updated = self.client.updated):
self.data(entity)
if 'creates' in self.body:
entities = []
for name in self.body['creates']:
kind = db.class_for_kind(name)
for data in self.body['create'][name]:
entity = kind(data = data)
entities.append(entity)
db.put(entities)
for entity in entities:
self.data(entity)
self.client.updated = datetime.utcnow()
self.client.put()
self.data(self.client)
return self.out()
'''
def clear_datastore(appid=('a', ''), host=('h', ''), path=('p', ''),
kinds=('k', ''), clear_memcache=('c', False), secure=True):
"""Clear all the data on GAE environment using remote_api.
"""
if not appid:
appid = get_appid()
if not host:
host = "%s.appspot.com" % appid
if not path:
path = DEFAULT_REMOTE_API_PATH
if not kinds:
models = None
else:
models_dict = get_all_models_as_dict()
models = []
for kind in kinds.split(','):
models.append(db.class_for_kind(kind))
remote_api_stub.ConfigureRemoteApi(None, path, auth_func,
host, secure=secure, save_cookies=True)
remote_api_stub.MaybeInvokeAuthentication()
delete_all_entities(models)
if clear_memcache:
from google.appengine.api import memcache
memcache.flush_all()
sys.stderr.write("Flushed memcache.\n")
def getQuery(self, context, obj):
""" Query for all entities before a given datetime. """
model = context['model']
dateattr = context['dateattr']
before = context['before']
modelClass = db.class_for_kind(model)
query = modelClass.all(keys_only=True, namespace='').filter('%s <' % dateattr, before)
return query
def getQuery(self, context, obj):
""" Query for all entities before a given datetime. """
model = context['model']
dateattr = context['dateattr']
before = context['before']
modelClass = db.class_for_kind(model)
query = modelClass.all(keys_only=True,
namespace='').filter('%s <' % dateattr, before)
return query
def execute_pre_save_hooks(kind, pb_key, entity):
put_type_id = put_type.UNKNOWN
func_list = pre_save_hooks.get(kind, None)
if func_list is not None:
key = db.Key._FromPb(pb_key)
if not key.id_or_name():
path = key.to_path()[:-1]
path.append(1)
model_key = db.Key.from_path(*path)
ids = datastore.AllocateIds(model_key, 1)
path = path[:-1]
path.append(ids[0])
new_key = db.Key.from_path(*path)
pb_key = new_key._ToPb()
entity.key().CopyFrom(pb_key)
group = entity.mutable_entity_group()
root = entity.key().path().element(0)
group.add_element().CopyFrom(root)
e = datastore.Entity._FromPb(entity)
instance = db.class_for_kind(kind).from_entity(e)
put_type_id = put_type.NEWLY_CREATED
else:
entity.key_.CopyFrom(pb_key)
e = datastore.Entity._FromPb(entity)
instance = db.class_for_kind(kind).from_entity(e)
created = get_created_datetime(instance)
updated = get_updated_datetime(instance)
if created:
threshold = datetime.timedelta(0,0,1000)
if updated:
if abs(created - updated) < threshold:
put_type_id = put_type.MAYBE_NEWLY_CREATED
else:
put_type_id = put_type.MAYBE_UPDATED
else:
if (datetime.datetime.now() - created) < threshold:
put_type_id = put_type.MAYBE_NEWLY_CREATED
else:
put_type_id = put_type.MAYBE_UPDATED
for func in func_list:
func(instance, put_type_id)
def execute_pre_save_hooks(kind, pb_key, entity):
put_type_id = put_type.UNKNOWN
func_list = pre_save_hooks.get(kind, None)
if func_list is not None:
key = db.Key._FromPb(pb_key)
if not key.id_or_name():
path = key.to_path()[:-1]
path.append(1)
model_key = db.Key.from_path(*path)
ids = datastore.AllocateIds(model_key, 1)
path = path[:-1]
path.append(ids[0])
new_key = db.Key.from_path(*path)
pb_key = new_key._ToPb()
entity.key().CopyFrom(pb_key)
group = entity.mutable_entity_group()
root = entity.key().path().element(0)
group.add_element().CopyFrom(root)
e = datastore.Entity._FromPb(entity)
instance = db.class_for_kind(kind).from_entity(e)
put_type_id = put_type.NEWLY_CREATED
else:
entity.key_.CopyFrom(pb_key)
e = datastore.Entity._FromPb(entity)
instance = db.class_for_kind(kind).from_entity(e)
created = get_created_datetime(instance)
updated = get_updated_datetime(instance)
if created:
threshold = datetime.timedelta(0, 0, 1000)
if updated:
if abs(created - updated) < threshold:
put_type_id = put_type.MAYBE_NEWLY_CREATED
else:
put_type_id = put_type.MAYBE_UPDATED
else:
if (datetime.datetime.now() - created) < threshold:
put_type_id = put_type.MAYBE_NEWLY_CREATED
else:
put_type_id = put_type.MAYBE_UPDATED
for func in func_list:
func(instance, put_type_id)
def dump(obj,):
key_name = None
try:
kind = obj.kind()
key_name = obj.name()
except AttributeError:
pass
if key_name is None:
raise TypeError("db.Key instance with a name is required")
cls = db.class_for_kind(kind)
try:
return cls.parse_key_name(key_name)
except AttributeError:
return key_name
def encode_xdb_key(key, encoder=None):
"""
Convert the `db.Key` to it's entity and encode it.
"""
gae_objects = getGAEObjects(encoder.context.extra)
klass = db.class_for_kind(key.kind())
try:
referenced_object = gae_objects.get(klass, key)
except KeyError:
referenced_object = db.get(key)
gae_objects.set(klass, key, referenced_object)
if not referenced_object:
encoder.writeElement(None)
else:
encoder.writeObject(referenced_object)
def encode_xdb_key(key, encoder=None):
"""
Convert the `db.Key` to it's entity and encode it.
"""
gae_objects = getGAEObjects(encoder.context.extra)
klass = db.class_for_kind(key.kind())
try:
referenced_object = gae_objects.get(klass, key)
except KeyError:
referenced_object = db.get(key)
gae_objects.set(klass, key, referenced_object)
if not referenced_object:
encoder.writeElement(None)
else:
encoder.writeObject(referenced_object)
def execute_hooks(kind, key, entity):
put_type_id = put_type.UNKOWN
func_list = post_save_hooks.get(kind, None)
if func_list is not None:
last_path = entity.key().path().element_list()[-1]
has_name = last_path.has_name()
if last_path.has_id():
if last_path.id() == 0:
put_type_id = put_type.NEWLY_CREATED
else:
put_type_id = put_type.UPDATED
entity.key_.CopyFrom(key)
e = datastore.Entity._FromPb(entity)
instance = db.class_for_kind(kind).from_entity(e)
if has_name:
created = get_created_datetime(instance)
updated = get_updated_datetime(instance)
if created:
import datetime
threshold = datetime.timedelta(0,0,1000)
if updated:
if abs(created - updated) < threshold:
put_type_id = put_type.MAYBE_NEWLY_CREATED
else:
put_type_id = put_type.MAYBE_UPDATED
else:
if (datetime.datetime.now() - created) < threshold:
put_type_id = put_type.MAYBE_NEWLY_CREATED
else:
put_type_id = put_type.MAYBE_UPDATED
if datastore._CurrentTransactionKey():
# This operation is inside the transaction. So, we reserve the
# func_list and parameters for later execution.
tmp_list = getattr(local, '_reserved_hooks', None)
if tmp_list is None:
tmp_list = []
tmp_list.append((func_list, instance, put_type_id))
local._reserved_hooks = tmp_list
else:
for func in func_list:
func(instance, put_type_id)
def execute_post_save_hooks(kind, key, entity):
put_type_id = put_type.UNKNOWN
func_list = post_save_hooks.get(kind, None)
if func_list is not None:
last_path = entity.key().path().element_list()[-1]
has_name = last_path.has_name()
if last_path.has_id():
if last_path.id() == 0:
key_auto_generated = True
put_type_id = put_type.NEWLY_CREATED
else:
key_auto_generated = False
entity.key_.CopyFrom(key)
e = datastore.Entity._FromPb(entity)
instance = db.class_for_kind(kind).from_entity(e)
if has_name or not key_auto_generated:
created = get_created_datetime(instance)
updated = get_updated_datetime(instance)
if created:
threshold = datetime.timedelta(0, 0, 1500)
if updated:
if abs(created - updated) < threshold:
put_type_id = put_type.MAYBE_NEWLY_CREATED
else:
put_type_id = put_type.MAYBE_UPDATED
else:
if (datetime.datetime.now() - created) < threshold:
put_type_id = put_type.MAYBE_NEWLY_CREATED
else:
put_type_id = put_type.MAYBE_UPDATED
if datastore._CurrentTransactionKey():
# This operation is inside the transaction. So, we reserve the
# func_list and parameters for later execution.
tmp_list = getattr(local, '_reserved_hooks', [])
tmp_list.append((func_list, instance, put_type_id))
local._reserved_hooks = tmp_list
else:
for func in func_list:
func(instance, put_type_id)
def __init__(self, obj):
self.val = db.model_to_protobuf(obj).Encode()
self.model_class = db.class_for_kind(obj.kind())
def __init__(self, db_model_name):
self._model = db.class_for_kind(db_model_name)
def benchmark_serialization(response, model_instance):
# How data gets from a db.Model subclass to bytes:
# 1. db.Model is converted to a datastore.Entity
# 2. datastore.Entity is converted to a protocol buffer object: EntityProto
# 3. EntityProto is serialized
start = time.time()
for _ in xrange(SERIALIZATION_ITERATIONS):
entity = model_instance._populate_entity(datastore.Entity)
entity_proto = entity.ToPb()
serialized = entity_proto.SerializeToString()
end = time.time()
output(
response, 'serialized from model %d times in %f s' %
(SERIALIZATION_ITERATIONS, end - start))
start = time.time()
for _ in xrange(SERIALIZATION_ITERATIONS):
entity_proto = entity.ToPb()
serialized = entity_proto.SerializeToString()
end = time.time()
output(
response, 'serialized from datastore.Entity %d times in %f s' %
(SERIALIZATION_ITERATIONS, end - start))
start = time.time()
for _ in xrange(SERIALIZATION_ITERATIONS):
serialized = entity_proto.SerializeToString()
end = time.time()
output(
response, 'serialized from entity_pb.EntityProto %d times in %f s' %
(SERIALIZATION_ITERATIONS, end - start))
start = time.time()
for _ in xrange(SERIALIZATION_ITERATIONS):
entity_proto = entity_pb.EntityProto(serialized)
entity = datastore.Entity.FromPb(entity_proto)
# from model_from_protobuf
deserialized = db.class_for_kind(entity.kind()).from_entity(entity)
end = time.time()
output(
response, 'deserialized to model %d times in %f s' %
(SERIALIZATION_ITERATIONS, end - start))
start = time.time()
for _ in xrange(SERIALIZATION_ITERATIONS):
entity_proto = entity_pb.EntityProto(serialized)
entity = datastore.Entity.FromPb(entity_proto)
end = time.time()
output(
response, 'deserialized to datastore.Entity %d times in %f s' %
(SERIALIZATION_ITERATIONS, end - start))
start = time.time()
for _ in xrange(SERIALIZATION_ITERATIONS):
entity_proto = entity_pb.EntityProto(serialized)
end = time.time()
output(
response, 'deserialized to entity_pb.EntityProto %d times in %f s' %
(SERIALIZATION_ITERATIONS, end - start))
start = time.time()
for _ in xrange(SERIALIZATION_ITERATIONS):
entity_proto.Clear()
entity_proto.MergeFromString(serialized)
end = time.time()
output(
response,
'deserialized to entity_pb.EntityProto with reuse %d times in %f s' %
(SERIALIZATION_ITERATIONS, end - start))
output(response, ' (entity has %d keys)' % len(entity))
# model / LazyEntity property access tests
response.write('\n### model / LazyEntity property access times\n')
total_length = 0
start = time.time()
for _ in xrange(SERIALIZATION_ITERATIONS):
entity_proto = entity_pb.EntityProto(serialized)
entity = datastore.Entity.FromPb(entity_proto)
# from model_from_protobuf
deserialized = db.class_for_kind(entity.kind()).from_entity(entity)
total_length += len(deserialized.prop_a)
end = time.time()
output(
response,
'model deserialized and accessed one property %d times in %f s' %
(SERIALIZATION_ITERATIONS, end - start))
total_length = 0
start = time.time()
for _ in xrange(SERIALIZATION_ITERATIONS):
entity_proto = entity_pb.EntityProto(serialized)
entity = datastore.Entity.FromPb(entity_proto)
# from model_from_protobuf
deserialized = db.class_for_kind(entity.kind()).from_entity(entity)
total_length += len(deserialized.prop_a)
total_length += len(deserialized.prop_b)
total_length += len(deserialized.prop_c)
total_length += len(deserialized.prop_d)
total_length += len(deserialized.prop_e)
end = time.time()
output(
response,
'model deserialized and accessed five properties %d times in %f s' %
(SERIALIZATION_ITERATIONS, end - start))
total_length = 0
start = time.time()
for _ in xrange(SERIALIZATION_ITERATIONS):
entity_proto = entity_pb.EntityProto(serialized)
deserialized = datastore_lazy.LazyEntity(entity_proto)
total_length += len(deserialized.prop_a)
end = time.time()
output(
response,
'LazyEntity deserialized and accessed one property %d times in %f s' %
(SERIALIZATION_ITERATIONS, end - start))
total_length = 0
start = time.time()
for _ in xrange(SERIALIZATION_ITERATIONS):
entity_proto = entity_pb.EntityProto(serialized)
deserialized = datastore_lazy.LazyEntity(entity_proto)
total_length += len(deserialized.prop_a)
total_length += len(deserialized.prop_b)
total_length += len(deserialized.prop_c)
total_length += len(deserialized.prop_d)
total_length += len(deserialized.prop_e)
end = time.time()
output(
response,
'LazyEntity deserialized and accessed five properties %d times in %f s'
% (SERIALIZATION_ITERATIONS, end - start))
response.write('\n### protocol buffer / pure python access times\n')
total = 0
start = time.time()
for _ in xrange(SERIALIZATION_ITERATIONS):
total += entity_proto.property_size()
end = time.time()
output(
response, 'protocol buffer entity_proto.property_size access in %f s' %
(end - start))
total = 0
obj = PythonListHolder()
start = time.time()
for _ in xrange(SERIALIZATION_ITERATIONS):
total += obj.property_size()
end = time.time()
output(response,
'PythonListHolder.property_size access in %f s' % (end - start))
def processItem(self, item):
self.processItemAsEntity(item)
logging.info("processing %s %s" % (item.kind(), item.key()))
modelClass = db.class_for_kind(item.kind()).from_entity(item)
self.processItemAsClass(modelClass)
modelClass.put()
def _model(self):
if isinstance(self.__model, basestring):
self.__model = db.class_for_kind(self.__model)
return self.__model
def create(model):
_class = db.class_for_kind(model)
return _class()
def makeProtoBufObj(obj): val = db.model_to_protobuf(obj).Encode() model_class = db.class_for_kind(obj.kind()) return ProtoBufObj(val, model_class)
def __init__(self, db_model_name):
self._model = db.class_for_kind(db_model_name)
def __init__(self, kind, is_Thing = False):
object.__setattr__(self, '_model' , db.class_for_kind(kind))
object.__setattr__(self, '_is_Thing', is_Thing)
object.__setattr__(self, '_obj', None)
object.__setattr__(self, '_single_entry', False)
