def get_by_id(cls, ids, parent=None, **kwargs):
"""Get instance of Model class by id.
Keeps the same comportement of db.Model.get_by_id()
TODO(sahid): Needs doc.
"""
if isinstance(parent, db.Model):
parent = parent.key()
ids, multiple = datastore.NormalizeAndTypeCheck(ids, (int, long))
keys = [datastore.Key.from_path(cls.kind(), id, parent=parent)
for id in ids]
ret = get(keys, **kwargs)
if multiple:
return ret
return ret[0]
def get_by_id(cls, ids, parent=None, **kwargs):
if not parent:
parent = cls.db_parent
rpc = datastore.GetRpcFromKwargs(kwargs)
if isinstance(parent, db.Model):
parent = parent.key()
ids, multiple = datastore.NormalizeAndTypeCheck(ids, (int, long))
keys = [
datastore.Key.from_path(cls.kind(), id, parent=parent)
for id in ids
]
if multiple:
return get(keys, rpc=rpc)
else:
return get(keys[0], rpc=rpc)
def put(models, **kwargs):
"""Store one or more Model instance, every stored
models are pushed also into memcache.
TODO(sahid): Needs a better doc.
"""
memclient = memcache.Client()
for retry in xrange(DATASTORE_NB_RETRY):
try:
models, multiple = datastore.NormalizeAndTypeCheck(models, db.Model)
if not any(models): return multiple and [] or None # Nothings to do.
async = db.put_async(models, **kwargs)
try:
debug("Needs to put models=%s" % ','.join(m.__class__.__name__ for m in models))
#TODO(sahid): Needs factorization.
k = [unicode(x.key()) for x in models]
v = serialize(models)
memclient.set_multi(dict(zip(k, v)),
time=MEMCACHE_TIME,
key_prefix=MEMCACHE_PREFIX)
ret = async.get_result()
except datastore_errors.BadKeyError:
debug("Incomplete key passed, "
"can't store in memcached before put in the datastore.")
# Incomplete key
# It's better to use key_name with mp.
ret = async.get_result()
if ret:
k = map(unicode, ret)
v = serialize(models)
memclient.set_multi(dict(zip(k, v)),
time=MEMCACHE_TIME,
key_prefix=MEMCACHE_PREFIX)
if multiple:
return ret
return ret[0]
except (db.Timeout,
db.TransactionFailedError,
apiproxy_errors.ApplicationError,
apiproxy_errors.DeadlineExceededError), e:
logging.warn("Error during the put process, "
"retry %d in %.2fs", retry, DATASTORE_TIME_RETRY)
logging.debug(e.message)
time.sleep(DATASTORE_TIME_RETRY)
logging.exception(e)
def get_by_key_name(cls, key_names, parent=None):
if isinstance(parent, db.Model):
parent = parent.key()
key_names, multiple = datastore.NormalizeAndTypeCheck(key_names, basestring)
keys = [datastore.Key.from_path(cls.kind(), name, parent=parent)
for name in key_names]
count = 0
if multiple:
while count < settings.rotmodel["RETRY_ATTEMPTS"]:
try:
return db.get(keys)
except db.Timeout:
count += 1
time.sleep(count * settings.rotmodel["RETRY_INTERVAL"])
else:
while count < settings.rotmodel["RETRY_ATTEMPTS"]:
try:
return db.get(*keys)
except db.Timeout:
count += 1
time.sleep(count * settings.rotmodel["RETRY_INTERVAL"])
class BaseModel(db.Model):
db_parent = BASEMODEL_PARENT
protect_attrs = []
additional_attrs = []
@classmethod
def create(cls, **kwargs):
return BaseModel._create(cls, **kwargs)
@staticmethod
def _create(cls, **kwargs):
parent = BASEMODEL_PARENT
return cls(parent, **kwargs)
@property
def str_key(self):
return str(self.key())
@property
def id(self):
return str(self.key().id())
@property
def key_name(self):
return self.key().name()
def to_dict(self):
_res = {}
if hasattr(self, "stats"):
_res["stats"] = self.stats.to_dict()
_res["id"] = str(self.id)
props = self.properties()
res = dict([(k, getattr(self, k)) for k in props.keys()
if (not k.startswith("_") and k not in self.protect_attrs)
])
res.update(_res)
return res
def update(self, commit=True, **kwargs):
props = self.properties()
need_save = False
for attr, val in kwargs.iteritems():
if getattr(self, attr, None) != val:
if (not attr.startswith("_")) and (attr != "id"):
if (attr in props) or (unicode(attr) in props
) or attr in self.additional_attrs:
need_save = True
val = gae_type_convert(val, props[attr].data_type)
setattr(self, attr, val)
if attr == "public" and hasattr(self, "stats"):
self.stats.public = val
self.stats.save()
if need_save:
self.save()
return self
def put(self):
count = 0
while count < 3:
try:
ret = db.Model.put(self)
if ret:
break
except db.Timeout:
count += 1
else:
raise db.Timeout()
remove(self.key())
return ret
def save(self, commit=True):
return self.put()
def delete(self, commit=False):
keys = [self.key()]
if hasattr(self, "stats"):
keys.append(self.stats.key())
remove(keys)
return db.delete(keys)
@classmethod
def get_by_key_name(cls, key_names, parent=None, **kwargs):
try:
if not parent:
parent = cls.db_parent
parent = db._coerce_to_key(parent)
except db.BadKeyError, e:
raise db.BadArgumentError(str(e))
rpc = datastore.GetRpcFromKwargs(kwargs)
key_names, multiple = datastore.NormalizeAndTypeCheck(
key_names, basestring)
keys = [
datastore.Key.from_path(cls.kind(), name, parent=parent)
for name in key_names
]
if multiple:
return get(keys, rpc=rpc)
else:
return get(keys[0], rpc=rpc)
class Model(db.Model):
"""A Model extends db.Model.
TODO(sahid): Needs doc.
"""
def put(self, **kwargs):
"""Writes this model instance to the datastore.
Keeps the same comportement of db.Model.put()
TODO(sahid): Needs doc.
"""
self._populate_internal_entity()
return put(self, **kwargs)
def delete(self, **kwargs):
"""Deletes this entity from the datastore.
Keeps the same comportement of db.Model.delete()
TODO(sahid): Needs doc.
"""
delete(self.key(), **kwargs)
self._key = self.key()
self._key_name = None
self._parent_key = None
self._entity = None
def prefetch(self, *props):
"""Prefetch all db.ReferenceProperty of this model.
TODO(sahid): Needs doc.
"""
prefetch(self, *props)
@classmethod
def get(cls, keys, **kwargs):
"""Fetch instance from the datastore of a specific Model type using key.
Keeps the same comportement of db.Model.get()
TODO(sahid): Needs doc.
"""
results = get(keys, **kwargs)
if results is None:
return None
if isinstance(results, db.Model):
instances = [results]
else:
instances = results
for instance in instances:
if not(instance is None or isinstance(instance, cls)):
raise db.KindError('Kind %r is not a subclass of kind %r' %
(instance.kind(), cls.kind()))
return results
@classmethod
def get_by_key_name(cls, key_names, parent=None, **kwargs):
"""Get instance of Model class by its key's name.
Keeps the same comportement of db.Model.get_by_key_name()
TODO(sahid): Needs doc.
"""
try:
parent = db._coerce_to_key(parent)
except db.BadKeyError, e:
raise db.BadArgumentError(str(e))
key_names, multiple = datastore.NormalizeAndTypeCheck(key_names, basestring)
keys = [datastore.Key.from_path(cls.kind(), name, parent=parent)
for name in key_names]
ret = get(keys, **kwargs)
if multiple:
return ret
return ret[0]
class BaseModel(db.Model):
key_template = ""
db_parent = BASEMODEL_PARENT
@property
def str_key(self):
return str(self.key())
@property
def id(self):
return self.key().id()
@property
def keyname(self):
return self.key().name()
def save_settings(self, **kwds):
props = self.properties()
for p in props.keys():
if unicode(p) in kwds:
val = kwds.get(p)
t = props[p].data_type
if t == list and not isinstance(val, list):
val = [
line.strip() for line in val.strip().split("\n")
if (line and line.strip())
]
elif t == bool and not isinstance(val, bool):
val = val.strip().capitalize()
if val == "False" or val == u"False":
val = False
else:
val = True
elif t == basestring:
try:
val = str(val).strip()
except:
val = unicode(val).strip()
else:
val = t(val)
setattr(self, p, val)
self.save()
return self
def put(self):
count = 0
while count < 3:
try:
ret = db.Model.put(self)
if ret:
break
except db.Timeout:
count += 1
else:
raise db.Timeout()
remove(self.key())
return ret
save = put
Save = put
def delete(self):
remove(self.key())
return super(BaseModel, self).delete()
@classmethod
def gen_key_name(cls, **kw):
try:
return cls.key_template % kw
except KeyError:
logging.warn('generate key_name failed: %s <- %s',
cls.key_template, kw)
@classmethod
def get_by_key_name(cls, key_names, parent=None, **kwargs):
try:
if not parent:
parent = cls.db_parent
parent = db._coerce_to_key(parent)
except db.BadKeyError, e:
raise db.BadArgumentError(str(e))
rpc = datastore.GetRpcFromKwargs(kwargs)
key_names, multiple = datastore.NormalizeAndTypeCheck(
key_names, basestring)
keys = [
datastore.Key.from_path(cls.kind(), name, parent=parent)
for name in key_names
]
if multiple:
return get(keys, rpc=rpc)
else:
return get(keys[0], rpc=rpc)
