def get_by_id(cls, ids, parent=None, **kwargs): """Get instance of Model class by id. Keeps the same comportement of db.Model.get_by_id() TODO(sahid): Needs doc. """ if isinstance(parent, db.Model): parent = parent.key() ids, multiple = datastore.NormalizeAndTypeCheck(ids, (int, long)) keys = [datastore.Key.from_path(cls.kind(), id, parent=parent) for id in ids] ret = get(keys, **kwargs) if multiple: return ret return ret[0]
def get_by_id(cls, ids, parent=None, **kwargs): if not parent: parent = cls.db_parent rpc = datastore.GetRpcFromKwargs(kwargs) if isinstance(parent, db.Model): parent = parent.key() ids, multiple = datastore.NormalizeAndTypeCheck(ids, (int, long)) keys = [ datastore.Key.from_path(cls.kind(), id, parent=parent) for id in ids ] if multiple: return get(keys, rpc=rpc) else: return get(keys[0], rpc=rpc)
def put(models, **kwargs): """Store one or more Model instance, every stored models are pushed also into memcache. TODO(sahid): Needs a better doc. """ memclient = memcache.Client() for retry in xrange(DATASTORE_NB_RETRY): try: models, multiple = datastore.NormalizeAndTypeCheck(models, db.Model) if not any(models): return multiple and [] or None # Nothings to do. async = db.put_async(models, **kwargs) try: debug("Needs to put models=%s" % ','.join(m.__class__.__name__ for m in models)) #TODO(sahid): Needs factorization. k = [unicode(x.key()) for x in models] v = serialize(models) memclient.set_multi(dict(zip(k, v)), time=MEMCACHE_TIME, key_prefix=MEMCACHE_PREFIX) ret = async.get_result() except datastore_errors.BadKeyError: debug("Incomplete key passed, " "can't store in memcached before put in the datastore.") # Incomplete key # It's better to use key_name with mp. ret = async.get_result() if ret: k = map(unicode, ret) v = serialize(models) memclient.set_multi(dict(zip(k, v)), time=MEMCACHE_TIME, key_prefix=MEMCACHE_PREFIX) if multiple: return ret return ret[0] except (db.Timeout, db.TransactionFailedError, apiproxy_errors.ApplicationError, apiproxy_errors.DeadlineExceededError), e: logging.warn("Error during the put process, " "retry %d in %.2fs", retry, DATASTORE_TIME_RETRY) logging.debug(e.message) time.sleep(DATASTORE_TIME_RETRY) logging.exception(e)
def get_by_key_name(cls, key_names, parent=None): if isinstance(parent, db.Model): parent = parent.key() key_names, multiple = datastore.NormalizeAndTypeCheck(key_names, basestring) keys = [datastore.Key.from_path(cls.kind(), name, parent=parent) for name in key_names] count = 0 if multiple: while count < settings.rotmodel["RETRY_ATTEMPTS"]: try: return db.get(keys) except db.Timeout: count += 1 time.sleep(count * settings.rotmodel["RETRY_INTERVAL"]) else: while count < settings.rotmodel["RETRY_ATTEMPTS"]: try: return db.get(*keys) except db.Timeout: count += 1 time.sleep(count * settings.rotmodel["RETRY_INTERVAL"])
class BaseModel(db.Model): db_parent = BASEMODEL_PARENT protect_attrs = [] additional_attrs = [] @classmethod def create(cls, **kwargs): return BaseModel._create(cls, **kwargs) @staticmethod def _create(cls, **kwargs): parent = BASEMODEL_PARENT return cls(parent, **kwargs) @property def str_key(self): return str(self.key()) @property def id(self): return str(self.key().id()) @property def key_name(self): return self.key().name() def to_dict(self): _res = {} if hasattr(self, "stats"): _res["stats"] = self.stats.to_dict() _res["id"] = str(self.id) props = self.properties() res = dict([(k, getattr(self, k)) for k in props.keys() if (not k.startswith("_") and k not in self.protect_attrs) ]) res.update(_res) return res def update(self, commit=True, **kwargs): props = self.properties() need_save = False for attr, val in kwargs.iteritems(): if getattr(self, attr, None) != val: if (not attr.startswith("_")) and (attr != "id"): if (attr in props) or (unicode(attr) in props ) or attr in self.additional_attrs: need_save = True val = gae_type_convert(val, props[attr].data_type) setattr(self, attr, val) if attr == "public" and hasattr(self, "stats"): self.stats.public = val self.stats.save() if need_save: self.save() return self def put(self): count = 0 while count < 3: try: ret = db.Model.put(self) if ret: break except db.Timeout: count += 1 else: raise db.Timeout() remove(self.key()) return ret def save(self, commit=True): return self.put() def delete(self, commit=False): keys = [self.key()] if hasattr(self, "stats"): keys.append(self.stats.key()) remove(keys) return db.delete(keys) @classmethod def get_by_key_name(cls, key_names, parent=None, **kwargs): try: if not parent: parent = cls.db_parent parent = db._coerce_to_key(parent) except db.BadKeyError, e: raise db.BadArgumentError(str(e)) rpc = datastore.GetRpcFromKwargs(kwargs) key_names, multiple = datastore.NormalizeAndTypeCheck( key_names, basestring) keys = [ datastore.Key.from_path(cls.kind(), name, parent=parent) for name in key_names ] if multiple: return get(keys, rpc=rpc) else: return get(keys[0], rpc=rpc)
class Model(db.Model): """A Model extends db.Model. TODO(sahid): Needs doc. """ def put(self, **kwargs): """Writes this model instance to the datastore. Keeps the same comportement of db.Model.put() TODO(sahid): Needs doc. """ self._populate_internal_entity() return put(self, **kwargs) def delete(self, **kwargs): """Deletes this entity from the datastore. Keeps the same comportement of db.Model.delete() TODO(sahid): Needs doc. """ delete(self.key(), **kwargs) self._key = self.key() self._key_name = None self._parent_key = None self._entity = None def prefetch(self, *props): """Prefetch all db.ReferenceProperty of this model. TODO(sahid): Needs doc. """ prefetch(self, *props) @classmethod def get(cls, keys, **kwargs): """Fetch instance from the datastore of a specific Model type using key. Keeps the same comportement of db.Model.get() TODO(sahid): Needs doc. """ results = get(keys, **kwargs) if results is None: return None if isinstance(results, db.Model): instances = [results] else: instances = results for instance in instances: if not(instance is None or isinstance(instance, cls)): raise db.KindError('Kind %r is not a subclass of kind %r' % (instance.kind(), cls.kind())) return results @classmethod def get_by_key_name(cls, key_names, parent=None, **kwargs): """Get instance of Model class by its key's name. Keeps the same comportement of db.Model.get_by_key_name() TODO(sahid): Needs doc. """ try: parent = db._coerce_to_key(parent) except db.BadKeyError, e: raise db.BadArgumentError(str(e)) key_names, multiple = datastore.NormalizeAndTypeCheck(key_names, basestring) keys = [datastore.Key.from_path(cls.kind(), name, parent=parent) for name in key_names] ret = get(keys, **kwargs) if multiple: return ret return ret[0]
class BaseModel(db.Model): key_template = "" db_parent = BASEMODEL_PARENT @property def str_key(self): return str(self.key()) @property def id(self): return self.key().id() @property def keyname(self): return self.key().name() def save_settings(self, **kwds): props = self.properties() for p in props.keys(): if unicode(p) in kwds: val = kwds.get(p) t = props[p].data_type if t == list and not isinstance(val, list): val = [ line.strip() for line in val.strip().split("\n") if (line and line.strip()) ] elif t == bool and not isinstance(val, bool): val = val.strip().capitalize() if val == "False" or val == u"False": val = False else: val = True elif t == basestring: try: val = str(val).strip() except: val = unicode(val).strip() else: val = t(val) setattr(self, p, val) self.save() return self def put(self): count = 0 while count < 3: try: ret = db.Model.put(self) if ret: break except db.Timeout: count += 1 else: raise db.Timeout() remove(self.key()) return ret save = put Save = put def delete(self): remove(self.key()) return super(BaseModel, self).delete() @classmethod def gen_key_name(cls, **kw): try: return cls.key_template % kw except KeyError: logging.warn('generate key_name failed: %s <- %s', cls.key_template, kw) @classmethod def get_by_key_name(cls, key_names, parent=None, **kwargs): try: if not parent: parent = cls.db_parent parent = db._coerce_to_key(parent) except db.BadKeyError, e: raise db.BadArgumentError(str(e)) rpc = datastore.GetRpcFromKwargs(kwargs) key_names, multiple = datastore.NormalizeAndTypeCheck( key_names, basestring) keys = [ datastore.Key.from_path(cls.kind(), name, parent=parent) for name in key_names ] if multiple: return get(keys, rpc=rpc) else: return get(keys[0], rpc=rpc)