class MultiRelationCls(object): c = operators.Slots() rels = rels_tmp def __init__(self, thing1, thing2, *a, **kw): r = self.rel(thing1, thing2) self.__class__ = r self.__init__(thing1, thing2, *a, **kw) @classmethod def rel(cls, thing1, thing2): t1 = thing1 if isinstance(thing1, ThingMeta) else thing1.__class__ t2 = thing2 if isinstance(thing2, ThingMeta) else thing2.__class__ return cls.rels[(t1, t2)] @classmethod def _query(cls, *rules, **kw): #TODO it should be possible to send the rules and kw to #the merge constructor queries = [r._query(*rules, **kw) for r in cls.rels.values()] if "sort" in kw: print "sorting MultiRelations is not supported" return Merge(queries) @classmethod def _fast_query(cls, sub, obj, name, data=True, eager_load=True, thing_data=False, timestamp_optimize=False): #divide into types def type_dict(items): types = {} for i in items: types.setdefault(i.__class__, []).append(i) return types sub_dict = type_dict(tup(sub)) obj_dict = type_dict(tup(obj)) #for each pair of types, see if we have a query to send res = {} for types, rel in cls.rels.iteritems(): t1, t2 = types if sub_dict.has_key(t1) and obj_dict.has_key(t2): res.update( rel._fast_query(sub_dict[t1], obj_dict[t2], name, data=data, eager_load=eager_load, thing_data=thing_data, timestamp_optimize=timestamp_optimize)) return res
class DataThing(object): _base_props = () _int_props = () _data_int_props = () _int_prop_suffix = None _defaults = {} _essentials = () c = operators.Slots() __safe__ = False _asked_for_data = False def __init__(self): safe_set_attr = SafeSetAttr(self) with safe_set_attr: self.safe_set_attr = safe_set_attr self._dirties = {} self._t = {} self._created = False self._loaded = True self._asked_for_data = True # You just created it; of course # you're allowed to touch its data #TODO some protection here? def __setattr__(self, attr, val, make_dirty=True): if attr.startswith('__') or self.__safe__: object.__setattr__(self, attr, val) return if attr.startswith('_'): #assume baseprops has the attr if make_dirty and hasattr(self, attr): old_val = getattr(self, attr) object.__setattr__(self, attr, val) if not attr in self._base_props: return else: old_val = self._t.get(attr, self._defaults.get(attr)) self._t[attr] = val if make_dirty and val != old_val: self._dirties[attr] = (old_val, val) def __getattr__(self, attr): #makes pickling work for some reason if attr.startswith('__'): raise AttributeError, attr if not (attr.startswith('_') or self._asked_for_data or getattr(self, "_nodb", False)): msg = ("getattr(%r) called on %r, " + "but you didn't say data=True") % (attr, self) raise ValueError(msg) try: if hasattr(self, '_t'): rv = self._t[attr] return rv else: raise AttributeError, attr except KeyError: try: return getattr(self, '_defaults')[attr] except KeyError: try: _id = object.__getattribute__(self, "_id") except AttributeError: _id = "???" try: cl = object.__getattribute__(self, "__class__").__name__ except AttributeError: cl = "???" if self._loaded: nl = "it IS loaded" else: nl = "it is NOT loaded" # The %d format is nicer since it has no "L" at the # end, but if we can't do that, fall back on %r. try: id_str = "%d" % _id except TypeError: id_str = "%r" % _id descr = '%s(%s).%s' % (cl, id_str, attr) try: essentials = object.__getattribute__(self, "_essentials") except AttributeError: print "%s has no _essentials" % descr essentials = () if isinstance(essentials, str): print "Some dumbass forgot a comma." essentials = essentials, deleted = object.__getattribute__(self, "_deleted") if deleted: nl += " and IS deleted." else: nl += " and is NOT deleted." if attr in essentials and not deleted: log_text ("essentials-bandaid-reload", "%s not found; %s Forcing reload." % (descr, nl), "warning") self._load() try: return self._t[attr] except KeyError: log_text ("essentials-bandaid-failed", "Reload of %s didn't help. I recommend deletion." % descr, "error") raise AttributeError, '%s not found; %s' % (descr, nl) def _cache_key(self): return thing_prefix(self.__class__.__name__, self._id) def _other_self(self): """Load from the cached version of myself. Skip the local cache.""" l = cache.get(self._cache_key(), allow_local = False) if l and l._id != self._id: g.log.error("thing.py: Doppleganger on read: got %s for %s", (l, self)) cache.delete(self._cache_key()) return return l def _cache_myself(self): ck = self._cache_key() cache.set(ck, self) def _sync_latest(self): """Load myself from the cache to and re-apply the .dirties list to make sure we don't overwrite a previous commit. """ other_self = self._other_self() if not other_self: return self._dirty #copy in the cache's version for prop in self._base_props: self.__setattr__(prop, getattr(other_self, prop), False) if other_self._loaded: self._t = other_self._t #re-apply the .dirties old_dirties = self._dirties self._dirties = {} for k, (old_val, new_val) in old_dirties.iteritems(): setattr(self, k, new_val) #return whether we're still dirty or not return self._dirty def _commit(self, keys=None): lock = None try: if not self._created: begin() self._create() just_created = True else: just_created = False lock = g.make_lock("thing_commit", 'commit_' + self._fullname) lock.acquire() if not just_created and not self._sync_latest(): #sync'd and we have nothing to do now, but we still cache anyway self._cache_myself() return # begin is a no-op if already done, but in the not-just-created # case we need to do this here because the else block is not # executed when the try block is exited prematurely in any way # (including the return in the above branch) begin() if keys: keys = tup(keys) to_set = dict((k, self._dirties[k][1]) for k in keys if self._dirties.has_key(k)) else: to_set = dict((k, v[1]) for k, v in self._dirties.iteritems()) data_props = {} thing_props = {} for k, v in to_set.iteritems(): if k.startswith('_'): thing_props[k[1:]] = v else: data_props[k] = v if data_props: self._set_data(self._type_id, self._id, just_created, **data_props) if thing_props: self._set_props(self._type_id, self._id, **thing_props) if keys: for k in keys: if self._dirties.has_key(k): del self._dirties[k] else: self._dirties.clear() self._cache_myself() except: rollback() raise else: commit() finally: if lock: lock.release() @classmethod def _load_multi(cls, need): need = tup(need) need_ids = [n._id for n in need] datas = cls._get_data(cls._type_id, need_ids) to_save = {} try: essentials = object.__getattribute__(cls, "_essentials") except AttributeError: essentials = () for i in need: #if there wasn't any data, keep the empty dict i._t.update(datas.get(i._id, i._t)) i._loaded = True for attr in essentials: if attr not in i._t: print "Warning: %s is missing %s" % (i._fullname, attr) i._asked_for_data = True to_save[i._id] = i prefix = thing_prefix(cls.__name__) #write the data to the cache cache.set_multi(to_save, prefix=prefix) def _load(self): self._load_multi(self) def _safe_load(self): if not self._loaded: self._load() def _incr(self, prop, amt = 1): if self._dirty: raise ValueError, "cannot incr dirty thing" #make sure we're incr'ing an _int_prop or _data_int_prop. if prop not in self._int_props: if (prop in self._data_int_props or self._int_prop_suffix and prop.endswith(self._int_prop_suffix)): #if we're incr'ing a data_prop, make sure we're loaded if not self._loaded: self._load() else: msg = ("cannot incr non int prop %r on %r -- it's not in %r or %r" % (prop, self, self._int_props, self._data_int_props)) raise ValueError, msg with g.make_lock("thing_commit", 'commit_' + self._fullname): self._sync_latest() old_val = getattr(self, prop) if self._defaults.has_key(prop) and self._defaults[prop] == old_val: #potential race condition if the same property gets incr'd #from default at the same time setattr(self, prop, old_val + amt) self._commit(prop) else: self.__setattr__(prop, old_val + amt, False) #db if prop.startswith('_'): tdb.incr_thing_prop(self._type_id, self._id, prop[1:], amt) else: self._incr_data(self._type_id, self._id, prop, amt) self._cache_myself() @property def _id36(self): return to36(self._id) @classmethod def _fullname_from_id36(cls, id36): return cls._type_prefix + to36(cls._type_id) + '_' + id36 @property def _fullname(self): return self._fullname_from_id36(self._id36) #TODO error when something isn't found? @classmethod def _byID(cls, ids, data=False, return_dict=True, extra_props=None, stale=False): ids, single = tup(ids, True) prefix = thing_prefix(cls.__name__) if not all(x <= tdb.MAX_THING_ID for x in ids): raise NotFound('huge thing_id in %r' % ids) def count_found(ret, still_need): cache.stats.cache_report( hits=len(ret), misses=len(still_need), cache_name='sgm.%s' % cls.__name__) if not cache.stats: count_found = None def items_db(ids): items = cls._get_item(cls._type_id, ids) for i in items.keys(): items[i] = cls._build(i, items[i]) return items bases = sgm(cache, ids, items_db, prefix, stale=stale, found_fn=count_found) #check to see if we found everything we asked for for i in ids: if i not in bases: missing = [i for i in ids if i not in bases] raise NotFound, '%s %s' % (cls.__name__, missing) if bases[i] and bases[i]._id != i: g.log.error("thing.py: Doppleganger on byID: %s got %s for %s" % (cls.__name__, bases[i]._id, i)) bases[i] = items_db([i]).values()[0] bases[i]._cache_myself() if data: need = [] for v in bases.itervalues(): v._asked_for_data = True if not v._loaded: need.append(v) if need: cls._load_multi(need) ### The following is really handy for debugging who's forgetting data=True: # else: # for v in bases.itervalues(): # if v._id in (1, 2, 123): # raise ValueError #e.g. add the sort prop if extra_props: for _id, props in extra_props.iteritems(): for k, v in props.iteritems(): bases[_id].__setattr__(k, v, False) if single: return bases[ids[0]] elif return_dict: return bases else: return filter(None, (bases.get(i) for i in ids)) @classmethod def _byID36(cls, id36s, return_dict = True, **kw): id36s, single = tup(id36s, True) # will fail if it's not a string ids = [ int(x, 36) for x in id36s ] things = cls._byID(ids, return_dict=True, **kw) if single: return things.values()[0] elif return_dict: return things else: return things.values() @classmethod def _by_fullname(cls, names, return_dict = True, **kw): names, single = tup(names, True) table = {} lookup = {} # build id list by type for fullname in names: try: real_type, thing_id = fullname.split('_') #distinguish between things and realtions if real_type[0] == 't': type_dict = thing_types elif real_type[0] == 'r': type_dict = rel_types real_type = type_dict[int(real_type[1:], 36)] thing_id = int(thing_id, 36) lookup[fullname] = (real_type, thing_id) table.setdefault(real_type, []).append(thing_id) except ValueError: if single: raise NotFound # lookup ids for each type identified = {} for real_type, thing_ids in table.iteritems(): i = real_type._byID(thing_ids, **kw) identified[real_type] = i # interleave types in original order of the name res = [] for fullname in names: if lookup.has_key(fullname): real_type, thing_id = lookup[fullname] res.append((fullname, identified.get(real_type, {}).get(thing_id))) if single: return res[0][1] elif return_dict: return dict(res) else: return [x for i, x in res] @property def _dirty(self): return bool(len(self._dirties)) @classmethod def _query(cls, *a, **kw): raise NotImplementedError() @classmethod def _build(*a, **kw): raise NotImplementedError() def _get_data(*a, **kw): raise NotImplementedError() def _set_data(*a, **kw): raise NotImplementedError() def _incr_data(*a, **kw): raise NotImplementedError() def _get_item(*a, **kw): raise NotImplementedError def _create(self): base_props = (getattr(self, prop) for prop in self._base_props) self._id = self._make_fn(self._type_id, *base_props) self._created = True
class DataThing(object): _base_props = () _int_props = () _data_int_props = () _int_prop_suffix = None _defaults = {} c = operators.Slots() __safe__ = False def __init__(self): safe_set_attr = SafeSetAttr(self) with safe_set_attr: self.safe_set_attr = safe_set_attr self._dirties = {} self._t = {} self._created = False self._loaded = True #TODO some protection here? def __setattr__(self, attr, val, make_dirty=True): if attr.startswith('__') or self.__safe__: object.__setattr__(self, attr, val) return if attr.startswith('_'): #assume baseprops has the attr if make_dirty and hasattr(self, attr): old_val = getattr(self, attr) object.__setattr__(self, attr, val) if not attr in self._base_props: return else: old_val = self._t.get(attr, self._defaults.get(attr)) self._t[attr] = val if make_dirty and val != old_val: self._dirties[attr] = val def __getattr__(self, attr): #makes pickling work for some reason if attr.startswith('__'): raise AttributeError try: if hasattr(self, '_t'): return self._t[attr] else: raise AttributeError, attr except KeyError: try: return getattr(self, '_defaults')[attr] except KeyError: if self._loaded: raise AttributeError, '%s not found' % attr else: raise AttributeError,\ attr + ' not found. thing is not loaded' def _commit(self, keys=None): if not self._created: self._create() if self._dirty: if keys: keys = tup(keys) to_set = dict((k, self._dirties[k]) for k in keys if self._dirties.has_key(k)) else: to_set = self._dirties data_props = {} thing_props = {} for k, v in to_set.iteritems(): if k.startswith('_'): thing_props[k[1:]] = v else: data_props[k] = v if data_props: self._set_data(self._type_id, self._id, **data_props) if thing_props: self._set_props(self._type_id, self._id, **thing_props) if keys: for k in keys: if self._dirties.has_key(k): del self._dirties[k] else: self._dirties.clear() # always set the cache cache.set(thing_prefix(self.__class__.__name__, self._id), self) @classmethod def _load_multi(cls, need): need = tup(need) need_ids = [n._id for n in need] datas = cls._get_data(cls._type_id, need_ids) to_save = {} for i in need: #if there wasn't any data, keep the empty dict i._t.update(datas.get(i._id, i._t)) i._loaded = True to_save[i._id] = i prefix = thing_prefix(cls.__name__) #avoid race condition when incrementing data int props by #putting all the int props into the cache. #prop prefix def pp(prop, id): return prop + '_' + str(i._id) #do defined data props first, this also checks default values for prop in cls._data_int_props: for i in need: to_save[pp(prop, i._id)] = getattr(i, prop) #int props based on the suffix for i in need: for prop, val in i._t.iteritems(): if cls._int_prop_suffix and prop.endswith(cls._int_prop_suffix): to_save[pp(prop, i._id)] = val cache.set_multi(to_save, prefix) def _load(self): self._load_multi(self) def _safe_load(self): if not self._loaded: self._load() def _incr(self, prop, amt = 1): if self._dirty: raise ValueError, "cannot incr dirty thing" prefix = thing_prefix(self.__class__.__name__) key = prefix + prop + '_' + str(self._id) cache_val = old_val = cache.get(key) if old_val is None: old_val = getattr(self, prop) if self._defaults.has_key(prop) and self._defaults[prop] == old_val: #potential race condition if the same property gets incr'd #from default at the same time setattr(self, prop, old_val + amt) self._commit(prop) else: self.__setattr__(prop, old_val + amt, False) #db if prop.startswith('_'): tdb.incr_thing_prop(self._type_id, self._id, prop[1:], amt) else: self._incr_data(self._type_id, self._id, prop, amt) cache.set(prefix + str(self._id), self) #cache if cache_val: cache.incr(key, amt) else: cache.set(key, getattr(self, prop)) @property def _id36(self): return to36(self._id) @property def _fullname(self): return self._type_prefix + to36(self._type_id) + '_' + to36(self._id) #TODO error when something isn't found? @classmethod def _byID(cls, ids, data=False, return_dict=True, extra_props=None): ids, single = tup(ids, True) prefix = thing_prefix(cls.__name__) def items_db(ids): items = cls._get_item(cls._type_id, ids) for i in items.keys(): items[i] = cls._build(i, items[i]) #avoid race condition when incrmenting int props (data int #props are set in load_multi) for prop in cls._int_props: keys = dict((i, getattr(item, prop)) for i, item in items.iteritems()) cache.set_multi(keys, prefix + prop + '_' ) return items bases = sgm(cache, ids, items_db, prefix) #check to see if we found everything we asked for if any(i not in bases for i in ids): missing = [i for i in ids if i not in bases] raise NotFound, '%s %s' % (cls.__name__, missing) if data: need = [v for v in bases.itervalues() if not v._loaded] if need: cls._load_multi(need) #e.g. add the sort prop if extra_props: for _id, props in extra_props.iteritems(): for k, v in props.iteritems(): bases[_id].__setattr__(k, v, False) if single: return bases[ids[0]] elif return_dict: return bases else: return filter(None, (bases.get(i) for i in ids)) @classmethod def _by_fullname(cls, names, return_dict = True, data=False, extra_props=None): names, single = tup(names, True) table = {} lookup = {} # build id list by type for fullname in names: try: real_type, thing_id = fullname.split('_') #distinguish between things and realtions if real_type[0] == 't': type_dict = thing_types elif real_type[0] == 'r': type_dict = rel_types real_type = type_dict[int(real_type[1:], 36)] thing_id = int(thing_id, 36) lookup[fullname] = (real_type, thing_id) table.setdefault(real_type, []).append(thing_id) except ValueError: if single: raise NotFound # lookup ids for each type identified = {} for real_type, thing_ids in table.iteritems(): i = real_type._byID(thing_ids, data = data, extra_props = extra_props) identified[real_type] = i # interleave types in original order of the name res = [] for fullname in names: if lookup.has_key(fullname): real_type, thing_id = lookup[fullname] res.append((fullname, identified.get(real_type, {}).get(thing_id))) if single: return res[0][1] elif return_dict: return dict(res) else: return [x for i, x in res] @property def _dirty(self): return bool(len(self._dirties)) @classmethod def _query(cls, *a, **kw): raise NotImplementedError() @classmethod def _build(*a, **kw): raise NotImplementedError() def _get_data(*a, **kw): raise NotImplementedError() def _set_data(*a, **kw): raise NotImplementedError() def _incr_data(*a, **kw): raise NotImplementedError() def _get_item(*a, **kw): raise NotImplementedError def _create(self): base_props = (getattr(self, prop) for prop in self._base_props) self._id = self._make_fn(self._type_id, *base_props) self._created = True