class MultiRelationCls(object):
c = operators.Slots()
rels = rels_tmp
def __init__(self, thing1, thing2, *a, **kw):
r = self.rel(thing1, thing2)
self.__class__ = r
self.__init__(thing1, thing2, *a, **kw)
@classmethod
def rel(cls, thing1, thing2):
t1 = thing1 if isinstance(thing1, ThingMeta) else thing1.__class__
t2 = thing2 if isinstance(thing2, ThingMeta) else thing2.__class__
return cls.rels[(t1, t2)]
@classmethod
def _query(cls, *rules, **kw):
#TODO it should be possible to send the rules and kw to
#the merge constructor
queries = [r._query(*rules, **kw) for r in cls.rels.values()]
if "sort" in kw:
print "sorting MultiRelations is not supported"
return Merge(queries)
@classmethod
def _fast_query(cls,
sub,
obj,
name,
data=True,
eager_load=True,
thing_data=False,
timestamp_optimize=False):
#divide into types
def type_dict(items):
types = {}
for i in items:
types.setdefault(i.__class__, []).append(i)
return types
sub_dict = type_dict(tup(sub))
obj_dict = type_dict(tup(obj))
#for each pair of types, see if we have a query to send
res = {}
for types, rel in cls.rels.iteritems():
t1, t2 = types
if sub_dict.has_key(t1) and obj_dict.has_key(t2):
res.update(
rel._fast_query(sub_dict[t1],
obj_dict[t2],
name,
data=data,
eager_load=eager_load,
thing_data=thing_data,
timestamp_optimize=timestamp_optimize))
return res
class DataThing(object):
_base_props = ()
_int_props = ()
_data_int_props = ()
_int_prop_suffix = None
_defaults = {}
_essentials = ()
c = operators.Slots()
__safe__ = False
_asked_for_data = False
def __init__(self):
safe_set_attr = SafeSetAttr(self)
with safe_set_attr:
self.safe_set_attr = safe_set_attr
self._dirties = {}
self._t = {}
self._created = False
self._loaded = True
self._asked_for_data = True # You just created it; of course
# you're allowed to touch its data
#TODO some protection here?
def __setattr__(self, attr, val, make_dirty=True):
if attr.startswith('__') or self.__safe__:
object.__setattr__(self, attr, val)
return
if attr.startswith('_'):
#assume baseprops has the attr
if make_dirty and hasattr(self, attr):
old_val = getattr(self, attr)
object.__setattr__(self, attr, val)
if not attr in self._base_props:
return
else:
old_val = self._t.get(attr, self._defaults.get(attr))
self._t[attr] = val
if make_dirty and val != old_val:
self._dirties[attr] = (old_val, val)
def __getattr__(self, attr):
#makes pickling work for some reason
if attr.startswith('__'):
raise AttributeError, attr
if not (attr.startswith('_')
or self._asked_for_data
or getattr(self, "_nodb", False)):
msg = ("getattr(%r) called on %r, " +
"but you didn't say data=True") % (attr, self)
raise ValueError(msg)
try:
if hasattr(self, '_t'):
rv = self._t[attr]
return rv
else:
raise AttributeError, attr
except KeyError:
try:
return getattr(self, '_defaults')[attr]
except KeyError:
try:
_id = object.__getattribute__(self, "_id")
except AttributeError:
_id = "???"
try:
cl = object.__getattribute__(self, "__class__").__name__
except AttributeError:
cl = "???"
if self._loaded:
nl = "it IS loaded"
else:
nl = "it is NOT loaded"
# The %d format is nicer since it has no "L" at the
# end, but if we can't do that, fall back on %r.
try:
id_str = "%d" % _id
except TypeError:
id_str = "%r" % _id
descr = '%s(%s).%s' % (cl, id_str, attr)
try:
essentials = object.__getattribute__(self, "_essentials")
except AttributeError:
print "%s has no _essentials" % descr
essentials = ()
if isinstance(essentials, str):
print "Some dumbass forgot a comma."
essentials = essentials,
deleted = object.__getattribute__(self, "_deleted")
if deleted:
nl += " and IS deleted."
else:
nl += " and is NOT deleted."
if attr in essentials and not deleted:
log_text ("essentials-bandaid-reload",
"%s not found; %s Forcing reload." % (descr, nl),
"warning")
self._load()
try:
return self._t[attr]
except KeyError:
log_text ("essentials-bandaid-failed",
"Reload of %s didn't help. I recommend deletion."
% descr, "error")
raise AttributeError, '%s not found; %s' % (descr, nl)
def _cache_key(self):
return thing_prefix(self.__class__.__name__, self._id)
def _other_self(self):
"""Load from the cached version of myself. Skip the local cache."""
l = cache.get(self._cache_key(), allow_local = False)
if l and l._id != self._id:
g.log.error("thing.py: Doppleganger on read: got %s for %s",
(l, self))
cache.delete(self._cache_key())
return
return l
def _cache_myself(self):
ck = self._cache_key()
cache.set(ck, self)
def _sync_latest(self):
"""Load myself from the cache to and re-apply the .dirties
list to make sure we don't overwrite a previous commit. """
other_self = self._other_self()
if not other_self:
return self._dirty
#copy in the cache's version
for prop in self._base_props:
self.__setattr__(prop, getattr(other_self, prop), False)
if other_self._loaded:
self._t = other_self._t
#re-apply the .dirties
old_dirties = self._dirties
self._dirties = {}
for k, (old_val, new_val) in old_dirties.iteritems():
setattr(self, k, new_val)
#return whether we're still dirty or not
return self._dirty
def _commit(self, keys=None):
lock = None
try:
if not self._created:
begin()
self._create()
just_created = True
else:
just_created = False
lock = g.make_lock("thing_commit", 'commit_' + self._fullname)
lock.acquire()
if not just_created and not self._sync_latest():
#sync'd and we have nothing to do now, but we still cache anyway
self._cache_myself()
return
# begin is a no-op if already done, but in the not-just-created
# case we need to do this here because the else block is not
# executed when the try block is exited prematurely in any way
# (including the return in the above branch)
begin()
if keys:
keys = tup(keys)
to_set = dict((k, self._dirties[k][1])
for k in keys if self._dirties.has_key(k))
else:
to_set = dict((k, v[1]) for k, v in self._dirties.iteritems())
data_props = {}
thing_props = {}
for k, v in to_set.iteritems():
if k.startswith('_'):
thing_props[k[1:]] = v
else:
data_props[k] = v
if data_props:
self._set_data(self._type_id,
self._id,
just_created,
**data_props)
if thing_props:
self._set_props(self._type_id, self._id, **thing_props)
if keys:
for k in keys:
if self._dirties.has_key(k):
del self._dirties[k]
else:
self._dirties.clear()
self._cache_myself()
except:
rollback()
raise
else:
commit()
finally:
if lock:
lock.release()
@classmethod
def _load_multi(cls, need):
need = tup(need)
need_ids = [n._id for n in need]
datas = cls._get_data(cls._type_id, need_ids)
to_save = {}
try:
essentials = object.__getattribute__(cls, "_essentials")
except AttributeError:
essentials = ()
for i in need:
#if there wasn't any data, keep the empty dict
i._t.update(datas.get(i._id, i._t))
i._loaded = True
for attr in essentials:
if attr not in i._t:
print "Warning: %s is missing %s" % (i._fullname, attr)
i._asked_for_data = True
to_save[i._id] = i
prefix = thing_prefix(cls.__name__)
#write the data to the cache
cache.set_multi(to_save, prefix=prefix)
def _load(self):
self._load_multi(self)
def _safe_load(self):
if not self._loaded:
self._load()
def _incr(self, prop, amt = 1):
if self._dirty:
raise ValueError, "cannot incr dirty thing"
#make sure we're incr'ing an _int_prop or _data_int_prop.
if prop not in self._int_props:
if (prop in self._data_int_props or
self._int_prop_suffix and prop.endswith(self._int_prop_suffix)):
#if we're incr'ing a data_prop, make sure we're loaded
if not self._loaded:
self._load()
else:
msg = ("cannot incr non int prop %r on %r -- it's not in %r or %r" %
(prop, self, self._int_props, self._data_int_props))
raise ValueError, msg
with g.make_lock("thing_commit", 'commit_' + self._fullname):
self._sync_latest()
old_val = getattr(self, prop)
if self._defaults.has_key(prop) and self._defaults[prop] == old_val:
#potential race condition if the same property gets incr'd
#from default at the same time
setattr(self, prop, old_val + amt)
self._commit(prop)
else:
self.__setattr__(prop, old_val + amt, False)
#db
if prop.startswith('_'):
tdb.incr_thing_prop(self._type_id, self._id, prop[1:], amt)
else:
self._incr_data(self._type_id, self._id, prop, amt)
self._cache_myself()
@property
def _id36(self):
return to36(self._id)
@classmethod
def _fullname_from_id36(cls, id36):
return cls._type_prefix + to36(cls._type_id) + '_' + id36
@property
def _fullname(self):
return self._fullname_from_id36(self._id36)
#TODO error when something isn't found?
@classmethod
def _byID(cls, ids, data=False, return_dict=True, extra_props=None,
stale=False):
ids, single = tup(ids, True)
prefix = thing_prefix(cls.__name__)
if not all(x <= tdb.MAX_THING_ID for x in ids):
raise NotFound('huge thing_id in %r' % ids)
def count_found(ret, still_need):
cache.stats.cache_report(
hits=len(ret), misses=len(still_need),
cache_name='sgm.%s' % cls.__name__)
if not cache.stats:
count_found = None
def items_db(ids):
items = cls._get_item(cls._type_id, ids)
for i in items.keys():
items[i] = cls._build(i, items[i])
return items
bases = sgm(cache, ids, items_db, prefix, stale=stale,
found_fn=count_found)
#check to see if we found everything we asked for
for i in ids:
if i not in bases:
missing = [i for i in ids if i not in bases]
raise NotFound, '%s %s' % (cls.__name__, missing)
if bases[i] and bases[i]._id != i:
g.log.error("thing.py: Doppleganger on byID: %s got %s for %s" %
(cls.__name__, bases[i]._id, i))
bases[i] = items_db([i]).values()[0]
bases[i]._cache_myself()
if data:
need = []
for v in bases.itervalues():
v._asked_for_data = True
if not v._loaded:
need.append(v)
if need:
cls._load_multi(need)
### The following is really handy for debugging who's forgetting data=True:
# else:
# for v in bases.itervalues():
# if v._id in (1, 2, 123):
# raise ValueError
#e.g. add the sort prop
if extra_props:
for _id, props in extra_props.iteritems():
for k, v in props.iteritems():
bases[_id].__setattr__(k, v, False)
if single:
return bases[ids[0]]
elif return_dict:
return bases
else:
return filter(None, (bases.get(i) for i in ids))
@classmethod
def _byID36(cls, id36s, return_dict = True, **kw):
id36s, single = tup(id36s, True)
# will fail if it's not a string
ids = [ int(x, 36) for x in id36s ]
things = cls._byID(ids, return_dict=True, **kw)
if single:
return things.values()[0]
elif return_dict:
return things
else:
return things.values()
@classmethod
def _by_fullname(cls, names,
return_dict = True,
**kw):
names, single = tup(names, True)
table = {}
lookup = {}
# build id list by type
for fullname in names:
try:
real_type, thing_id = fullname.split('_')
#distinguish between things and realtions
if real_type[0] == 't':
type_dict = thing_types
elif real_type[0] == 'r':
type_dict = rel_types
real_type = type_dict[int(real_type[1:], 36)]
thing_id = int(thing_id, 36)
lookup[fullname] = (real_type, thing_id)
table.setdefault(real_type, []).append(thing_id)
except ValueError:
if single:
raise NotFound
# lookup ids for each type
identified = {}
for real_type, thing_ids in table.iteritems():
i = real_type._byID(thing_ids, **kw)
identified[real_type] = i
# interleave types in original order of the name
res = []
for fullname in names:
if lookup.has_key(fullname):
real_type, thing_id = lookup[fullname]
res.append((fullname,
identified.get(real_type, {}).get(thing_id)))
if single:
return res[0][1]
elif return_dict:
return dict(res)
else:
return [x for i, x in res]
@property
def _dirty(self):
return bool(len(self._dirties))
@classmethod
def _query(cls, *a, **kw):
raise NotImplementedError()
@classmethod
def _build(*a, **kw):
raise NotImplementedError()
def _get_data(*a, **kw):
raise NotImplementedError()
def _set_data(*a, **kw):
raise NotImplementedError()
def _incr_data(*a, **kw):
raise NotImplementedError()
def _get_item(*a, **kw):
raise NotImplementedError
def _create(self):
base_props = (getattr(self, prop) for prop in self._base_props)
self._id = self._make_fn(self._type_id, *base_props)
self._created = True
class DataThing(object):
_base_props = ()
_int_props = ()
_data_int_props = ()
_int_prop_suffix = None
_defaults = {}
c = operators.Slots()
__safe__ = False
def __init__(self):
safe_set_attr = SafeSetAttr(self)
with safe_set_attr:
self.safe_set_attr = safe_set_attr
self._dirties = {}
self._t = {}
self._created = False
self._loaded = True
#TODO some protection here?
def __setattr__(self, attr, val, make_dirty=True):
if attr.startswith('__') or self.__safe__:
object.__setattr__(self, attr, val)
return
if attr.startswith('_'):
#assume baseprops has the attr
if make_dirty and hasattr(self, attr):
old_val = getattr(self, attr)
object.__setattr__(self, attr, val)
if not attr in self._base_props:
return
else:
old_val = self._t.get(attr, self._defaults.get(attr))
self._t[attr] = val
if make_dirty and val != old_val:
self._dirties[attr] = val
def __getattr__(self, attr):
#makes pickling work for some reason
if attr.startswith('__'):
raise AttributeError
try:
if hasattr(self, '_t'):
return self._t[attr]
else:
raise AttributeError, attr
except KeyError:
try:
return getattr(self, '_defaults')[attr]
except KeyError:
if self._loaded:
raise AttributeError, '%s not found' % attr
else:
raise AttributeError,\
attr + ' not found. thing is not loaded'
def _commit(self, keys=None):
if not self._created:
self._create()
if self._dirty:
if keys:
keys = tup(keys)
to_set = dict((k, self._dirties[k])
for k in keys if self._dirties.has_key(k))
else:
to_set = self._dirties
data_props = {}
thing_props = {}
for k, v in to_set.iteritems():
if k.startswith('_'):
thing_props[k[1:]] = v
else:
data_props[k] = v
if data_props:
self._set_data(self._type_id, self._id, **data_props)
if thing_props:
self._set_props(self._type_id, self._id, **thing_props)
if keys:
for k in keys:
if self._dirties.has_key(k):
del self._dirties[k]
else:
self._dirties.clear()
# always set the cache
cache.set(thing_prefix(self.__class__.__name__, self._id), self)
@classmethod
def _load_multi(cls, need):
need = tup(need)
need_ids = [n._id for n in need]
datas = cls._get_data(cls._type_id, need_ids)
to_save = {}
for i in need:
#if there wasn't any data, keep the empty dict
i._t.update(datas.get(i._id, i._t))
i._loaded = True
to_save[i._id] = i
prefix = thing_prefix(cls.__name__)
#avoid race condition when incrementing data int props by
#putting all the int props into the cache.
#prop prefix
def pp(prop, id):
return prop + '_' + str(i._id)
#do defined data props first, this also checks default values
for prop in cls._data_int_props:
for i in need:
to_save[pp(prop, i._id)] = getattr(i, prop)
#int props based on the suffix
for i in need:
for prop, val in i._t.iteritems():
if cls._int_prop_suffix and prop.endswith(cls._int_prop_suffix):
to_save[pp(prop, i._id)] = val
cache.set_multi(to_save, prefix)
def _load(self):
self._load_multi(self)
def _safe_load(self):
if not self._loaded:
self._load()
def _incr(self, prop, amt = 1):
if self._dirty:
raise ValueError, "cannot incr dirty thing"
prefix = thing_prefix(self.__class__.__name__)
key = prefix + prop + '_' + str(self._id)
cache_val = old_val = cache.get(key)
if old_val is None:
old_val = getattr(self, prop)
if self._defaults.has_key(prop) and self._defaults[prop] == old_val:
#potential race condition if the same property gets incr'd
#from default at the same time
setattr(self, prop, old_val + amt)
self._commit(prop)
else:
self.__setattr__(prop, old_val + amt, False)
#db
if prop.startswith('_'):
tdb.incr_thing_prop(self._type_id, self._id, prop[1:], amt)
else:
self._incr_data(self._type_id, self._id, prop, amt)
cache.set(prefix + str(self._id), self)
#cache
if cache_val:
cache.incr(key, amt)
else:
cache.set(key, getattr(self, prop))
@property
def _id36(self):
return to36(self._id)
@property
def _fullname(self):
return self._type_prefix + to36(self._type_id) + '_' + to36(self._id)
#TODO error when something isn't found?
@classmethod
def _byID(cls, ids, data=False, return_dict=True, extra_props=None):
ids, single = tup(ids, True)
prefix = thing_prefix(cls.__name__)
def items_db(ids):
items = cls._get_item(cls._type_id, ids)
for i in items.keys():
items[i] = cls._build(i, items[i])
#avoid race condition when incrmenting int props (data int
#props are set in load_multi)
for prop in cls._int_props:
keys = dict((i, getattr(item, prop))
for i, item in items.iteritems())
cache.set_multi(keys, prefix + prop + '_' )
return items
bases = sgm(cache, ids, items_db, prefix)
#check to see if we found everything we asked for
if any(i not in bases for i in ids):
missing = [i for i in ids if i not in bases]
raise NotFound, '%s %s' % (cls.__name__, missing)
if data:
need = [v for v in bases.itervalues() if not v._loaded]
if need:
cls._load_multi(need)
#e.g. add the sort prop
if extra_props:
for _id, props in extra_props.iteritems():
for k, v in props.iteritems():
bases[_id].__setattr__(k, v, False)
if single:
return bases[ids[0]]
elif return_dict:
return bases
else:
return filter(None, (bases.get(i) for i in ids))
@classmethod
def _by_fullname(cls, names,
return_dict = True,
data=False, extra_props=None):
names, single = tup(names, True)
table = {}
lookup = {}
# build id list by type
for fullname in names:
try:
real_type, thing_id = fullname.split('_')
#distinguish between things and realtions
if real_type[0] == 't':
type_dict = thing_types
elif real_type[0] == 'r':
type_dict = rel_types
real_type = type_dict[int(real_type[1:], 36)]
thing_id = int(thing_id, 36)
lookup[fullname] = (real_type, thing_id)
table.setdefault(real_type, []).append(thing_id)
except ValueError:
if single:
raise NotFound
# lookup ids for each type
identified = {}
for real_type, thing_ids in table.iteritems():
i = real_type._byID(thing_ids, data = data,
extra_props = extra_props)
identified[real_type] = i
# interleave types in original order of the name
res = []
for fullname in names:
if lookup.has_key(fullname):
real_type, thing_id = lookup[fullname]
res.append((fullname,
identified.get(real_type, {}).get(thing_id)))
if single:
return res[0][1]
elif return_dict:
return dict(res)
else:
return [x for i, x in res]
@property
def _dirty(self):
return bool(len(self._dirties))
@classmethod
def _query(cls, *a, **kw):
raise NotImplementedError()
@classmethod
def _build(*a, **kw):
raise NotImplementedError()
def _get_data(*a, **kw):
raise NotImplementedError()
def _set_data(*a, **kw):
raise NotImplementedError()
def _incr_data(*a, **kw):
raise NotImplementedError()
def _get_item(*a, **kw):
raise NotImplementedError
def _create(self):
base_props = (getattr(self, prop) for prop in self._base_props)
self._id = self._make_fn(self._type_id, *base_props)
self._created = True
