def marshal(obj, keys=None, marshallerName='', objs=None):
"""
Convert an object to a dictionary. keys is an optional list of keys to
include in the returned dictionary. if keys is None then all public
attributes are returned. marshallerName is an optional marshalling
adapter name. if it is an empty string then the default marshaller will be
used.
"""
#to prevent recursing back over something twice, keep track of seen objs
if objs is None:
objs = OOSet()
# obj is itself marshallable, so make a marshaller and marshal away
if IMarshallable.providedBy(obj):
marshaller = component.getAdapter(obj, IMarshaller, marshallerName)
verify.verifyObject(IMarshaller, marshaller)
if IInfo.providedBy(obj):
key = (obj._object._p_oid, obj.__class__)
if key in objs:
raise AlreadySeenException()
else:
objs.insert(key)
try:
return marshal(marshaller.marshal(keys),
keys, marshallerName, objs)
except AlreadySeenException:
pass
finally:
objs.remove(key)
else:
return marshal(marshaller.marshal(keys), keys, marshallerName, objs)
# obj is a dict, so marshal its values recursively
# Zuul.marshal({'foo':1, 'bar':2})
if isinstance(obj, dict):
marshalled_dict = {}
for k in obj:
try:
marshalled_dict[k] = marshal(obj[k], keys, marshallerName, objs)
except AlreadySeenException:
pass
return marshalled_dict
# obj is a non-string iterable, so marshal its members recursively
# Zuul.marshal(set([o1, o2]))
elif hasattr(obj, '__iter__'):
marshalled_list = []
for o in obj:
try:
marshalled_list.append(marshal(o, keys, marshallerName, objs))
except AlreadySeenException:
pass
return marshalled_list
elif isinstance(obj, DateTime ):
return str(obj)
# Nothing matched, so it's a string or number or other unmarshallable.
else:
return obj
def read(self):
"""Return messages added and removed from folder.
Two sets of message objects are returned. The first set is
messages that were added to the folder since the last read.
The second set is the messages that were removed from the
folder since the last read.
The code assumes messages are added and removed but not edited.
"""
mbox = mailbox.UnixMailbox(open(self.path, "rb"), factory)
self._stat()
cur = OOSet()
new = OOSet()
while 1:
msg = mbox.next()
if msg is None:
break
msgid = msg["message-id"]
cur.insert(msgid)
if not self.messages.has_key(msgid):
self.messages[msgid] = msg
new.insert(msg)
removed = difference(self.messages, cur)
for msgid in removed.keys():
del self.messages[msgid]
# XXX perhaps just return the OOBTree for removed?
return new, OOSet(removed.values())
def marshal(obj, keys=None, marshallerName='', objs=None):
"""
Convert an object to a dictionary. keys is an optional list of keys to
include in the returned dictionary. if keys is None then all public
attributes are returned. marshallerName is an optional marshalling
adapter name. if it is an empty string then the default marshaller will be
used.
"""
#to prevent recursing back over something twice, keep track of seen objs
if objs is None:
objs = OOSet()
# obj is itself marshallable, so make a marshaller and marshal away
if IMarshallable.providedBy(obj):
marshaller = component.getAdapter(obj, IMarshaller, marshallerName)
verify.verifyObject(IMarshaller, marshaller)
if IInfo.providedBy(obj):
key = (getattr(obj._object, '_p_oid', id(obj)), obj.__class__)
if key in objs:
raise AlreadySeenException()
else:
objs.insert(key)
try:
return marshal(marshaller.marshal(keys),
keys, marshallerName, objs)
except AlreadySeenException:
pass
finally:
objs.remove(key)
else:
return marshal(marshaller.marshal(keys), keys, marshallerName, objs)
# obj is a dict, so marshal its values recursively
# Zuul.marshal({'foo':1, 'bar':2})
if isinstance(obj, dict):
marshalled_dict = {}
for k in obj:
try:
marshalled_dict[k] = marshal(obj[k], keys, marshallerName, objs)
except AlreadySeenException:
pass
return marshalled_dict
# obj is a non-string iterable, so marshal its members recursively
# Zuul.marshal(set([o1, o2]))
elif hasattr(obj, '__iter__'):
marshalled_list = []
for o in obj:
try:
marshalled_list.append(marshal(o, keys, marshallerName, objs))
except AlreadySeenException:
pass
return marshalled_list
elif isinstance(obj, DateTime ):
return str(obj)
# Nothing matched, so it's a string or number or other unmarshallable.
else:
return obj
def read(self):
"""Return messages added and removed from folder.
Two sets of message objects are returned. The first set is
messages that were added to the folder since the last read.
The second set is the messages that were removed from the
folder since the last read.
The code assumes messages are added and removed but not edited.
"""
mbox = mailbox.UnixMailbox(open(self.path, "rb"), factory)
self._stat()
cur = OOSet()
new = OOSet()
while 1:
msg = mbox.next()
if msg is None:
break
msgid = msg["message-id"]
cur.insert(msgid)
if not self.messages.has_key(msgid):
self.messages[msgid] = msg
new.insert(msg)
removed = difference(self.messages, cur)
for msgid in removed.keys():
del self.messages[msgid]
# XXX perhaps just return the OOBTree for removed?
return new, OOSet(removed.values())
class StarredTasks(grok.Annotation):
grok.context(ifaces.IAccount)
grok.implements(ifaces.IStarredTasks)
def __init__(self):
self._starred_tasks = OOSet()
def getStarredTasks(self):
return list(self._starred_tasks)
def addStarredTask(self, task_intid):
self._starred_tasks.insert(task_intid)
def removeStarredTask(self, task_intid):
if task_intid in self._starred_tasks:
self._starred_tasks.remove(task_intid)
def setTags(self, pid, oid, tags):
self.removeTags(pid, oid)
if not tags:
return
t = []
for tag in tags:
tag = tag.lower().strip()
if tag:
t.append(tag)
if not t:
return
tags = t
pdata = self.tags.get(pid)
if pdata is None:
pdata = IOBTree()
self.tags[pid] = pdata
oids = pdata.get(oid)
if oids is None:
oids = OOSet()
pdata[oid] = oids
oids.update(tags)
# insert oid -> pid reference
oids = self.oids.get(oid)
if oids is None:
oids = OOSet()
self.oids[oid] = oids
oids.insert(pid)
#update tagging engine
engine = self.getEngine(pid)
try:
engine.update(oid, tags)
except:
engine.clear()
engine.update(oid, tags)
def sort(self, result):
index = self.index
values = self.index.documents_to_values
seq = OOBTree()
for key in result:
idx = values.get(key, None)
if idx is None:
continue
data = seq.get(idx, None)
if data is None:
data = OOSet()
data.insert(key)
seq[idx] = data
result = []
for idx, data in seq.items():
result.extend(data)
return result
def learn(self, rid, value):
# Validate
# ========
if not isinstance(value, basestring):
raise TypeError("value is not a string: '%s'" % value)
if not self.case_sensitive:
value = value.lower()
# Add to the (one:many) mapping, value to rids.
# =============================================
if value in self.values:
self.values[value].insert(rid)
else:
self.values[value] = IITreeSet([rid])
# Add to the (one:many) beginnings, middles, and endings indices.
# ==================================================================
# These are for the startswith, contains, and endswith searches, which
# function basically like the is search, but we have to learn all
# substrings of the string.
substrings = OOSet()
for i in range(len(value)):
j = i + 1
# beginnings/startswith
# =====================
part = value[:j]
if part in self.beginnings:
self.beginnings[part].insert(rid)
else:
self.beginnings[part] = IITreeSet([rid])
# middles/contains
# ================
for k in range(len(value)-i):
part = value[k:k+j]
if part in self.middles:
self.middles[part].insert(rid)
else:
self.middles[part] = IITreeSet([rid])
substrings.insert(part)
# endings/endswith
# ================
part = value[i:]
if part in self.endings:
self.endings[part].insert(rid)
else:
self.endings[part] = IITreeSet([rid])
# Add to the (one:one) mapping of rid to value.
# =============================================
# This exists so we know where the rid is located when the time comes
# to forget it.
self.rids[rid] = substrings
class TaggingEngine(Persistent):
interface.implements(ITaggingEngine)
def __init__(self):
self.clear()
@property
def tagsCount(self):
return len(self.tag_oids)
@property
def itemsCount(self):
return len(self.oid_tags)
def clear(self):
self.tagsmap = LOBTree()
# index
self.tag_oids = LOBTree()
self.oid_tags = LOBTree()
# tag weight
self.weights = OOSet()
self.tag_weight = LFBTree()
def getHash(self, str):
if not str:
return 0 # empty
res = hash(str)
# NOTE: workaround for 64bit
if struct.calcsize("P") * 8 == 64:
res = ord(str[0]) << 7
for char in str:
res = c_mul(1000003, res) ^ ord(char)
res = res ^ len(str)
if res == -1:
res = -2
if res >= 2**31:
res -= 2**32
return res
def update(self, oid, tags):
self.remove(oid)
for tag in tags:
htag = self.getHash(tag)
self.tagsmap[htag] = tag
# add oid -> tag
oids = self.tag_oids.get(htag)
if oids is None:
oids = LFSet()
self.tag_oids[htag] = oids
if oid not in oids:
oids.insert(oid)
# add tag -> oid
oid_tags = self.oid_tags.get(oid)
if oid_tags is None:
oid_tags = LLSet()
self.oid_tags[oid] = oid_tags
if htag not in oid_tags:
oid_tags.insert(htag)
# culculate weight
weight = self.tag_weight.get(htag)
if weight is not None:
key = (weight, htag)
if key in self.weights:
self.weights.remove(key)
weight = float(len(oids))
self.tag_weight[htag] = weight
self.weights.insert((weight, htag))
def remove(self, oid):
for tag in self.oid_tags.get(oid, ()):
# remove oid from tag -> oids reference
oids = self.tag_oids.get(tag)
if oid in oids:
oids.remove(oid)
oids_len = float(len(oids))
# reculculate weight
weight = self.tag_weight.get(tag)
if weight is not None:
key = (weight, tag)
if key in self.weights:
self.weights.remove(key)
if oids_len:
self.tag_weight[tag] = oids_len
self.weights.insert((oids_len, tag))
# remove tag
if not oids_len:
del self.tag_oids[tag]
del self.tagsmap[tag]
if oid in self.oid_tags:
del self.oid_tags[oid]
def getItems(self, tags):
oids = self.tag_oids
weights = self.tag_weight
weight, result = 0, LFBTree()
for tag in tags:
htag = self.getHash(tag)
if htag in oids:
weight, result = weightedUnion(
oids.get(htag), result, weights.get(htag), weight)
return IFBucket(result)
def getUniqueItems(self, tags):
oids = self.tag_oids
weights = self.tag_weight
weight, result = 1.0, None
for tag in tags:
htag = self.getHash(tag)
if htag in oids:
if result is None:
weight, result = weightedUnion(
oids.get(htag), LFBTree(), weights.get(htag), weight)
else:
weight, result = weightedIntersection(
result, oids.get(htag), weight, weights.get(htag))
return IFBucket(result)
def getTagCloud(self, reverse=False):
total = len(self.oid_tags)
if not total:
return
tags = self.tagsmap
data = self.weights.keys()
percent = total / 100.0
if reverse:
first = len(data)-1
for idx in xrange(first, -1, -1):
weight, htag = data[idx]
yield weight / percent, tags.get(htag)
else:
for weight, htag in data:
yield weight / percent, tags.get(htag)
def getItemsTagCloud(self, items):
oid_tags = self.oid_tags
tags = [oid_tags[oid] for oid in items if oid in oid_tags]
if tags:
tags = multiunion(tags)
else:
return
total = len(oid_tags)
data = self.weights.keys()
weights = self.tag_weight
percent = total / 100.0
for tag in tags:
yield weights[tag] / percent, self.tagsmap.get(tag)
def getFrequency(self, tags):
tagsmap = self.tagsmap
tag_weight = self.tag_weight
for tag in tags:
yield (tag, tag_weight.get(self.getHash(tag), 0))
def __nonzero__(self):
return bool(self.tag_oids)
def __contains__(self, tag):
return self.getHash(tag) in self.tagsmap
class BaseBlog(ContentContainer):
interface.implements(IBlog, IBlogPostListing)
pageSize = FieldProperty(IBlog["pageSize"])
def __init__(self, *args, **kw):
super(BaseBlog, self).__init__(*args, **kw)
self.dates = OOSet()
self.index = IFBTree()
def entries(self, index=None):
if index is None:
result = self.index
else:
_t, result = weightedIntersection(index, self.index)
return result.byValue(0)
def update(self, post):
uid = getUtility(IIntIds).getId(post)
if uid in self.index:
self.remove(post)
date = post.date.timetuple()
idx = time.mktime(date)
self.index[uid] = idx
# update in dates
yearName = str(date[0])
year = self.get(yearName)
if year is None:
year = Year()
event.notify(ObjectCreatedEvent(year))
self[yearName] = year
self.dates.insert(yearName)
year.update(date, uid, idx)
# update categories
self["category"].update(post, uid, idx)
# update tags info
self["tags"].update(post, uid)
def remove(self, post):
uid = getUtility(IIntIds).getId(post)
if uid not in self.index:
return
date = time.localtime(self.index[uid])
# remove from dates
yearName = str(date[0])
year = self.get(yearName)
if year is not None:
year.remove(date, uid)
if not len(year):
if yearName in self.dates:
self.dates.remove(yearName)
del self[yearName]
# remove from category
self["category"].remove(uid)
# remove tags info
self["tags"].remove(post, uid)
# remove from index
del self.index[uid]
