def marshal(obj, keys=None, marshallerName='', objs=None): """ Convert an object to a dictionary. keys is an optional list of keys to include in the returned dictionary. if keys is None then all public attributes are returned. marshallerName is an optional marshalling adapter name. if it is an empty string then the default marshaller will be used. """ #to prevent recursing back over something twice, keep track of seen objs if objs is None: objs = OOSet() # obj is itself marshallable, so make a marshaller and marshal away if IMarshallable.providedBy(obj): marshaller = component.getAdapter(obj, IMarshaller, marshallerName) verify.verifyObject(IMarshaller, marshaller) if IInfo.providedBy(obj): key = (obj._object._p_oid, obj.__class__) if key in objs: raise AlreadySeenException() else: objs.insert(key) try: return marshal(marshaller.marshal(keys), keys, marshallerName, objs) except AlreadySeenException: pass finally: objs.remove(key) else: return marshal(marshaller.marshal(keys), keys, marshallerName, objs) # obj is a dict, so marshal its values recursively # Zuul.marshal({'foo':1, 'bar':2}) if isinstance(obj, dict): marshalled_dict = {} for k in obj: try: marshalled_dict[k] = marshal(obj[k], keys, marshallerName, objs) except AlreadySeenException: pass return marshalled_dict # obj is a non-string iterable, so marshal its members recursively # Zuul.marshal(set([o1, o2])) elif hasattr(obj, '__iter__'): marshalled_list = [] for o in obj: try: marshalled_list.append(marshal(o, keys, marshallerName, objs)) except AlreadySeenException: pass return marshalled_list elif isinstance(obj, DateTime ): return str(obj) # Nothing matched, so it's a string or number or other unmarshallable. else: return obj
def read(self): """Return messages added and removed from folder. Two sets of message objects are returned. The first set is messages that were added to the folder since the last read. The second set is the messages that were removed from the folder since the last read. The code assumes messages are added and removed but not edited. """ mbox = mailbox.UnixMailbox(open(self.path, "rb"), factory) self._stat() cur = OOSet() new = OOSet() while 1: msg = mbox.next() if msg is None: break msgid = msg["message-id"] cur.insert(msgid) if not self.messages.has_key(msgid): self.messages[msgid] = msg new.insert(msg) removed = difference(self.messages, cur) for msgid in removed.keys(): del self.messages[msgid] # XXX perhaps just return the OOBTree for removed? return new, OOSet(removed.values())
def marshal(obj, keys=None, marshallerName='', objs=None): """ Convert an object to a dictionary. keys is an optional list of keys to include in the returned dictionary. if keys is None then all public attributes are returned. marshallerName is an optional marshalling adapter name. if it is an empty string then the default marshaller will be used. """ #to prevent recursing back over something twice, keep track of seen objs if objs is None: objs = OOSet() # obj is itself marshallable, so make a marshaller and marshal away if IMarshallable.providedBy(obj): marshaller = component.getAdapter(obj, IMarshaller, marshallerName) verify.verifyObject(IMarshaller, marshaller) if IInfo.providedBy(obj): key = (getattr(obj._object, '_p_oid', id(obj)), obj.__class__) if key in objs: raise AlreadySeenException() else: objs.insert(key) try: return marshal(marshaller.marshal(keys), keys, marshallerName, objs) except AlreadySeenException: pass finally: objs.remove(key) else: return marshal(marshaller.marshal(keys), keys, marshallerName, objs) # obj is a dict, so marshal its values recursively # Zuul.marshal({'foo':1, 'bar':2}) if isinstance(obj, dict): marshalled_dict = {} for k in obj: try: marshalled_dict[k] = marshal(obj[k], keys, marshallerName, objs) except AlreadySeenException: pass return marshalled_dict # obj is a non-string iterable, so marshal its members recursively # Zuul.marshal(set([o1, o2])) elif hasattr(obj, '__iter__'): marshalled_list = [] for o in obj: try: marshalled_list.append(marshal(o, keys, marshallerName, objs)) except AlreadySeenException: pass return marshalled_list elif isinstance(obj, DateTime ): return str(obj) # Nothing matched, so it's a string or number or other unmarshallable. else: return obj
def read(self): """Return messages added and removed from folder. Two sets of message objects are returned. The first set is messages that were added to the folder since the last read. The second set is the messages that were removed from the folder since the last read. The code assumes messages are added and removed but not edited. """ mbox = mailbox.UnixMailbox(open(self.path, "rb"), factory) self._stat() cur = OOSet() new = OOSet() while 1: msg = mbox.next() if msg is None: break msgid = msg["message-id"] cur.insert(msgid) if not self.messages.has_key(msgid): self.messages[msgid] = msg new.insert(msg) removed = difference(self.messages, cur) for msgid in removed.keys(): del self.messages[msgid] # XXX perhaps just return the OOBTree for removed? return new, OOSet(removed.values())
class StarredTasks(grok.Annotation): grok.context(ifaces.IAccount) grok.implements(ifaces.IStarredTasks) def __init__(self): self._starred_tasks = OOSet() def getStarredTasks(self): return list(self._starred_tasks) def addStarredTask(self, task_intid): self._starred_tasks.insert(task_intid) def removeStarredTask(self, task_intid): if task_intid in self._starred_tasks: self._starred_tasks.remove(task_intid)
def setTags(self, pid, oid, tags): self.removeTags(pid, oid) if not tags: return t = [] for tag in tags: tag = tag.lower().strip() if tag: t.append(tag) if not t: return tags = t pdata = self.tags.get(pid) if pdata is None: pdata = IOBTree() self.tags[pid] = pdata oids = pdata.get(oid) if oids is None: oids = OOSet() pdata[oid] = oids oids.update(tags) # insert oid -> pid reference oids = self.oids.get(oid) if oids is None: oids = OOSet() self.oids[oid] = oids oids.insert(pid) #update tagging engine engine = self.getEngine(pid) try: engine.update(oid, tags) except: engine.clear() engine.update(oid, tags)
def sort(self, result): index = self.index values = self.index.documents_to_values seq = OOBTree() for key in result: idx = values.get(key, None) if idx is None: continue data = seq.get(idx, None) if data is None: data = OOSet() data.insert(key) seq[idx] = data result = [] for idx, data in seq.items(): result.extend(data) return result
def learn(self, rid, value): # Validate # ======== if not isinstance(value, basestring): raise TypeError("value is not a string: '%s'" % value) if not self.case_sensitive: value = value.lower() # Add to the (one:many) mapping, value to rids. # ============================================= if value in self.values: self.values[value].insert(rid) else: self.values[value] = IITreeSet([rid]) # Add to the (one:many) beginnings, middles, and endings indices. # ================================================================== # These are for the startswith, contains, and endswith searches, which # function basically like the is search, but we have to learn all # substrings of the string. substrings = OOSet() for i in range(len(value)): j = i + 1 # beginnings/startswith # ===================== part = value[:j] if part in self.beginnings: self.beginnings[part].insert(rid) else: self.beginnings[part] = IITreeSet([rid]) # middles/contains # ================ for k in range(len(value)-i): part = value[k:k+j] if part in self.middles: self.middles[part].insert(rid) else: self.middles[part] = IITreeSet([rid]) substrings.insert(part) # endings/endswith # ================ part = value[i:] if part in self.endings: self.endings[part].insert(rid) else: self.endings[part] = IITreeSet([rid]) # Add to the (one:one) mapping of rid to value. # ============================================= # This exists so we know where the rid is located when the time comes # to forget it. self.rids[rid] = substrings
class TaggingEngine(Persistent): interface.implements(ITaggingEngine) def __init__(self): self.clear() @property def tagsCount(self): return len(self.tag_oids) @property def itemsCount(self): return len(self.oid_tags) def clear(self): self.tagsmap = LOBTree() # index self.tag_oids = LOBTree() self.oid_tags = LOBTree() # tag weight self.weights = OOSet() self.tag_weight = LFBTree() def getHash(self, str): if not str: return 0 # empty res = hash(str) # NOTE: workaround for 64bit if struct.calcsize("P") * 8 == 64: res = ord(str[0]) << 7 for char in str: res = c_mul(1000003, res) ^ ord(char) res = res ^ len(str) if res == -1: res = -2 if res >= 2**31: res -= 2**32 return res def update(self, oid, tags): self.remove(oid) for tag in tags: htag = self.getHash(tag) self.tagsmap[htag] = tag # add oid -> tag oids = self.tag_oids.get(htag) if oids is None: oids = LFSet() self.tag_oids[htag] = oids if oid not in oids: oids.insert(oid) # add tag -> oid oid_tags = self.oid_tags.get(oid) if oid_tags is None: oid_tags = LLSet() self.oid_tags[oid] = oid_tags if htag not in oid_tags: oid_tags.insert(htag) # culculate weight weight = self.tag_weight.get(htag) if weight is not None: key = (weight, htag) if key in self.weights: self.weights.remove(key) weight = float(len(oids)) self.tag_weight[htag] = weight self.weights.insert((weight, htag)) def remove(self, oid): for tag in self.oid_tags.get(oid, ()): # remove oid from tag -> oids reference oids = self.tag_oids.get(tag) if oid in oids: oids.remove(oid) oids_len = float(len(oids)) # reculculate weight weight = self.tag_weight.get(tag) if weight is not None: key = (weight, tag) if key in self.weights: self.weights.remove(key) if oids_len: self.tag_weight[tag] = oids_len self.weights.insert((oids_len, tag)) # remove tag if not oids_len: del self.tag_oids[tag] del self.tagsmap[tag] if oid in self.oid_tags: del self.oid_tags[oid] def getItems(self, tags): oids = self.tag_oids weights = self.tag_weight weight, result = 0, LFBTree() for tag in tags: htag = self.getHash(tag) if htag in oids: weight, result = weightedUnion( oids.get(htag), result, weights.get(htag), weight) return IFBucket(result) def getUniqueItems(self, tags): oids = self.tag_oids weights = self.tag_weight weight, result = 1.0, None for tag in tags: htag = self.getHash(tag) if htag in oids: if result is None: weight, result = weightedUnion( oids.get(htag), LFBTree(), weights.get(htag), weight) else: weight, result = weightedIntersection( result, oids.get(htag), weight, weights.get(htag)) return IFBucket(result) def getTagCloud(self, reverse=False): total = len(self.oid_tags) if not total: return tags = self.tagsmap data = self.weights.keys() percent = total / 100.0 if reverse: first = len(data)-1 for idx in xrange(first, -1, -1): weight, htag = data[idx] yield weight / percent, tags.get(htag) else: for weight, htag in data: yield weight / percent, tags.get(htag) def getItemsTagCloud(self, items): oid_tags = self.oid_tags tags = [oid_tags[oid] for oid in items if oid in oid_tags] if tags: tags = multiunion(tags) else: return total = len(oid_tags) data = self.weights.keys() weights = self.tag_weight percent = total / 100.0 for tag in tags: yield weights[tag] / percent, self.tagsmap.get(tag) def getFrequency(self, tags): tagsmap = self.tagsmap tag_weight = self.tag_weight for tag in tags: yield (tag, tag_weight.get(self.getHash(tag), 0)) def __nonzero__(self): return bool(self.tag_oids) def __contains__(self, tag): return self.getHash(tag) in self.tagsmap
class BaseBlog(ContentContainer): interface.implements(IBlog, IBlogPostListing) pageSize = FieldProperty(IBlog["pageSize"]) def __init__(self, *args, **kw): super(BaseBlog, self).__init__(*args, **kw) self.dates = OOSet() self.index = IFBTree() def entries(self, index=None): if index is None: result = self.index else: _t, result = weightedIntersection(index, self.index) return result.byValue(0) def update(self, post): uid = getUtility(IIntIds).getId(post) if uid in self.index: self.remove(post) date = post.date.timetuple() idx = time.mktime(date) self.index[uid] = idx # update in dates yearName = str(date[0]) year = self.get(yearName) if year is None: year = Year() event.notify(ObjectCreatedEvent(year)) self[yearName] = year self.dates.insert(yearName) year.update(date, uid, idx) # update categories self["category"].update(post, uid, idx) # update tags info self["tags"].update(post, uid) def remove(self, post): uid = getUtility(IIntIds).getId(post) if uid not in self.index: return date = time.localtime(self.index[uid]) # remove from dates yearName = str(date[0]) year = self.get(yearName) if year is not None: year.remove(date, uid) if not len(year): if yearName in self.dates: self.dates.remove(yearName) del self[yearName] # remove from category self["category"].remove(uid) # remove tags info self["tags"].remove(post, uid) # remove from index del self.index[uid]