class QueryStatBucket(item.Item): """ Obsolete. Only present for schema compatibility. Do not use. """ type = attributes.text("the SQL query string") value = attributes.ieee754_double(default=0.0, doc='Total number of events for this time period') interval = attributes.text(doc='A time period, e.g. "quarter-hour" or "minute" or "day"') index = attributes.integer(doc='The position in the round-robin list for non-daily stats') time = attributes.timestamp(doc='When this bucket was last updated') attributes.compoundIndex(interval, type, index)
class KitchenSink(Item): """ An item with one of everything, more or less. """ t = text() i = integer() ts = timestamp() tl = textlist() d = ieee754_double() p1d = point1decimal() m = money()
# XXX This really should use in-database state, otherwise a restart in # the middle will muck things up. def go(): for msg in work: for f in filters: f.train(msg._spam, msg) yield None self.reclassify() return coiterate(go()) registerAttributeCopyingUpgrader(Filter, 1, 2) item.declareLegacyItem(Filter.typeName, 2, dict(installedOn = attributes.reference(), usePostiniScore = attributes.boolean(default=False,allowNone=False), postiniThreshhold = attributes.ieee754_double(default=0.03))) def _filter2to3(old): """ add dependencies as attributes, remove installedOn """ filter = old.upgradeVersion(old.typeName, 2, 3) s = old.store filter.usePostiniScore = old.usePostiniScore filter.postiniThreshhold = old.postiniThreshhold filter.messageSource = s.findOrCreate(MessageSource) filter.tiSource = s.findOrCreate(_TrainingInstructionSource) return filter
class Number(Item): typeName = 'test_number' schemaVersion = 1 value = ieee754_double()
def test_float(self): self._test_typeFor(attributes.ieee754_double(), amp.Float)
class Exit(item.Item): """ An L{Exit} is an oriented pathway between two L{Thing}s which each represent a room. """ implements(iimaginary.INameable, iimaginary.IExit) fromLocation = attributes.reference( doc=""" Where this exit leads from. """, allowNone=False, whenDeleted=attributes.reference.CASCADE, reftype=Thing) toLocation = attributes.reference( doc=""" Where this exit leads to. """, allowNone=False, whenDeleted=attributes.reference.CASCADE, reftype=Thing) name = attributes.text(doc=""" What this exit is called/which direction it is in. """, allowNone=False) sibling = attributes.reference(doc=""" The reverse exit object, if one exists. """) distance = attributes.ieee754_double( doc=""" How far, in meters, does a user have to travel to traverse this exit? """, allowNone=False, default=1.0) def knownTo(self, observer, name): """ Implement L{iimaginary.INameable.knownTo} to identify this L{Exit} as its C{name} attribute. """ return name == self.name def traverse(self, thing): """ Implement L{iimaginary.IExit} to move the given L{Thing} to this L{Exit}'s C{toLocation}. """ if self.sibling is not None: arriveDirection = self.sibling.name else: arriveDirection = OPPOSITE_DIRECTIONS.get(self.name) thing.moveTo( self.toLocation, arrivalEventFactory=lambda player: events.MovementArrivalEvent( thing=thing, origin=None, direction=arriveDirection)) # XXX This really needs to be renamed now that links are a thing. @classmethod def link(cls, a, b, forwardName, backwardName=None, distance=1.0): """ Create two L{Exit}s connecting two rooms. @param a: The first room. @type a: L{Thing} @param b: The second room. @type b: L{Thing} @param forwardName: The name of the link going from C{a} to C{b}. For example, u'east'. @type forwardName: L{unicode} @param backwardName: the name of the link going from C{b} to C{a}. For example, u'west'. If not provided or L{None}, this will be computed based on L{OPPOSITE_DIRECTIONS}. @type backwardName: L{unicode} """ if backwardName is None: backwardName = OPPOSITE_DIRECTIONS[forwardName] forward = cls(store=a.store, fromLocation=a, toLocation=b, name=forwardName, distance=distance) backward = cls(store=b.store, fromLocation=b, toLocation=a, name=backwardName, distance=distance) forward.sibling = backward backward.sibling = forward def destroy(self): if self.sibling is not None: self.sibling.deleteFromStore() self.deleteFromStore() @remembered def exitIdea(self): """ This property is the L{Idea} representing this L{Exit}; this is a fairly simple L{Idea} that will link only to the L{Exit.toLocation} pointed to by this L{Exit}, with a distance annotation indicating the distance traversed to go through this L{Exit}. """ x = Idea(self) x.linkers.append(self) return x def links(self): """ Generate a link to the location that this exit points at. @return: an iterator which yields a single L{Link}, annotated with a L{Vector} that indicates a distance of 1.0 (a temporary measure, since L{Exit}s don't have distances yet) and a direction of this exit's C{name}. """ l = Link(self.exitIdea, self.toLocation.idea) l.annotate([Vector(self.distance, self.name), # We annotate this link with ourselves because the 'Named' # retriever will use the last link in the path to determine # if an object has any aliases. We want this direction # name to be an alias for the room itself as well as the # exit, so we want to annotate the link with an INameable. # This also has an effect of annotating the link with an # IExit, and possibly one day an IItem as well (if such a # thing ever comes to exist), so perhaps we eventually want # a wrapper which elides all references here except # INameable since that's what we want. proxyForInterface # perhaps? However, for the moment, the extra annotations # do no harm, so we'll leave them there. self]) yield l def shouldEvenAttemptTraversalFrom(self, where, observer): """ Normal (i.e. room-to-room) L{Exit}s appear traversable to observers located inside the location that they lead away from. """ return (self.fromLocation is where)
class Filter(item.Item): """ Aggregates message classification tools and calls appropriate methods on Message objects according to their results. Items will power this up for L{iquotient.IHamFilter} to participate in the spam/ham classification process. Only one Filter is currently supported. Future versions may expand on this and allow multiple filters to contribute to the final decision. These items will also receive training feedback from the user, though they may choose to disregard it if they are not trainable. C{Filter} can also be configured to just look at Postini headers and make a determination based on them. """ schemaVersion = 4 usePostiniScore = attributes.boolean(doc=""" Indicate whether or not to classify based on Postini headers. """, default=False, allowNone=False) postiniThreshhold = attributes.ieee754_double(doc=""" A C{float} between 0 and 100 indicating at what Postini level messages are considered spam. """, default=0.03) _filters = attributes.inmemory() messageSource = dependsOn(MessageSource) tiSource = dependsOn(_TrainingInstructionSource) def installed(self): self.messageSource.addReliableListener(self, style=iaxiom.REMOTE) self.tiSource.addReliableListener(self, style=iaxiom.REMOTE) def processItem(self, item): assert isinstance(item, (_TrainingInstruction, Message)) if isinstance(item, _TrainingInstruction): self._train(item) else: self._classify(item) def _filters(self): return list(self.store.powerupsFor(iquotient.IHamFilter)) def _train(self, instruction): for f in self._filters(): f.train(instruction.spam, instruction.message) instruction.deleteFromStore() def _parsePostiniHeader(self, s): """ Postini spam headers look like this: X-pstn-levels: (S:99.9000 R:95.9108 P:91.9078 M:100.0000 C:96.6797 ) X-pstn-levels: (S: 0.0901 R:95.9108 P:95.9108 M:99.5542 C:79.5348 ) X-pstn-levels: (S:99.9000/99.9000 R:95.9108 P:91.9078 M:100.0000 C:96.6797 ) S means "spam level". Smaller is spammier. Value after the slash is the 'blatant spam blocking' score, ignored by this code. R means "Racially insensitive spam score" P means "Sexually explicit (pornography) spam score" M means "Make-money-fast (MMF) spam score" C means "Commercial or 'special offer' spam score" @return: A mapping from 'R', 'P', 'M', 'C', and 'S' to the values from the header, or None if the header could not be parsed. """ s = s.strip() if s[:1] == '(' and s[-1:] == ')': parts = filter(None, s[1:-1].replace(':', ' ').split()) if '/' in parts[1]: parts[1], _ = parts[1].split('/') return dict( (parts[i], Decimal(parts[i + 1])) for i in (0, 2, 4, 6, 8)) return None def _classify(self, msg): if not msg.shouldBeClassified: log.msg( "Skipping classification of message already user-specified as " + msg.spamStatus()) return # Allow Postini to override anything we might determine, if the user # has indicated that is desirable. if self.usePostiniScore: try: postiniHeader = msg.impl.getHeader(u'X-pstn-levels') except NoSuchHeader: pass else: postiniLevels = self._parsePostiniHeader(postiniHeader) if postiniLevels is not None: postiniScore = postiniLevels['S'] if float(postiniScore) < self.postiniThreshhold: msg.classifySpam() log.msg("Postini classified message as spam") else: msg.classifyClean() log.msg("Postini classified message as clean") return _filters = self._filters() if len(_filters) > 1: raise NotImplementedError( "multiple spam filters not yet supported") if not _filters: msg.classifyClean() log.msg("Message classified as clean due to an absence of filters") return isSpam, score = _filters[0].classify(msg) if isSpam: msg.classifySpam() else: msg.classifyClean() log.msg("spam batch processor scored message at %0.2f: %r" % (score, isSpam)) def reclassify(self): """ Forget whatever progress has been made in processing messages and start over. This should only be called in the batch process. """ self.messageSource.removeReliableListener(self) self.messageSource.addReliableListener(self, style=iaxiom.REMOTE) def retrain(self): """ Force all L{iquotient.IHamFilter}s to forget their trained state, then retrain them based on L{exmess.Message}s with C{trained} set to C{True}, then reclassify all messages. This should only be called in the batch process. """ filters = list(self.store.powerupsFor(iquotient.IHamFilter)) for f in filters: f.forgetTraining() sq = MailboxSelector(self.store) sq.setLimit(5000) sq.refineByStatus(TRAINED_STATUS) work = iter(list(sq)) # XXX This really should use in-database state, otherwise a restart in # the middle will muck things up. def go(): for msg in work: for f in filters: f.train(msg._spam, msg) yield None self.reclassify() return coiterate(go())
for msg in work: for f in filters: f.train(msg._spam, msg) yield None self.reclassify() return coiterate(go()) registerAttributeCopyingUpgrader(Filter, 1, 2) item.declareLegacyItem( Filter.typeName, 2, dict(installedOn=attributes.reference(), usePostiniScore=attributes.boolean(default=False, allowNone=False), postiniThreshhold=attributes.ieee754_double(default=0.03))) def _filter2to3(old): """ add dependencies as attributes, remove installedOn """ filter = old.upgradeVersion(old.typeName, 2, 3) s = old.store filter.usePostiniScore = old.usePostiniScore filter.postiniThreshhold = old.postiniThreshhold filter.messageSource = s.findOrCreate(MessageSource) filter.tiSource = s.findOrCreate(_TrainingInstructionSource) return filter