class QueryStatBucket(item.Item):
"""
Obsolete. Only present for schema compatibility. Do not use.
"""
type = attributes.text("the SQL query string")
value = attributes.ieee754_double(default=0.0, doc='Total number of events for this time period')
interval = attributes.text(doc='A time period, e.g. "quarter-hour" or "minute" or "day"')
index = attributes.integer(doc='The position in the round-robin list for non-daily stats')
time = attributes.timestamp(doc='When this bucket was last updated')
attributes.compoundIndex(interval, type, index)
class KitchenSink(Item):
"""
An item with one of everything, more or less.
"""
t = text()
i = integer()
ts = timestamp()
tl = textlist()
d = ieee754_double()
p1d = point1decimal()
m = money()
# XXX This really should use in-database state, otherwise a restart in
# the middle will muck things up.
def go():
for msg in work:
for f in filters:
f.train(msg._spam, msg)
yield None
self.reclassify()
return coiterate(go())
registerAttributeCopyingUpgrader(Filter, 1, 2)
item.declareLegacyItem(Filter.typeName, 2,
dict(installedOn = attributes.reference(),
usePostiniScore = attributes.boolean(default=False,allowNone=False),
postiniThreshhold = attributes.ieee754_double(default=0.03)))
def _filter2to3(old):
"""
add dependencies as attributes, remove installedOn
"""
filter = old.upgradeVersion(old.typeName, 2, 3)
s = old.store
filter.usePostiniScore = old.usePostiniScore
filter.postiniThreshhold = old.postiniThreshhold
filter.messageSource = s.findOrCreate(MessageSource)
filter.tiSource = s.findOrCreate(_TrainingInstructionSource)
return filter
class Number(Item):
typeName = 'test_number'
schemaVersion = 1
value = ieee754_double()
def test_float(self):
self._test_typeFor(attributes.ieee754_double(), amp.Float)
class Exit(item.Item):
"""
An L{Exit} is an oriented pathway between two L{Thing}s which each
represent a room.
"""
implements(iimaginary.INameable, iimaginary.IExit)
fromLocation = attributes.reference(
doc="""
Where this exit leads from.
""", allowNone=False,
whenDeleted=attributes.reference.CASCADE, reftype=Thing)
toLocation = attributes.reference(
doc="""
Where this exit leads to.
""", allowNone=False,
whenDeleted=attributes.reference.CASCADE, reftype=Thing)
name = attributes.text(doc="""
What this exit is called/which direction it is in.
""", allowNone=False)
sibling = attributes.reference(doc="""
The reverse exit object, if one exists.
""")
distance = attributes.ieee754_double(
doc="""
How far, in meters, does a user have to travel to traverse this exit?
""", allowNone=False, default=1.0)
def knownTo(self, observer, name):
"""
Implement L{iimaginary.INameable.knownTo} to identify this L{Exit} as
its C{name} attribute.
"""
return name == self.name
def traverse(self, thing):
"""
Implement L{iimaginary.IExit} to move the given L{Thing} to this
L{Exit}'s C{toLocation}.
"""
if self.sibling is not None:
arriveDirection = self.sibling.name
else:
arriveDirection = OPPOSITE_DIRECTIONS.get(self.name)
thing.moveTo(
self.toLocation,
arrivalEventFactory=lambda player: events.MovementArrivalEvent(
thing=thing,
origin=None,
direction=arriveDirection))
# XXX This really needs to be renamed now that links are a thing.
@classmethod
def link(cls, a, b, forwardName, backwardName=None, distance=1.0):
"""
Create two L{Exit}s connecting two rooms.
@param a: The first room.
@type a: L{Thing}
@param b: The second room.
@type b: L{Thing}
@param forwardName: The name of the link going from C{a} to C{b}. For
example, u'east'.
@type forwardName: L{unicode}
@param backwardName: the name of the link going from C{b} to C{a}. For
example, u'west'. If not provided or L{None}, this will be
computed based on L{OPPOSITE_DIRECTIONS}.
@type backwardName: L{unicode}
"""
if backwardName is None:
backwardName = OPPOSITE_DIRECTIONS[forwardName]
forward = cls(store=a.store, fromLocation=a, toLocation=b,
name=forwardName, distance=distance)
backward = cls(store=b.store, fromLocation=b, toLocation=a,
name=backwardName, distance=distance)
forward.sibling = backward
backward.sibling = forward
def destroy(self):
if self.sibling is not None:
self.sibling.deleteFromStore()
self.deleteFromStore()
@remembered
def exitIdea(self):
"""
This property is the L{Idea} representing this L{Exit}; this is a
fairly simple L{Idea} that will link only to the L{Exit.toLocation}
pointed to by this L{Exit}, with a distance annotation indicating the
distance traversed to go through this L{Exit}.
"""
x = Idea(self)
x.linkers.append(self)
return x
def links(self):
"""
Generate a link to the location that this exit points at.
@return: an iterator which yields a single L{Link}, annotated with a
L{Vector} that indicates a distance of 1.0 (a temporary measure,
since L{Exit}s don't have distances yet) and a direction of this
exit's C{name}.
"""
l = Link(self.exitIdea, self.toLocation.idea)
l.annotate([Vector(self.distance, self.name),
# We annotate this link with ourselves because the 'Named'
# retriever will use the last link in the path to determine
# if an object has any aliases. We want this direction
# name to be an alias for the room itself as well as the
# exit, so we want to annotate the link with an INameable.
# This also has an effect of annotating the link with an
# IExit, and possibly one day an IItem as well (if such a
# thing ever comes to exist), so perhaps we eventually want
# a wrapper which elides all references here except
# INameable since that's what we want. proxyForInterface
# perhaps? However, for the moment, the extra annotations
# do no harm, so we'll leave them there.
self])
yield l
def shouldEvenAttemptTraversalFrom(self, where, observer):
"""
Normal (i.e. room-to-room) L{Exit}s appear traversable to observers
located inside the location that they lead away from.
"""
return (self.fromLocation is where)
class Filter(item.Item):
"""
Aggregates message classification tools and calls appropriate methods on
Message objects according to their results.
Items will power this up for L{iquotient.IHamFilter} to participate in the
spam/ham classification process. Only one Filter is currently supported.
Future versions may expand on this and allow multiple filters to contribute
to the final decision. These items will also receive training feedback
from the user, though they may choose to disregard it if they are not
trainable.
C{Filter} can also be configured to just look at Postini headers and make a
determination based on them.
"""
schemaVersion = 4
usePostiniScore = attributes.boolean(doc="""
Indicate whether or not to classify based on Postini headers.
""",
default=False,
allowNone=False)
postiniThreshhold = attributes.ieee754_double(doc="""
A C{float} between 0 and 100 indicating at what Postini level messages are
considered spam.
""",
default=0.03)
_filters = attributes.inmemory()
messageSource = dependsOn(MessageSource)
tiSource = dependsOn(_TrainingInstructionSource)
def installed(self):
self.messageSource.addReliableListener(self, style=iaxiom.REMOTE)
self.tiSource.addReliableListener(self, style=iaxiom.REMOTE)
def processItem(self, item):
assert isinstance(item, (_TrainingInstruction, Message))
if isinstance(item, _TrainingInstruction):
self._train(item)
else:
self._classify(item)
def _filters(self):
return list(self.store.powerupsFor(iquotient.IHamFilter))
def _train(self, instruction):
for f in self._filters():
f.train(instruction.spam, instruction.message)
instruction.deleteFromStore()
def _parsePostiniHeader(self, s):
"""
Postini spam headers look like this:
X-pstn-levels: (S:99.9000 R:95.9108 P:91.9078 M:100.0000 C:96.6797 )
X-pstn-levels: (S: 0.0901 R:95.9108 P:95.9108 M:99.5542 C:79.5348 )
X-pstn-levels: (S:99.9000/99.9000 R:95.9108 P:91.9078 M:100.0000 C:96.6797 )
S means "spam level". Smaller is spammier. Value after the slash is
the 'blatant spam blocking' score, ignored by this code.
R means "Racially insensitive spam score"
P means "Sexually explicit (pornography) spam score"
M means "Make-money-fast (MMF) spam score"
C means "Commercial or 'special offer' spam score"
@return: A mapping from 'R', 'P', 'M', 'C', and 'S' to the values
from the header, or None if the header could not be parsed.
"""
s = s.strip()
if s[:1] == '(' and s[-1:] == ')':
parts = filter(None, s[1:-1].replace(':', ' ').split())
if '/' in parts[1]:
parts[1], _ = parts[1].split('/')
return dict(
(parts[i], Decimal(parts[i + 1])) for i in (0, 2, 4, 6, 8))
return None
def _classify(self, msg):
if not msg.shouldBeClassified:
log.msg(
"Skipping classification of message already user-specified as "
+ msg.spamStatus())
return
# Allow Postini to override anything we might determine, if the user
# has indicated that is desirable.
if self.usePostiniScore:
try:
postiniHeader = msg.impl.getHeader(u'X-pstn-levels')
except NoSuchHeader:
pass
else:
postiniLevels = self._parsePostiniHeader(postiniHeader)
if postiniLevels is not None:
postiniScore = postiniLevels['S']
if float(postiniScore) < self.postiniThreshhold:
msg.classifySpam()
log.msg("Postini classified message as spam")
else:
msg.classifyClean()
log.msg("Postini classified message as clean")
return
_filters = self._filters()
if len(_filters) > 1:
raise NotImplementedError(
"multiple spam filters not yet supported")
if not _filters:
msg.classifyClean()
log.msg("Message classified as clean due to an absence of filters")
return
isSpam, score = _filters[0].classify(msg)
if isSpam:
msg.classifySpam()
else:
msg.classifyClean()
log.msg("spam batch processor scored message at %0.2f: %r" %
(score, isSpam))
def reclassify(self):
"""
Forget whatever progress has been made in processing messages and start
over.
This should only be called in the batch process.
"""
self.messageSource.removeReliableListener(self)
self.messageSource.addReliableListener(self, style=iaxiom.REMOTE)
def retrain(self):
"""
Force all L{iquotient.IHamFilter}s to forget their trained state,
then retrain them based on L{exmess.Message}s with C{trained} set to
C{True}, then reclassify all messages.
This should only be called in the batch process.
"""
filters = list(self.store.powerupsFor(iquotient.IHamFilter))
for f in filters:
f.forgetTraining()
sq = MailboxSelector(self.store)
sq.setLimit(5000)
sq.refineByStatus(TRAINED_STATUS)
work = iter(list(sq))
# XXX This really should use in-database state, otherwise a restart in
# the middle will muck things up.
def go():
for msg in work:
for f in filters:
f.train(msg._spam, msg)
yield None
self.reclassify()
return coiterate(go())
for msg in work:
for f in filters:
f.train(msg._spam, msg)
yield None
self.reclassify()
return coiterate(go())
registerAttributeCopyingUpgrader(Filter, 1, 2)
item.declareLegacyItem(
Filter.typeName, 2,
dict(installedOn=attributes.reference(),
usePostiniScore=attributes.boolean(default=False, allowNone=False),
postiniThreshhold=attributes.ieee754_double(default=0.03)))
def _filter2to3(old):
"""
add dependencies as attributes, remove installedOn
"""
filter = old.upgradeVersion(old.typeName, 2, 3)
s = old.store
filter.usePostiniScore = old.usePostiniScore
filter.postiniThreshhold = old.postiniThreshhold
filter.messageSource = s.findOrCreate(MessageSource)
filter.tiSource = s.findOrCreate(_TrainingInstructionSource)
return filter
