def test_retrain(self):
"""
Verify that retraining a few messages will tell our new test filter to
learn about a bunch of new messages.
"""
f = Filter(store=self.store)
installOn(f, self.store)
tf = TestFilter(store=self.store, test=self)
installOn(tf, self.store)
COUNT = 10
for j in range(2):
for x in range(COUNT):
msg = Message.createIncoming(
self.store, DummyMessageImplementation(store=self.store),
u'test://retrain')
_spamState(msg, (x % 2), j)
# This isn't quite correct. We're relying on the fact that the batch
# processor is supposed to run in a subprocess (which isn't going) so
# the callback is only going to be triggered for our set of messages
# during the test. Bleah.
def _(ign):
self.assertEquals(tf.trainCount, COUNT)
return f.retrain().addCallback(_)
def test_retrain(self):
"""
Verify that retraining a few messages will tell our new test filter to
learn about a bunch of new messages.
"""
f = Filter(store=self.store)
installOn(f, self.store)
tf = TestFilter(store=self.store, test=self)
installOn(tf, self.store)
COUNT = 10
for j in range(2):
for x in range(COUNT):
msg = Message.createIncoming(
self.store, DummyMessageImplementation(store=self.store), u"test://retrain"
)
_spamState(msg, (x % 2), j)
# This isn't quite correct. We're relying on the fact that the batch
# processor is supposed to run in a subprocess (which isn't going) so
# the callback is only going to be triggered for our set of messages
# during the test. Bleah.
def _(ign):
self.assertEquals(tf.trainCount, COUNT)
return f.retrain().addCallback(_)
def test_postiniHeaderParsing(self):
"""
Test that Postini's spam levels header can be parsed and structured
data extracted from it.
"""
f = Filter()
self.assertEquals(
f._parsePostiniHeader(
'(S:99.9000 R:95.9108 P:91.9078 M:100.0000 C:96.6797 )'), {
'S': Decimal('99.9'),
'R': Decimal('95.9108'),
'P': Decimal('91.9078'),
'M': Decimal('100'),
'C': Decimal('96.6797')
})
self.assertEquals(
f._parsePostiniHeader(
'(S: 0.0901 R:95.9108 P:95.9108 M:99.5542 C:79.5348 )'), {
'S': Decimal('.0901'),
'R': Decimal('95.9108'),
'P': Decimal('95.9108'),
'M': Decimal('99.5542'),
'C': Decimal('79.5348')
})
self.assertEquals(
f._parsePostiniHeader(
'(S:99.90000/99.90000 R:95.9108 P:95.9108 M:97.0282 C:98.6951 )'
), {
'S': Decimal('99.9'),
'R': Decimal('95.9108'),
'P': Decimal('95.9108'),
'M': Decimal('97.0282'),
'C': Decimal('98.6951')
})
def test_processTrainingInstructions(self):
"""
When a user trains a message, a _TrainingInstruction item gets
created to signal the batch processor to do the training. Make
that gets run OK.
"""
f = Filter(store=self.store, usePostiniScore=True, postiniThreshhold=1.0)
ti = _TrainingInstruction(store=self.store, spam=True)
f.processItem(ti)
def test_postiniWithoutHeaderHamFiltering(self):
"""
Check that when postini filtering is enabled but a message has no
postini header then the other filter is consulted.
"""
msg = Message.createIncoming(self.store, Part(), u"test://postiniWithoutHeaderHamFiltering")
f = Filter(store=self.store, usePostiniScore=True, postiniThreshhold=1.0)
installOn(TestFilter(store=self.store, result=False), self.store)
f.processItem(msg)
self.assertNotIn(SPAM_STATUS, list(msg.iterStatuses()))
def test_disablePostiniHamFiltering(self):
"""
Test that if C{usePostiniScore} is False the header is ignored and
another filter is consulted.
"""
msg = self._messageWithPostiniHeader(u"(S: 0.9000 R:95.9108 P:91.9078 M:100.0000 C:96.6797 )")
f = Filter(store=self.store, usePostiniScore=False, postiniThreshhold=None)
installOn(TestFilter(store=self.store, result=False), self.store)
f.processItem(msg)
self.failIf(msg.hasStatus(SPAM_STATUS))
def test_postiniHeaderSpamFiltering(self):
"""
Test that if a message has a low enough spam level in a Postini
C{X-pstn-levels} header and the Filter has been configured to use it,
it is classified as spam.
"""
msg = self._messageWithPostiniHeader(u"(S: 0.9000 R:95.9108 P:91.9078 M:100.0000 C:96.6797 )")
f = Filter(usePostiniScore=True, postiniThreshhold=1.0)
f.processItem(msg)
self.failUnless(msg.hasStatus(SPAM_STATUS))
def test_postiniHeaderSpamFiltering(self):
"""
Test that if a message has a low enough spam level in a Postini
C{X-pstn-levels} header and the Filter has been configured to use it,
it is classified as spam.
"""
msg = self._messageWithPostiniHeader(
u'(S: 0.9000 R:95.9108 P:91.9078 M:100.0000 C:96.6797 )')
f = Filter(usePostiniScore=True, postiniThreshhold=1.0)
f.processItem(msg)
self.failUnless(msg.hasStatus(SPAM_STATUS))
def test_postiniRespectsTraining(self):
"""
If a user trains a message as ham or spam, the postini code should not
clobber that value, even though postini is not really trainable itself.
"""
msg = self._messageWithPostiniHeader(u"(S:99.9000 R:95.9108 P:91.9078 M:100.0000 C:96.6797 )")
msg.trainSpam()
f = Filter(store=self.store, usePostiniScore=True, postiniThreshhold=1.0)
f.processItem(msg)
self.failUnless(msg.hasStatus(SPAM_STATUS))
self.failIf(msg.shouldBeClassified)
def test_processTrainingInstructions(self):
"""
When a user trains a message, a _TrainingInstruction item gets
created to signal the batch processor to do the training. Make
that gets run OK.
"""
f = Filter(store=self.store,
usePostiniScore=True,
postiniThreshhold=1.0)
ti = _TrainingInstruction(store=self.store, spam=True)
f.processItem(ti)
def test_disablePostiniHamFiltering(self):
"""
Test that if C{usePostiniScore} is False the header is ignored and
another filter is consulted.
"""
msg = self._messageWithPostiniHeader(
u'(S: 0.9000 R:95.9108 P:91.9078 M:100.0000 C:96.6797 )')
f = Filter(store=self.store,
usePostiniScore=False,
postiniThreshhold=None)
installOn(TestFilter(store=self.store, result=False), self.store)
f.processItem(msg)
self.failIf(msg.hasStatus(SPAM_STATUS))
def test_postiniWithoutHeaderHamFiltering(self):
"""
Check that when postini filtering is enabled but a message has no
postini header then the other filter is consulted.
"""
msg = Message.createIncoming(
self.store, Part(), u'test://postiniWithoutHeaderHamFiltering')
f = Filter(store=self.store,
usePostiniScore=True,
postiniThreshhold=1.0)
installOn(TestFilter(store=self.store, result=False), self.store)
f.processItem(msg)
self.assertNotIn(SPAM_STATUS, list(msg.iterStatuses()))
def test_postiniRespectsTraining(self):
"""
If a user trains a message as ham or spam, the postini code should not
clobber that value, even though postini is not really trainable itself.
"""
msg = self._messageWithPostiniHeader(
u'(S:99.9000 R:95.9108 P:91.9078 M:100.0000 C:96.6797 )')
msg.trainSpam()
f = Filter(store=self.store,
usePostiniScore=True,
postiniThreshhold=1.0)
f.processItem(msg)
self.failUnless(msg.hasStatus(SPAM_STATUS))
self.failIf(msg.shouldBeClassified)
def test_postiniHeaderWithWhitespace(self):
"""
Test that a Postini header with leading or trailing whitespace can
also be parsed correctly. Headers like this probably shouldn't ever
show up, but investigation of old messages indicates they seem to
sometimes.
"""
f = Filter()
self.assertEquals(
f._parsePostiniHeader(" (S:99.9000 R:95.9108 P:91.9078 M:100.0000 C:96.6797 ) \r"),
{
"S": Decimal("99.9"),
"R": Decimal("95.9108"),
"P": Decimal("91.9078"),
"M": Decimal("100"),
"C": Decimal("96.6797"),
},
)
def test_postiniHeaderWithWhitespace(self):
"""
Test that a Postini header with leading or trailing whitespace can
also be parsed correctly. Headers like this probably shouldn't ever
show up, but investigation of old messages indicates they seem to
sometimes.
"""
f = Filter()
self.assertEquals(
f._parsePostiniHeader(
' (S:99.9000 R:95.9108 P:91.9078 M:100.0000 C:96.6797 ) \r'),
{
'S': Decimal('99.9'),
'R': Decimal('95.9108'),
'P': Decimal('91.9078'),
'M': Decimal('100'),
'C': Decimal('96.6797')
})
def test_postiniHeaderParsing(self):
"""
Test that Postini's spam levels header can be parsed and structured
data extracted from it.
"""
f = Filter()
self.assertEquals(
f._parsePostiniHeader("(S:99.9000 R:95.9108 P:91.9078 M:100.0000 C:96.6797 )"),
{
"S": Decimal("99.9"),
"R": Decimal("95.9108"),
"P": Decimal("91.9078"),
"M": Decimal("100"),
"C": Decimal("96.6797"),
},
)
self.assertEquals(
f._parsePostiniHeader("(S: 0.0901 R:95.9108 P:95.9108 M:99.5542 C:79.5348 )"),
{
"S": Decimal(".0901"),
"R": Decimal("95.9108"),
"P": Decimal("95.9108"),
"M": Decimal("99.5542"),
"C": Decimal("79.5348"),
},
)
self.assertEquals(
f._parsePostiniHeader("(S:99.90000/99.90000 R:95.9108 P:95.9108 M:97.0282 C:98.6951 )"),
{
"S": Decimal("99.9"),
"R": Decimal("95.9108"),
"P": Decimal("95.9108"),
"M": Decimal("97.0282"),
"C": Decimal("98.6951"),
},
)
def createDatabase(s):
Scheduler(store=s).installOn(s)
MessageSource(store=s)
Filter(store=s).installOn(s)
def setUp(self):
self.store = Store()
self.filter = Filter(store=self.store)
self.widget = HamFilterFragment(self.filter)
self.widget.setFragmentParent(self)
return self.widget
def createDatabase(store):
installOn(
Filter(store=store, usePostiniScore=USE_POSTINI_SCORE,
postiniThreshhold=POSTINI_THRESHHOLD),
store)
def createDatabase(s):
Filter(store=s)
