def main(args):
opts, args = getopt.getopt(args, "h")
for opt, arg in opts:
if opt == '-h':
usage()
return 0
msg = mboxutils.get_message(sys.stdin)
try:
del msg["X-Spambayes-Classification"]
except KeyError:
pass
for pair in args:
tag, db = pair.split('=', 1)
h = hammie.open(db, True, 'r')
score = h.score(msg)
if score >= Options.options["Categorization", "spam_cutoff"]:
msg["X-Spambayes-Classification"] = "%s; %.2f" % (tag, score)
break
else:
msg["X-Spambayes-Classification"] = "unsure"
sys.stdout.write(msg.as_string(unixfrom=(msg.get_unixfrom() is not None)))
return 0
def untrain_from_header(self, msg):
"""Untrain bayes based on X-Spambayes-Trained header.
msg can be a string, a file object, or a Message object.
If no such header is present, nothing happens.
If add_header is True, add a header with how it was trained (in
case we need to untrain later)
"""
msg = mboxutils.get_message(msg)
trained = msg.get(options["Headers", "trained_header_name"])
if not trained:
return
del msg[options["Headers", "trained_header_name"]]
if trained == options["Headers", "header_ham_string"]:
self.untrain_ham(msg)
elif trained == options["Headers", "header_spam_string"]:
self.untrain_spam(msg)
else:
raise ValueError('%s header value unrecognized'
% options["Headers", "trained_header_name"])
def process_mailbox(self, mailbox):
count, size = mailbox.stat()
log = Logger()
for i in range(1, count+1):
if (i-1) % 10 == 0:
print " == %d/%d ==" % (i, count)
# Kevin's code used -1, but -1 doesn't work for one of
# my POP accounts, while a million does.
# Don't use retr because that may mark the message as
# read (so says Kevin's code)
message_tuple = mailbox.top(i, 1000000)
text = "\n".join(message_tuple[1])
msg = mboxutils.get_message(text)
mi = MessageInfo(mailbox, i, msg, text)
_log_subject(mi, log)
for filter in self:
result = filter.process(mi, log)
if result:
log.accept(result)
break
else:
# don't know what to do with this so just
# keep it on the server
log.pass_test("unknown")
log.do_action(KEEP_IN_MAILBOX)
log.accept("unknown")
return log
def process_mailbox(self, mailbox):
count, size = mailbox.stat()
log = Logger()
for i in range(1, count + 1):
if (i - 1) % 10 == 0:
print " == %d/%d ==" % (i, count)
# Kevin's code used -1, but -1 doesn't work for one of
# my POP accounts, while a million does.
# Don't use retr because that may mark the message as
# read (so says Kevin's code)
message_tuple = mailbox.top(i, 1000000)
text = "\n".join(message_tuple[1])
msg = mboxutils.get_message(text)
mi = MessageInfo(mailbox, i, msg, text)
_log_subject(mi, log)
for filter in self:
result = filter.process(mi, log)
if result:
log.accept(result)
break
else:
# don't know what to do with this so just
# keep it on the server
log.pass_test("unknown")
log.do_action(KEEP_IN_MAILBOX)
log.accept("unknown")
return log
def _calc_response(self, switches, body):
switches = switches.split()
actions = []
opts, args = getopt.getopt(switches, 'fgstGS')
h = self.server.hammie
for opt, arg in opts:
if opt == '-f':
actions.append(h.filter)
elif opt == '-g':
actions.append(h.train_ham)
elif opt == '-s':
actions.append(h.train_spam)
elif opt == '-t':
actions.append(h.filter_train)
elif opt == '-G':
actions.append(h.untrain_ham)
elif opt == '-S':
actions.append(h.untrain_spam)
if actions == []:
actions = [h.filter]
from spambayes import mboxutils
msg = mboxutils.get_message(body)
for action in actions:
action(msg)
return mboxutils.as_string(msg, 1)
def main(args):
opts, args = getopt.getopt(args, "h")
for opt, arg in opts:
if opt == '-h':
help()
return 0
tagdb_list = []
msg = mboxutils.get_message(sys.stdin)
try:
del msg["X-Spambayes-Classification"]
except KeyError:
pass
for pair in args:
tag, db = pair.split('=', 1)
h = hammie.open(db, True, 'r')
score = h.score(msg)
if score >= Options.options["Categorization", "spam_cutoff"]:
msg["X-Spambayes-Classification"] = "%s; %.2f" % (tag, score)
break
else:
msg["X-Spambayes-Classification"] = "unsure"
sys.stdout.write(msg.as_string(unixfrom=(msg.get_unixfrom()
is not None)))
return 0
def _calc_response(self, switches, body):
switches = switches.split()
actions = []
opts, args = getopt.getopt(switches, 'fgstGS')
h = self.server.hammie
for opt, arg in opts:
if opt == '-f':
actions.append(h.filter)
elif opt == '-g':
actions.append(h.train_ham)
elif opt == '-s':
actions.append(h.train_spam)
elif opt == '-t':
actions.append(h.filter_train)
elif opt == '-G':
actions.append(h.untrain_ham)
elif opt == '-S':
actions.append(h.untrain_spam)
if actions == []:
actions = [h.filter]
from spambayes import mboxutils
msg = mboxutils.get_message(body)
for action in actions:
action(msg)
return mboxutils.as_string(msg, 1)
def main(argv):
opts, args = getopt.getopt(argv, "h", ["help"])
for opt, arg in opts:
if opt in ("-h", "--help"):
usage()
return
if options["pop3proxy", "cache_use_gzip"]:
messageFactory = GzipFileMessageFactory()
else:
messageFactory = FileMessageFactory()
sc = get_pathname_option("Storage", "spam_cache")
hc = get_pathname_option("Storage", "ham_cache")
spamCorpus = FileCorpus(messageFactory, sc)
hamCorpus = FileCorpus(messageFactory, hc)
allTrained = {}
for corpus, disposition in [(spamCorpus, 'Yes'), (hamCorpus, 'No')]:
for m in corpus:
message = mboxutils.get_message(m.getSubstance())
message._pop3CacheDisposition = disposition
allTrained[m.key()] = message
keys = allTrained.keys()
keys.sort()
limit = 70
if len(keys) < limit:
scale = 1
else:
scale = len(keys) // (limit//2)
count = successful = 0
successByCount = []
for key in keys:
message = allTrained[key]
disposition = message[options["Headers",
"classification_header_name"]]
if (message._pop3CacheDisposition == disposition):
successful += 1
count += 1
if count % scale == (scale-1):
successByCount.append(successful // scale)
size = count // scale
graph = [[" " for i in range(size+3)] for j in range(size)]
for c in range(size):
graph[c][1] = "|"
graph[c][c+3] = "."
graph[successByCount[c]][c+3] = "*"
graph.reverse()
print "\n Success of the classifier over time:\n"
print " . - Number of messages over time"
print " * - Number of correctly classified messages over time\n\n"
for row in range(size):
line = ''.join(graph[row])
if row == 0:
print line + " %d" % count
elif row == (count - successful) // scale:
print line + " %d" % successful
else:
print line
print " " + "_" * (size+2)
def process_mailbox(self, mailbox):
count, size = mailbox.stat()
log = Logger()
for i in range(1, count+1):
if (i-1) % 10 == 0:
print " == %d/%d ==" % (i, count)
message_tuple = mailbox.top(i, 1000000)
text = "\n".join(message_tuple[1])
msg = mboxutils.get_message(text)
mi = MessageInfo(mailbox, i, msg, text)
_log_subject(mi, log)
for filter in self:
result = filter.process(mi, log)
if result:
log.accept(result)
break
else:
log.pass_test("unknown")
log.do_action(KEEP_IN_MAILBOX)
log.accept("unknown")
return log
def untrain_from_header(self, msg):
"""Untrain bayes based on X-Spambayes-Trained header.
msg can be a string, a file object, or a Message object.
If no such header is present, nothing happens.
If add_header is True, add a header with how it was trained (in
case we need to untrain later)
"""
msg = mboxutils.get_message(msg)
trained = msg.get(options["Headers", "trained_header_name"])
if not trained:
return
del msg[options["Headers", "trained_header_name"]]
if trained == options["Headers", "header_ham_string"]:
self.untrain_ham(msg)
elif trained == options["Headers", "header_spam_string"]:
self.untrain_spam(msg)
else:
raise ValueError('%s header value unrecognized' %
options["Headers", "trained_header_name"])
def main(argv):
opts, args = getopt.getopt(argv, "h", ["help"])
for opt, arg in opts:
if opt in ("-h", "--help"):
usage()
return
# Create the corpuses and the factory that reads the messages.
if options["pop3proxy", "cache_use_gzip"]:
messageFactory = GzipFileMessageFactory()
else:
messageFactory = FileMessageFactory()
sc = get_pathname_option("Storage", "spam_cache")
hc = get_pathname_option("Storage", "ham_cache")
spamCorpus = FileCorpus(messageFactory, sc)
hamCorpus = FileCorpus(messageFactory, hc)
# Read in all the trained messages.
allTrained = {}
for corpus, disposition in [(spamCorpus, 'Yes'), (hamCorpus, 'No')]:
for m in corpus:
message = mboxutils.get_message(m.getSubstance())
message._pop3CacheDisposition = disposition
allTrained[m.key()] = message
# Sort the messages into the order they arrived, then work out a scaling
# factor for the graph - 'limit' is the widest it can be in characters.
keys = allTrained.keys()
keys.sort()
limit = 70
if len(keys) < limit:
scale = 1
else:
scale = len(keys) // (limit//2)
# Build the data - an array of cumulative success indexed by count.
count = successful = 0
successByCount = []
for key in keys:
message = allTrained[key]
disposition = message[options["Headers",
"classification_header_name"]]
if (message._pop3CacheDisposition == disposition):
successful += 1
count += 1
if count % scale == (scale-1):
successByCount.append(successful // scale)
# Build the graph, as a list of rows of characters.
size = count // scale
graph = [[" " for i in range(size+3)] for j in range(size)]
for c in range(size):
graph[c][1] = "|"
graph[c][c+3] = "."
graph[successByCount[c]][c+3] = "*"
graph.reverse()
# Print the graph.
print "\n Success of the classifier over time:\n"
print " . - Number of messages over time"
print " * - Number of correctly classified messages over time\n\n"
for row in range(size):
line = ''.join(graph[row])
if row == 0:
print line + " %d" % count
elif row == (count - successful) // scale:
print line + " %d" % successful
else:
print line
print " " + "_" * (size+2)
spamcount = 0
hamcount = 0
spamday = [0] * expire
hamday = [0] * expire
unsureday = [0] * expire
date_re = re.compile(
r";.* (\d{1,2} (?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec) \d{2,4})"
)
now = time.mktime(time.strptime(time.strftime("%d %b %Y"), "%d %b %Y"))
if loud: print "Scanning everything"
for f in os.listdir(everything):
if f[0] in ('1', '2', '3', '4', '5', '6', '7', '8', '9'):
name = os.path.join(everything, f)
fh = file(name, "rb")
msg = mboxutils.get_message(fh)
fh.close()
# Figure out how old the message is
age = 2 * expire
try:
received = (msg.get_all("Received"))[0]
received = date_re.search(received).group(1)
# if loud: print " %s" % received
date = time.mktime(time.strptime(received, "%d %b %Y"))
# if loud: print " %d" % date
age = (now - date) // day
# Can't just continue here... we're in a try
if age < 0:
age = 2 * expire
except:
pass
def main(argv):
opts, args = getopt.getopt(argv, "h", ["help"])
for opt, arg in opts:
if opt in ("-h", "--help"):
usage()
return
# Create the corpuses and the factory that reads the messages.
if options["pop3proxy", "cache_use_gzip"]:
messageFactory = GzipFileMessageFactory()
else:
messageFactory = FileMessageFactory()
sc = get_pathname_option("Storage", "spam_cache")
hc = get_pathname_option("Storage", "ham_cache")
spamCorpus = FileCorpus(messageFactory, sc)
hamCorpus = FileCorpus(messageFactory, hc)
# Read in all the trained messages.
allTrained = {}
for corpus, disposition in [(spamCorpus, 'Yes'), (hamCorpus, 'No')]:
for m in corpus:
message = mboxutils.get_message(m.getSubstance())
message._pop3CacheDisposition = disposition
allTrained[m.key()] = message
# Sort the messages into the order they arrived, then work out a scaling
# factor for the graph - 'limit' is the widest it can be in characters.
keys = allTrained.keys()
keys.sort()
limit = 70
if len(keys) < limit:
scale = 1
else:
scale = len(keys) // (limit // 2)
# Build the data - an array of cumulative success indexed by count.
count = successful = 0
successByCount = []
for key in keys:
message = allTrained[key]
disposition = message[options["Headers", "classification_header_name"]]
if (message._pop3CacheDisposition == disposition):
successful += 1
count += 1
if count % scale == (scale - 1):
successByCount.append(successful // scale)
# Build the graph, as a list of rows of characters.
size = count // scale
graph = [[" " for i in range(size + 3)] for j in range(size)]
for c in range(size):
graph[c][1] = "|"
graph[c][c + 3] = "."
graph[successByCount[c]][c + 3] = "*"
graph.reverse()
# Print the graph.
print "\n Success of the classifier over time:\n"
print " . - Number of messages over time"
print " * - Number of correctly classified messages over time\n\n"
for row in range(size):
line = ''.join(graph[row])
if row == 0:
print line + " %d" % count
elif row == (count - successful) // scale:
print line + " %d" % successful
else:
print line
print " " + "_" * (size + 2)
def get_message(self, obj):
return get_message(obj)
def score_and_filter(self,
msg,
header=None,
spam_cutoff=None,
ham_cutoff=None,
debugheader=None,
debug=None,
train=None):
"""Score (judge) a message and add a disposition header.
msg can be a string, a file object, or a Message object.
Optionally, set header to the name of the header to add, and/or
spam_cutoff/ham_cutoff to the probability values which must be met
or exceeded for a message to get a 'Spam' or 'Ham' classification.
An extra debugging header can be added if 'debug' is set to True.
The name of the debugging header is given as 'debugheader'.
If 'train' is True, also train on the result of scoring the
message (ie. train as ham if it's ham, train as spam if it's
spam). If the message already has a trained header, it will be
untrained first. You'll want to be very dilligent about
retraining mistakes if you use this option.
All defaults for optional parameters come from the Options file.
Returns the score and same message with a new disposition header.
"""
if header == None:
header = options["Headers", "classification_header_name"]
if spam_cutoff == None:
spam_cutoff = options["Categorization", "spam_cutoff"]
if ham_cutoff == None:
ham_cutoff = options["Categorization", "ham_cutoff"]
if debugheader == None:
debugheader = options["Headers", "evidence_header_name"]
if debug == None:
debug = options["Headers", "include_evidence"]
if train == None:
train = options["Hammie", "train_on_filter"]
msg = mboxutils.get_message(msg)
try:
del msg[header]
except KeyError:
pass
if train:
self.untrain_from_header(msg)
prob, clues = self._scoremsg(msg, True)
if prob < ham_cutoff:
is_spam = False
disp = options["Headers", "header_ham_string"]
elif prob > spam_cutoff:
is_spam = True
disp = options["Headers", "header_spam_string"]
else:
is_spam = False
disp = options["Headers", "header_unsure_string"]
if train:
self.train(msg, is_spam, True)
basic_disp = disp
disp += "; %.*f" % (options["Headers", "header_score_digits"], prob)
if options["Headers", "header_score_logarithm"]:
if prob <= 0.005 and prob > 0.0:
import math
x = -math.log10(prob)
disp += " (%d)" % x
if prob >= 0.995 and prob < 1.0:
x = -math.log10(1.0 - prob)
disp += " (%d)" % x
del msg[header]
msg.add_header(header, disp)
# Obey notate_to and notate_subject.
for header in ('to', 'subject'):
if basic_disp in options["Headers", "notate_" + header]:
orig = msg[header]
del msg[header]
msg[header] = "%s,%s" % (basic_disp, orig)
if debug:
disp = self.formatclues(clues)
del msg[debugheader]
msg.add_header(debugheader, disp)
result = mboxutils.as_string(msg,
unixfrom=(msg.get_unixfrom() is not None))
return prob, result
skipcount = 0
spamcount = 0
hamcount = 0
spamday = [0] * expire
hamday = [0] * expire
unsureday = [0] * expire
date_re = re.compile(
r";.* (\d{1,2} (?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec) \d{2,4})")
now = time.mktime(time.strptime(time.strftime("%d %b %Y"), "%d %b %Y"))
if loud: print "Scanning everything"
for f in os.listdir(everything):
if f[0] in ('1', '2', '3', '4', '5', '6', '7', '8', '9'):
name = os.path.join(everything, f)
fh = file(name, "rb")
msg = mboxutils.get_message(fh)
fh.close()
# Figure out how old the message is
age = 2 * expire
try:
received = (msg.get_all("Received"))[0]
received = date_re.search(received).group(1)
# if loud: print " %s" % received
date = time.mktime(time.strptime(received, "%d %b %Y"))
# if loud: print " %d" % date
age = (now - date) // day
# Can't just continue here... we're in a try
if age < 0:
age = 2 * expire
except:
pass
def score_and_filter(self, msg, header=None, spam_cutoff=None,
ham_cutoff=None, debugheader=None,
debug=None, train=None):
"""Score (judge) a message and add a disposition header.
msg can be a string, a file object, or a Message object.
Optionally, set header to the name of the header to add, and/or
spam_cutoff/ham_cutoff to the probability values which must be met
or exceeded for a message to get a 'Spam' or 'Ham' classification.
An extra debugging header can be added if 'debug' is set to True.
The name of the debugging header is given as 'debugheader'.
If 'train' is True, also train on the result of scoring the
message (ie. train as ham if it's ham, train as spam if it's
spam). If the message already has a trained header, it will be
untrained first. You'll want to be very dilligent about
retraining mistakes if you use this option.
All defaults for optional parameters come from the Options file.
Returns the score and same message with a new disposition header.
"""
if header == None:
header = options["Headers", "classification_header_name"]
if spam_cutoff == None:
spam_cutoff = options["Categorization", "spam_cutoff"]
if ham_cutoff == None:
ham_cutoff = options["Categorization", "ham_cutoff"]
if debugheader == None:
debugheader = options["Headers", "evidence_header_name"]
if debug == None:
debug = options["Headers", "include_evidence"]
if train == None:
train = options["Hammie", "train_on_filter"]
msg = mboxutils.get_message(msg)
try:
del msg[header]
except KeyError:
pass
if train:
self.untrain_from_header(msg)
prob, clues = self._scoremsg(msg, True)
if prob < ham_cutoff:
is_spam = False
disp = options["Headers", "header_ham_string"]
elif prob > spam_cutoff:
is_spam = True
disp = options["Headers", "header_spam_string"]
else:
is_spam = False
disp = options["Headers", "header_unsure_string"]
if train:
self.train(msg, is_spam, True)
basic_disp = disp
disp += "; %.*f" % (options["Headers", "header_score_digits"], prob)
if options["Headers", "header_score_logarithm"]:
if prob<=0.005 and prob>0.0:
import math
x=-math.log10(prob)
disp += " (%d)"%x
if prob>=0.995 and prob<1.0:
import math
x=-math.log10(1.0-prob)
disp += " (%d)"%x
del msg[header]
msg.add_header(header, disp)
for header in ('to', 'subject'):
if basic_disp in options["Headers", "notate_"+header]:
orig = msg[header]
del msg[header]
msg[header] = "%s,%s" % (basic_disp, orig)
if debug:
disp = self.formatclues(clues)
del msg[debugheader]
msg.add_header(debugheader, disp)
result = mboxutils.as_string(msg, unixfrom=(msg.get_unixfrom()
is not None))
return prob, result
