Пример #1
0
def main(args):
    opts, args = getopt.getopt(args, "h")

    for opt, arg in opts:
        if opt == '-h':
            usage()
            return 0

    msg = mboxutils.get_message(sys.stdin)
    try:
        del msg["X-Spambayes-Classification"]
    except KeyError:
        pass
    for pair in args:
        tag, db = pair.split('=', 1)
        h = hammie.open(db, True, 'r')
        score = h.score(msg)
        if score >= Options.options["Categorization", "spam_cutoff"]:
            msg["X-Spambayes-Classification"] = "%s; %.2f" % (tag, score)
            break
    else:
        msg["X-Spambayes-Classification"] = "unsure"

    sys.stdout.write(msg.as_string(unixfrom=(msg.get_unixfrom() is not None)))
    return 0
Пример #2
0
     def untrain_from_header(self, msg):

        """Untrain bayes based on X-Spambayes-Trained header.
        msg can be a string, a file object, or a Message object.
        If no such header is present, nothing happens.
        If add_header is True, add a header with how it was trained (in
        case we need to untrain later)
        """

        msg = mboxutils.get_message(msg)

        trained = msg.get(options["Headers", "trained_header_name"])

        if not trained:

            return

        del msg[options["Headers", "trained_header_name"]]

        if trained == options["Headers", "header_ham_string"]:

            self.untrain_ham(msg)

        elif trained == options["Headers", "header_spam_string"]:

            self.untrain_spam(msg)

        else:

            raise ValueError('%s header value unrecognized'
                             % options["Headers", "trained_header_name"])
Пример #3
0
    def process_mailbox(self, mailbox):
        count, size = mailbox.stat()
        log = Logger()

        for i in range(1, count+1):
            if (i-1) % 10 == 0:
                print " == %d/%d ==" % (i, count)
            # Kevin's code used -1, but -1 doesn't work for one of
            # my POP accounts, while a million does.
            # Don't use retr because that may mark the message as
            # read (so says Kevin's code)
            message_tuple = mailbox.top(i, 1000000)
            text = "\n".join(message_tuple[1])
            msg = mboxutils.get_message(text)

            mi = MessageInfo(mailbox, i, msg, text)
            
            _log_subject(mi, log)

            for filter in self:
                result = filter.process(mi, log)
                if result:
                    log.accept(result)
                    break
            else:
                # don't know what to do with this so just
                # keep it on the server
                log.pass_test("unknown")
                log.do_action(KEEP_IN_MAILBOX)
                log.accept("unknown")
            
        return log
Пример #4
0
    def process_mailbox(self, mailbox):
        count, size = mailbox.stat()
        log = Logger()

        for i in range(1, count + 1):
            if (i - 1) % 10 == 0:
                print " == %d/%d ==" % (i, count)
            # Kevin's code used -1, but -1 doesn't work for one of
            # my POP accounts, while a million does.
            # Don't use retr because that may mark the message as
            # read (so says Kevin's code)
            message_tuple = mailbox.top(i, 1000000)
            text = "\n".join(message_tuple[1])
            msg = mboxutils.get_message(text)

            mi = MessageInfo(mailbox, i, msg, text)

            _log_subject(mi, log)

            for filter in self:
                result = filter.process(mi, log)
                if result:
                    log.accept(result)
                    break
            else:
                # don't know what to do with this so just
                # keep it on the server
                log.pass_test("unknown")
                log.do_action(KEEP_IN_MAILBOX)
                log.accept("unknown")

        return log
Пример #5
0
 def _calc_response(self, switches, body):
     switches = switches.split()
     actions = []
     opts, args = getopt.getopt(switches, 'fgstGS')
     h = self.server.hammie
     for opt, arg in opts:
         if opt == '-f':
             actions.append(h.filter)
         elif opt == '-g':
             actions.append(h.train_ham)
         elif opt == '-s':
             actions.append(h.train_spam)
         elif opt == '-t':
             actions.append(h.filter_train)
         elif opt == '-G':
             actions.append(h.untrain_ham)
         elif opt == '-S':
             actions.append(h.untrain_spam)
     if actions == []:
         actions = [h.filter]
     from spambayes import mboxutils
     msg = mboxutils.get_message(body)
     for action in actions:
         action(msg)
     return mboxutils.as_string(msg, 1)
Пример #6
0
def main(args):
    opts, args = getopt.getopt(args, "h")

    for opt, arg in opts:
        if opt == '-h':
            help()
            return 0

    tagdb_list = []
    msg = mboxutils.get_message(sys.stdin)
    try:
        del msg["X-Spambayes-Classification"]
    except KeyError:
        pass
    for pair in args:
        tag, db = pair.split('=', 1)
        h = hammie.open(db, True, 'r')
        score = h.score(msg)
        if score >= Options.options["Categorization", "spam_cutoff"]:
            msg["X-Spambayes-Classification"] = "%s; %.2f" % (tag, score)
            break
    else:
        msg["X-Spambayes-Classification"] = "unsure"

    sys.stdout.write(msg.as_string(unixfrom=(msg.get_unixfrom()
                                             is not None)))
    return 0
Пример #7
0
 def _calc_response(self, switches, body):
     switches = switches.split()
     actions = []
     opts, args = getopt.getopt(switches, 'fgstGS')
     h = self.server.hammie
     for opt, arg in opts:
         if opt == '-f':
             actions.append(h.filter)
         elif opt == '-g':
             actions.append(h.train_ham)
         elif opt == '-s':
             actions.append(h.train_spam)
         elif opt == '-t':
             actions.append(h.filter_train)
         elif opt == '-G':
             actions.append(h.untrain_ham)
         elif opt == '-S':
             actions.append(h.untrain_spam)
     if actions == []:
         actions = [h.filter]
     from spambayes import mboxutils
     msg = mboxutils.get_message(body)
     for action in actions:
         action(msg)
     return mboxutils.as_string(msg, 1)
Пример #8
0
def main(argv):
    opts, args = getopt.getopt(argv, "h", ["help"])
    for opt, arg in opts:
        if opt in ("-h", "--help"):
            usage()
            return
    if options["pop3proxy", "cache_use_gzip"]:
        messageFactory = GzipFileMessageFactory()
    else:
        messageFactory = FileMessageFactory()
    sc = get_pathname_option("Storage", "spam_cache")
    hc = get_pathname_option("Storage", "ham_cache")
    spamCorpus = FileCorpus(messageFactory, sc)
    hamCorpus = FileCorpus(messageFactory, hc)
    allTrained = {}
    for corpus, disposition in [(spamCorpus, 'Yes'), (hamCorpus, 'No')]:
        for m in corpus:
            message = mboxutils.get_message(m.getSubstance())
            message._pop3CacheDisposition = disposition
            allTrained[m.key()] = message
    keys = allTrained.keys()
    keys.sort()
    limit = 70
    if len(keys) < limit:
        scale = 1
    else:
        scale = len(keys) // (limit//2)
    count = successful = 0
    successByCount = []
    for key in keys:
        message = allTrained[key]
        disposition = message[options["Headers",
                                      "classification_header_name"]]
        if (message._pop3CacheDisposition == disposition):
            successful += 1
        count += 1
        if count % scale == (scale-1):
            successByCount.append(successful // scale)
    size = count // scale
    graph = [[" " for i in range(size+3)] for j in range(size)]
    for c in range(size):
        graph[c][1] = "|"
        graph[c][c+3] = "."
        graph[successByCount[c]][c+3] = "*"
    graph.reverse()
    print "\n   Success of the classifier over time:\n"
    print "   . - Number of messages over time"
    print "   * - Number of correctly classified messages over time\n\n"
    for row in range(size):
        line = ''.join(graph[row])
        if row == 0:
            print line + " %d" % count
        elif row == (count - successful) // scale:
            print line + " %d" % successful
        else:
            print line
    print " " + "_" * (size+2)
Пример #9
0
 def process_mailbox(self, mailbox):
     count, size = mailbox.stat()
     log = Logger()
     for i in range(1, count+1):
         if (i-1) % 10 == 0:
             print " == %d/%d ==" % (i, count)
         message_tuple = mailbox.top(i, 1000000)
         text = "\n".join(message_tuple[1])
         msg = mboxutils.get_message(text)
         mi = MessageInfo(mailbox, i, msg, text)
         _log_subject(mi, log)
         for filter in self:
             result = filter.process(mi, log)
             if result:
                 log.accept(result)
                 break
         else:
             log.pass_test("unknown")
             log.do_action(KEEP_IN_MAILBOX)
             log.accept("unknown")
     return log
Пример #10
0
    def untrain_from_header(self, msg):
        """Untrain bayes based on X-Spambayes-Trained header.

        msg can be a string, a file object, or a Message object.

        If no such header is present, nothing happens.

        If add_header is True, add a header with how it was trained (in
        case we need to untrain later)

        """

        msg = mboxutils.get_message(msg)
        trained = msg.get(options["Headers", "trained_header_name"])
        if not trained:
            return
        del msg[options["Headers", "trained_header_name"]]
        if trained == options["Headers", "header_ham_string"]:
            self.untrain_ham(msg)
        elif trained == options["Headers", "header_spam_string"]:
            self.untrain_spam(msg)
        else:
            raise ValueError('%s header value unrecognized' %
                             options["Headers", "trained_header_name"])
Пример #11
0
def main(argv):
    opts, args = getopt.getopt(argv, "h", ["help"])
    for opt, arg in opts:
        if opt in ("-h", "--help"):
            usage()
            return

    # Create the corpuses and the factory that reads the messages.
    if options["pop3proxy", "cache_use_gzip"]:
        messageFactory = GzipFileMessageFactory()
    else:
        messageFactory = FileMessageFactory()
    sc = get_pathname_option("Storage", "spam_cache")
    hc = get_pathname_option("Storage", "ham_cache")
    spamCorpus = FileCorpus(messageFactory, sc)
    hamCorpus = FileCorpus(messageFactory, hc)

    # Read in all the trained messages.
    allTrained = {}
    for corpus, disposition in [(spamCorpus, 'Yes'), (hamCorpus, 'No')]:
        for m in corpus:
            message = mboxutils.get_message(m.getSubstance())
            message._pop3CacheDisposition = disposition
            allTrained[m.key()] = message

    # Sort the messages into the order they arrived, then work out a scaling
    # factor for the graph - 'limit' is the widest it can be in characters.
    keys = allTrained.keys()
    keys.sort()
    limit = 70
    if len(keys) < limit:
        scale = 1
    else:
        scale = len(keys) // (limit//2)

    # Build the data - an array of cumulative success indexed by count.
    count = successful = 0
    successByCount = []
    for key in keys:
        message = allTrained[key]
        disposition = message[options["Headers",
                                      "classification_header_name"]]
        if (message._pop3CacheDisposition == disposition):
            successful += 1
        count += 1
        if count % scale == (scale-1):
            successByCount.append(successful // scale)

    # Build the graph, as a list of rows of characters.
    size = count // scale
    graph = [[" " for i in range(size+3)] for j in range(size)]
    for c in range(size):
        graph[c][1] = "|"
        graph[c][c+3] = "."
        graph[successByCount[c]][c+3] = "*"
    graph.reverse()

    # Print the graph.
    print "\n   Success of the classifier over time:\n"
    print "   . - Number of messages over time"
    print "   * - Number of correctly classified messages over time\n\n"
    for row in range(size):
        line = ''.join(graph[row])
        if row == 0:
            print line + " %d" % count
        elif row == (count - successful) // scale:
            print line + " %d" % successful
        else:
            print line
    print " " + "_" * (size+2)
Пример #12
0
    spamcount = 0
    hamcount = 0
    spamday = [0] * expire
    hamday = [0] * expire
    unsureday = [0] * expire
    date_re = re.compile(
        r";.* (\d{1,2} (?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec) \d{2,4})"
    )
    now = time.mktime(time.strptime(time.strftime("%d %b %Y"), "%d %b %Y"))
    if loud: print "Scanning everything"
    for f in os.listdir(everything):
        if f[0] in ('1', '2', '3', '4', '5', '6', '7', '8', '9'):
            name = os.path.join(everything, f)

            fh = file(name, "rb")
            msg = mboxutils.get_message(fh)
            fh.close()
            # Figure out how old the message is
            age = 2 * expire
            try:
                received = (msg.get_all("Received"))[0]
                received = date_re.search(received).group(1)
                # if loud: print "  %s" % received
                date = time.mktime(time.strptime(received, "%d %b %Y"))
                # if loud: print "  %d" % date
                age = (now - date) // day
                # Can't just continue here... we're in a try
                if age < 0:
                    age = 2 * expire
            except:
                pass
Пример #13
0
def main(argv):
    opts, args = getopt.getopt(argv, "h", ["help"])
    for opt, arg in opts:
        if opt in ("-h", "--help"):
            usage()
            return

    # Create the corpuses and the factory that reads the messages.
    if options["pop3proxy", "cache_use_gzip"]:
        messageFactory = GzipFileMessageFactory()
    else:
        messageFactory = FileMessageFactory()
    sc = get_pathname_option("Storage", "spam_cache")
    hc = get_pathname_option("Storage", "ham_cache")
    spamCorpus = FileCorpus(messageFactory, sc)
    hamCorpus = FileCorpus(messageFactory, hc)

    # Read in all the trained messages.
    allTrained = {}
    for corpus, disposition in [(spamCorpus, 'Yes'), (hamCorpus, 'No')]:
        for m in corpus:
            message = mboxutils.get_message(m.getSubstance())
            message._pop3CacheDisposition = disposition
            allTrained[m.key()] = message

    # Sort the messages into the order they arrived, then work out a scaling
    # factor for the graph - 'limit' is the widest it can be in characters.
    keys = allTrained.keys()
    keys.sort()
    limit = 70
    if len(keys) < limit:
        scale = 1
    else:
        scale = len(keys) // (limit // 2)

    # Build the data - an array of cumulative success indexed by count.
    count = successful = 0
    successByCount = []
    for key in keys:
        message = allTrained[key]
        disposition = message[options["Headers", "classification_header_name"]]
        if (message._pop3CacheDisposition == disposition):
            successful += 1
        count += 1
        if count % scale == (scale - 1):
            successByCount.append(successful // scale)

    # Build the graph, as a list of rows of characters.
    size = count // scale
    graph = [[" " for i in range(size + 3)] for j in range(size)]
    for c in range(size):
        graph[c][1] = "|"
        graph[c][c + 3] = "."
        graph[successByCount[c]][c + 3] = "*"
    graph.reverse()

    # Print the graph.
    print "\n   Success of the classifier over time:\n"
    print "   . - Number of messages over time"
    print "   * - Number of correctly classified messages over time\n\n"
    for row in range(size):
        line = ''.join(graph[row])
        if row == 0:
            print line + " %d" % count
        elif row == (count - successful) // scale:
            print line + " %d" % successful
        else:
            print line
    print " " + "_" * (size + 2)
Пример #14
0
 def get_message(self, obj):
     return get_message(obj)
Пример #15
0
    def score_and_filter(self,
                         msg,
                         header=None,
                         spam_cutoff=None,
                         ham_cutoff=None,
                         debugheader=None,
                         debug=None,
                         train=None):
        """Score (judge) a message and add a disposition header.

        msg can be a string, a file object, or a Message object.

        Optionally, set header to the name of the header to add, and/or
        spam_cutoff/ham_cutoff to the probability values which must be met
        or exceeded for a message to get a 'Spam' or 'Ham' classification.

        An extra debugging header can be added if 'debug' is set to True.
        The name of the debugging header is given as 'debugheader'.

        If 'train' is True, also train on the result of scoring the
        message (ie. train as ham if it's ham, train as spam if it's
        spam).  If the message already has a trained header, it will be
        untrained first.  You'll want to be very dilligent about
        retraining mistakes if you use this option.

        All defaults for optional parameters come from the Options file.

        Returns the score and same message with a new disposition header.
        """

        if header == None:
            header = options["Headers", "classification_header_name"]
        if spam_cutoff == None:
            spam_cutoff = options["Categorization", "spam_cutoff"]
        if ham_cutoff == None:
            ham_cutoff = options["Categorization", "ham_cutoff"]
        if debugheader == None:
            debugheader = options["Headers", "evidence_header_name"]
        if debug == None:
            debug = options["Headers", "include_evidence"]
        if train == None:
            train = options["Hammie", "train_on_filter"]

        msg = mboxutils.get_message(msg)
        try:
            del msg[header]
        except KeyError:
            pass
        if train:
            self.untrain_from_header(msg)
        prob, clues = self._scoremsg(msg, True)
        if prob < ham_cutoff:
            is_spam = False
            disp = options["Headers", "header_ham_string"]
        elif prob > spam_cutoff:
            is_spam = True
            disp = options["Headers", "header_spam_string"]
        else:
            is_spam = False
            disp = options["Headers", "header_unsure_string"]
        if train:
            self.train(msg, is_spam, True)
        basic_disp = disp
        disp += "; %.*f" % (options["Headers", "header_score_digits"], prob)
        if options["Headers", "header_score_logarithm"]:
            if prob <= 0.005 and prob > 0.0:
                import math
                x = -math.log10(prob)
                disp += " (%d)" % x
            if prob >= 0.995 and prob < 1.0:
                x = -math.log10(1.0 - prob)
                disp += " (%d)" % x
        del msg[header]
        msg.add_header(header, disp)

        # Obey notate_to and notate_subject.
        for header in ('to', 'subject'):
            if basic_disp in options["Headers", "notate_" + header]:
                orig = msg[header]
                del msg[header]
                msg[header] = "%s,%s" % (basic_disp, orig)

        if debug:
            disp = self.formatclues(clues)
            del msg[debugheader]
            msg.add_header(debugheader, disp)
        result = mboxutils.as_string(msg,
                                     unixfrom=(msg.get_unixfrom() is not None))
        return prob, result
Пример #16
0
    skipcount = 0
    spamcount = 0
    hamcount = 0
    spamday = [0] * expire
    hamday = [0] * expire
    unsureday = [0] * expire
    date_re = re.compile(
        r";.* (\d{1,2} (?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec) \d{2,4})")
    now = time.mktime(time.strptime(time.strftime("%d %b %Y"), "%d %b %Y"))
    if loud: print "Scanning everything"
    for f in os.listdir(everything):
        if f[0] in ('1', '2', '3', '4', '5', '6', '7', '8', '9'):
            name = os.path.join(everything, f)

            fh = file(name, "rb")
            msg = mboxutils.get_message(fh)
            fh.close()
            # Figure out how old the message is
            age = 2 * expire
            try:
                received = (msg.get_all("Received"))[0]
                received = date_re.search(received).group(1)
                # if loud: print "  %s" % received
                date = time.mktime(time.strptime(received, "%d %b %Y"))
                # if loud: print "  %d" % date
                age = (now - date) // day
                # Can't just continue here... we're in a try
                if age < 0:
                    age = 2 * expire
            except:
                pass
Пример #17
0
     def score_and_filter(self, msg, header=None, spam_cutoff=None,
                         ham_cutoff=None, debugheader=None,
                         debug=None, train=None):

        """Score (judge) a message and add a disposition header.
        msg can be a string, a file object, or a Message object.
        Optionally, set header to the name of the header to add, and/or
        spam_cutoff/ham_cutoff to the probability values which must be met
        or exceeded for a message to get a 'Spam' or 'Ham' classification.
        An extra debugging header can be added if 'debug' is set to True.
        The name of the debugging header is given as 'debugheader'.
        If 'train' is True, also train on the result of scoring the
        message (ie. train as ham if it's ham, train as spam if it's
        spam).  If the message already has a trained header, it will be
        untrained first.  You'll want to be very dilligent about
        retraining mistakes if you use this option.
        All defaults for optional parameters come from the Options file.
        Returns the score and same message with a new disposition header.
        """

        if header == None:

            header = options["Headers", "classification_header_name"]

        if spam_cutoff == None:

            spam_cutoff = options["Categorization", "spam_cutoff"]

        if ham_cutoff == None:

            ham_cutoff = options["Categorization", "ham_cutoff"]

        if debugheader == None:

            debugheader = options["Headers", "evidence_header_name"]

        if debug == None:

            debug = options["Headers", "include_evidence"]

        if train == None:

            train = options["Hammie", "train_on_filter"]

        msg = mboxutils.get_message(msg)

        try:

            del msg[header]

        except KeyError:

            pass

        if train:

            self.untrain_from_header(msg)

        prob, clues = self._scoremsg(msg, True)

        if prob < ham_cutoff:

            is_spam = False

            disp = options["Headers", "header_ham_string"]

        elif prob > spam_cutoff:

            is_spam = True

            disp = options["Headers", "header_spam_string"]

        else:

            is_spam = False

            disp = options["Headers", "header_unsure_string"]

        if train:

            self.train(msg, is_spam, True)

        basic_disp = disp

        disp += "; %.*f" % (options["Headers", "header_score_digits"], prob)

        if options["Headers", "header_score_logarithm"]:

            if prob<=0.005 and prob>0.0:

                import math

                x=-math.log10(prob)

                disp += " (%d)"%x

            if prob>=0.995 and prob<1.0:

                import math

                x=-math.log10(1.0-prob)

                disp += " (%d)"%x

        del msg[header]

        msg.add_header(header, disp)

        for header in ('to', 'subject'):

            if basic_disp in options["Headers", "notate_"+header]:

                orig = msg[header]

                del msg[header]

                msg[header] = "%s,%s" % (basic_disp, orig)

        if debug:

            disp = self.formatclues(clues)

            del msg[debugheader]

            msg.add_header(debugheader, disp)

        result = mboxutils.as_string(msg, unixfrom=(msg.get_unixfrom()
                                                    is not None))

        return prob, result