def LoadAutoTagger(config, at_config):
if not config.real_hasattr('autotag'):
config.real_setattr('autotag', {})
aid = at_identify(at_config)
at = config.autotag.get(aid)
if aid not in config.autotag:
cfn = 'pickled-autotag.%s' % aid
try:
config.autotag[aid] = config.load_pickle(cfn)
except (IOError, EOFError):
tagger = at_config.tagger
trainer = at_config.trainer
config.autotag[aid] = AutoTagger(
TAGGERS.get(tagger, TAGGERS['_default'])(tagger),
TRAINERS.get(trainer, TRAINERS['_default'])(trainer),
)
SaveAutoTagger(config, at_config)
return config.autotag[aid]
def LoadAutoTagger(config, at_config):
if not config.real_hasattr('autotag'):
config.real_setattr('autotag', {})
aid = at_identify(at_config)
at = config.autotag.get(aid)
if aid not in config.autotag:
cfn = 'pickled-autotag.%s' % aid
try:
config.autotag[aid] = config.load_pickle(cfn)
except (IOError, EOFError):
tagger = at_config.tagger
trainer = at_config.trainer
config.autotag[aid] = AutoTagger(
TAGGERS.get(tagger, TAGGERS['_default'])(tagger),
TRAINERS.get(trainer, TRAINERS['_default'])(trainer),
)
SaveAutoTagger(config, at_config)
return config.autotag[aid]
def _retrain(self, tags=None):
"Retrain autotaggers"
session, config, idx = self.session, self.session.config, self._idx()
tags = tags or [asb.match_tag for asb in config.prefs.autotag]
tids = [config.get_tag(t)._key for t in tags if t]
session.ui.mark(_('Retraining SpamBayes autotaggers'))
if not config.real_hasattr('autotag'):
config.real_setattr('autotag', {})
# Find all the interesting messages! We don't look in the trash,
# but we do look at interesting spam.
#
# Note: By specifically stating that we DON'T want trash, we
# disable the search engine's default result suppression
# and guarantee these results don't corrupt the somewhat
# lame/broken result cache.
#
no_trash = ['-in:%s' % t._key for t in config.get_tags(type='trash')]
interest = {}
for ttype in ('replied', 'fwded', 'read', 'tagged'):
interest[ttype] = set()
for tag in config.get_tags(type=ttype):
interest[ttype] |= idx.search(session, ['in:%s' % tag.slug] +
no_trash).as_set()
session.ui.notify(
_('Have %d interesting %s messages') %
(len(interest[ttype]), ttype))
retrained, unreadable = [], []
count_all = 0
for at_config in config.prefs.autotag:
at_tag = config.get_tag(at_config.match_tag)
if at_tag and at_tag._key in tids:
session.ui.mark('Retraining: %s' % at_tag.name)
yn = [(set(), set(), 'in:%s' % at_tag.slug, True),
(set(), set(), '-in:%s' % at_tag.slug, False)]
# Get the current message sets: tagged and untagged messages
# excluding trash.
for tset, mset, srch, which in yn:
mset |= idx.search(session, [srch] + no_trash).as_set()
# If we have any exclude_tags, they are particularly
# interesting, so we'll look at them first.
interesting = []
for etagid in at_config.exclude_tags:
etag = config.get_tag(etagid)
if etag._key not in interest:
srch = ['in:%s' % etag._key] + no_trash
interest[etag._key] = idx.search(session,
srch).as_set()
interesting.append(etag._key)
interesting.extend(
['replied', 'fwded', 'read', 'tagged', None])
# Go through the interest types in order of preference and
# while we still lack training data, add to the training set.
for ttype in interesting:
for tset, mset, srch, which in yn:
# False positives are really annoying, and generally
# speaking any autotagged subset should be a small
# part of the Universe. So we divide the corpus
# budget 33% True, 67% False.
full_size = int(at_config.corpus_size *
(0.33 if which else 0.67))
want = min(full_size // 4, max(0,
full_size - len(tset)))
if want:
if ttype:
adding = sorted(list(mset & interest[ttype]))
else:
adding = sorted(list(mset))
adding = set(list(reversed(adding))[:want])
tset |= adding
mset -= adding
# Load classifier, reset
atagger = config.load_auto_tagger(at_config)
atagger.reset(at_config)
for tset, mset, srch, which in yn:
count = 0
# We go through the liste of message in order, to avoid
# thrashing caches too badly.
for msg_idx in sorted(list(tset)):
try:
e = Email(idx, msg_idx)
count += 1
count_all += 1
session.ui.mark(
_('Reading %s (%d/%d, %s=%s)') %
(e.msg_mid(), count, len(tset), at_tag.name,
which))
atagger.learn(at_config, e.get_msg(),
self._get_keywords(e), which)
except (IndexError, TypeError, ValueError, OSError,
IOError):
if session.config.sys.debug:
import traceback
traceback.print_exc()
unreadable.append(msg_idx)
session.ui.warning(
_('Failed to process message at =%s') %
(b36(msg_idx)))
# We got this far without crashing, so save the result.
config.save_auto_tagger(at_config)
retrained.append(at_tag.name)
message = _('Retrained SpamBayes auto-tagging for %s') % ', '.join(
retrained)
session.ui.mark(message)
return self._success(message,
result={
'retrained': retrained,
'unreadable': unreadable,
'read_messages': count_all
})
def _retrain(self, tags=None):
"Retrain autotaggers"
session, config, idx = self.session, self.session.config, self._idx()
tags = tags or [asb.match_tag for asb in config.prefs.autotag]
tids = [config.get_tag(t)._key for t in tags if t]
session.ui.mark(_('Retraining SpamBayes autotaggers'))
if not config.real_hasattr('autotag'):
config.real_setattr('autotag', {})
# Find all the interesting messages! We don't look in the trash,
# but we do look at interesting spam.
#
# Note: By specifically stating that we DON'T want trash, we
# disable the search engine's default result suppression
# and guarantee these results don't corrupt the somewhat
# lame/broken result cache.
#
no_trash = ['-in:%s' % t._key for t in config.get_tags(type='trash')]
interest = {}
for ttype in ('replied', 'fwded', 'read', 'tagged'):
interest[ttype] = set()
for tag in config.get_tags(type=ttype):
interest[ttype] |= idx.search(session,
['in:%s' % tag.slug] + no_trash
).as_set()
session.ui.notify(_('Have %d interesting %s messages'
) % (len(interest[ttype]), ttype))
retrained, unreadable = [], []
count_all = 0
for at_config in config.prefs.autotag:
at_tag = config.get_tag(at_config.match_tag)
if at_tag and at_tag._key in tids:
session.ui.mark('Retraining: %s' % at_tag.name)
yn = [(set(), set(), 'in:%s' % at_tag.slug, True),
(set(), set(), '-in:%s' % at_tag.slug, False)]
# Get the current message sets: tagged and untagged messages
# excluding trash.
for tset, mset, srch, which in yn:
mset |= idx.search(session, [srch] + no_trash).as_set()
# If we have any exclude_tags, they are particularly
# interesting, so we'll look at them first.
interesting = []
for etagid in at_config.exclude_tags:
etag = config.get_tag(etagid)
if etag._key not in interest:
srch = ['in:%s' % etag._key] + no_trash
interest[etag._key] = idx.search(session, srch
).as_set()
interesting.append(etag._key)
interesting.extend(['replied', 'fwded', 'read', 'tagged',
None])
# Go through the interest types in order of preference and
# while we still lack training data, add to the training set.
for ttype in interesting:
for tset, mset, srch, which in yn:
# FIXME: Is this a good idea? No single data source
# is allowed to be more than 50% of the corpus, to
# try and encourage diversity.
want = min(at_config.corpus_size / 4,
max(0,
at_config.corpus_size / 2 - len(tset)))
if want:
if ttype:
adding = sorted(list(mset & interest[ttype]))
else:
adding = sorted(list(mset))
adding = set(list(reversed(adding))[:want])
tset |= adding
mset -= adding
# Load classifier, reset
atagger = config.load_auto_tagger(at_config)
atagger.reset(at_config)
for tset, mset, srch, which in yn:
count = 0
for msg_idx in tset:
try:
e = Email(idx, msg_idx)
count += 1
count_all += 1
session.ui.mark(
_('Reading %s (%d/%d, %s=%s)'
) % (e.msg_mid(), count, len(tset),
at_tag.name, which))
atagger.learn(at_config,
e.get_msg(),
self._get_keywords(e),
which)
except (IndexError, TypeError, ValueError,
OSError, IOError):
if session.config.sys.debug:
import traceback
traceback.print_exc()
unreadable.append(msg_idx)
session.ui.warning(
_('Failed to process message at =%s'
) % (b36(msg_idx)))
# We got this far without crashing, so save the result.
config.save_auto_tagger(at_config)
retrained.append(at_tag.name)
message = _('Retrained SpamBayes auto-tagging for %s'
) % ', '.join(retrained)
session.ui.mark(message)
return self._success(message, result={
'retrained': retrained,
'unreadable': unreadable,
'read_messages': count_all
})