def _render_monitoring_panel(self, req, cat, page):
req.perm.assert_permission('SPAM_MONITOR')
try:
pagenum = int(req.args.get('page', 1)) - 1
except ValueError:
pagenum = 1
total = LogEntry.count(self.env)
offset = pagenum * self.MAX_PER_PAGE
entries = list(LogEntry.select(self.env, limit=self.MAX_PER_PAGE,
offset=offset))
if pagenum > 0:
add_link(req, 'prev', req.href.admin(cat, page, page=pagenum),
'Previous Page')
if offset + self.MAX_PER_PAGE < total:
add_link(req, 'next', req.href.admin(cat, page, page=pagenum+2),
'Next Page')
return {
'enabled': FilterSystem(self.env).logging_enabled,
'entries': entries,
'offset': offset + 1,
'page': pagenum + 1,
'total': total
}
def _process_monitoring_panel(self, req):
req.perm.assert_permission('SPAM_TRAIN')
filtersys = FilterSystem(self.env)
if 'markspam' in req.args or 'markham' in req.args:
spam = 'markspam' in req.args
for entry_id in req.args.getlist('sel'):
filtersys.train(req, entry_id, spam=spam)
elif 'delete' in req.args:
for entry_id in req.args.getlist('sel'):
LogEntry.delete(self.env, entry_id)
return True
def train(self, req, log_id, spam=True):
environ = {}
for name, value in req.environ.items():
if not name.startswith('HTTP_'):
environ[name] = value
entry = LogEntry.fetch(self.env, log_id)
if entry:
self.log.debug('Marking as %s: %r submitted by "%s"',
spam and 'spam' or 'ham',
shorten_line(entry.content),
entry.author)
fakeenv = environ.copy()
for header in entry.headers.splitlines():
name, value = header.split(':', 1)
if name == 'Cookie': # breaks SimpleCookie somehow
continue
cgi_name = 'HTTP_%s' % name.strip().replace('-', '_').upper()
fakeenv[cgi_name] = value.strip()
fakeenv['REQUEST_METHOD'] = 'POST'
fakeenv['PATH_INFO'] = entry.path
fakeenv['wsgi.input'] = StringIO('')
fakeenv['REMOTE_ADDR'] = entry.ipnr
if entry.authenticated:
fakeenv['REMOTE_USER'] = entry.author
for strategy in self.strategies:
strategy.train(Request(fakeenv, None),
entry.author or 'anonymous',
entry.content, spam=spam)
entry.update(rejected=spam)
def test_purge(self):
now = datetime.now()
oneweekago = time.mktime((now - timedelta(weeks=1)).timetuple())
onedayago = time.mktime((now - timedelta(days=1)).timetuple())
LogEntry(self.env, oneweekago, '/foo', 'john', False, '127.0.0.1',
'', 'Test', False, 5, []).insert()
LogEntry(self.env, onedayago, '/foo', 'anonymous', False, '127.0.0.1',
'', 'Test', True, -3, []).insert()
LogEntry.purge(self.env, days=4)
log = list(LogEntry.select(self.env))
self.assertEqual(1, len(log))
entry = log[0]
self.assertEqual('anonymous', entry.author)
def test_train_ham(self):
req = MockRequest(self.env, path_info='/foo')
entry = LogEntry(self.env, time.time(), '/foo', 'john', False,
'127.0.0.1', '', 'Test', True, -5, [], req)
entry.insert()
FilterSystem(self.env).train(req, entry.id, spam=False)
strategy = DummyStrategy(self.env)
self.assertEqual(True, strategy.train_called)
self.assertEqual('john', strategy.author)
self.assertEqual('Test', strategy.content)
self.assertEqual(False, strategy.spam)
log = list(LogEntry.select(self.env))
self.assertEqual(1, len(log))
entry = log[0]
self.assertEqual(False, entry.rejected)
def delete(self, req, ids, stats):
if not isinstance(ids, list):
ids = [ids]
if stats:
for log_id in ids:
entry = LogEntry.fetch(self.env, log_id)
if entry:
type = "spam" if entry.rejected else "ham"
status = "delete"
for strategy in self.strategies:
count = entry.findreasonvalue(get_strategy_name(strategy))
if count:
spamstatus = count < 0
self._record_action(status, type, ("ok" if spamstatus == entry.rejected else "error"), strategy, 0)
else:
self._record_action(status, type, '', strategy, 0)
self._record_action(status, type, '', '', 0)
LogEntry.delete(self.env, ids)
def test(self, req, author, content):
threshold = datetime.now() - timedelta(hours=1)
num_posts = 0
for entry in LogEntry.select(self.env, ipnr=req.remote_addr):
if datetime.fromtimestamp(entry.time) < threshold:
break
num_posts += 1
if num_posts > self.max_posts:
return -abs(self.karma_points) * num_posts / self.max_posts, \
'Maximum number of posts per hour for this IP exceeded'
def test_train_ham(self):
entry = LogEntry(self.env, time.time(), "/foo", "john", False, "127.0.0.1", "", "Test", True, -5, [])
entry.insert()
req = Mock(
environ={"SERVER_NAME": "localhost", "SERVER_PORT": "80", "wsgi.url_scheme": "http"},
path_info="/foo",
authname="anonymous",
remote_addr="127.0.0.1",
)
FilterSystem(self.env).train(req, entry.id, spam=False)
strategy = DummyStrategy(self.env)
self.assertEqual(True, strategy.train_called)
self.assertEqual("john", strategy.author)
self.assertEqual("Test", strategy.content)
self.assertEqual(False, strategy.spam)
log = list(LogEntry.select(self.env))
self.assertEqual(1, len(log))
entry = log[0]
self.assertEqual(False, entry.rejected)
def _render_monitoring_panel(self, req, cat, page):
req.perm.require('SPAM_MONITOR')
pagenum = req.args.as_int('page', 1) - 1
pagesize = req.args.as_int('num', self.DEF_PER_PAGE,
min=self.MIN_PER_PAGE,
max=self.MAX_PER_PAGE)
total = LogEntry.count(self.env)
if total < pagesize:
pagenum = 0
elif total <= pagenum * pagesize:
pagenum = (total - 1) / pagesize
offset = pagenum * pagesize
entries = list(LogEntry.select(self.env, limit=pagesize,
offset=offset))
if pagenum > 0:
add_link(req, 'prev',
req.href.admin(cat, page, page=pagenum, num=pagesize),
_("Previous Page"))
if offset + pagesize < total:
add_link(req, 'next',
req.href.admin(cat, page, page=pagenum + 2, num=pagesize),
_("Next Page"))
return {
'enabled': FilterSystem(self.env).logging_enabled,
'entries': entries,
'offset': offset + 1,
'page': pagenum + 1,
'num': pagesize,
'total': total,
'train_only': self.train_only
}
def test_log_accept(self):
req = MockRequest(self.env, path_info='/foo')
DummyStrategy(self.env).configure(5)
FilterSystem(self.env).test(req, 'John Doe', [(None, 'Test')])
log = list(LogEntry.select(self.env))
self.assertEqual(1, len(log))
entry = log[0]
self.assertEqual('/foo', entry.path)
self.assertEqual('John Doe', entry.author)
self.assertEqual(False, entry.authenticated)
self.assertEqual('127.0.0.1', entry.ipnr)
self.assertEqual('Test', entry.content)
self.assertEqual(False, entry.rejected)
self.assertEqual(5, entry.karma)
self.assertEqual([['DummyStrategy', '5', 'Dummy']], entry.reasons)
def test_log_accept(self):
req = Mock(environ={}, path_info="/foo", authname="anonymous", remote_addr="127.0.0.1")
DummyStrategy(self.env).configure(5)
FilterSystem(self.env).test(req, "John Doe", [(None, "Test")])
log = list(LogEntry.select(self.env))
self.assertEqual(1, len(log))
entry = log[0]
self.assertEqual("/foo", entry.path)
self.assertEqual("John Doe", entry.author)
self.assertEqual(False, entry.authenticated)
self.assertEqual("127.0.0.1", entry.ipnr)
self.assertEqual("Test", entry.content)
self.assertEqual(False, entry.rejected)
self.assertEqual(5, entry.karma)
self.assertEqual([], entry.reasons)
def _render_monitoring_entry(self, req, cat, page, entry_id):
req.perm.assert_permission('SPAM_MONITOR')
entry = LogEntry.fetch(self.env, entry_id)
if not entry:
raise HTTPNotFound('Log entry not found')
previous = entry.get_previous()
if previous:
add_link(req, 'prev', req.href.admin(cat, page, previous.id),
'Log Entry %d' % previous.id)
add_link(req, 'up', req.href.admin(cat, page), 'Log Entry List')
next = entry.get_next()
if next:
add_link(req, 'next', req.href.admin(cat, page, next.id),
'Log Entry %d' % next.id)
return {'entry': entry}
def _render_monitoring_entry(self, req, cat, page, entry_id):
req.perm.require('SPAM_MONITOR')
entry = LogEntry.fetch(self.env, entry_id)
if not entry:
raise HTTPNotFound(_("Log entry not found"))
previous = entry.get_previous()
if previous:
add_link(req, 'prev', req.href.admin(cat, page, previous.id),
_("Log Entry %(id)s", id=previous.id))
add_link(req, 'up', req.href.admin(cat, page), _("Log Entry List"))
next = entry.get_next()
if next:
add_link(req, 'next', req.href.admin(cat, page, next.id),
_("Log Entry %(id)s", id=next.id))
return {'entry': entry, 'train_only': self.train_only}
def train(self, req, ids, spam=True, delete=False):
environ = {}
for name, value in req.environ.items():
if not name.startswith('HTTP_'):
environ[name] = value
if not isinstance(ids, list):
ids = [ids]
for log_id in ids:
start = time.time()
entry = LogEntry.fetch(self.env, log_id)
if entry:
extint = "trainint"
self.log.debug('Marking as %s: %r submitted by "%s"',
spam and 'spam' or 'ham',
shorten_line(entry.content),
entry.author)
fakeenv = environ.copy()
for header in entry.headers.splitlines():
name, value = header.split(':', 1)
if name == 'Cookie': # breaks SimpleCookie somehow
continue
cgi_name = 'HTTP_%s' % name.strip().replace('-', '_').upper()
fakeenv[cgi_name] = value.strip()
fakeenv['REQUEST_METHOD'] = 'POST'
fakeenv['PATH_INFO'] = entry.path
fakeenv['wsgi.input'] = StringIO('')
fakeenv['REMOTE_ADDR'] = entry.ipnr
if entry.authenticated:
fakeenv['REMOTE_USER'] = entry.author
type = "spam" if spam else "ham"
for strategy in self.strategies:
status = "trainskip"
if (self.use_external and self.train_external) or not strategy.is_external():
tim = time.time()
extint = "trainext"
res = strategy.train(Request(fakeenv, None),
entry.author or 'anonymous',
entry.content, entry.ipnr, spam=spam)
tim = time.time()-tim
if tim > 3:
self.log.warn('Training %s took %d seconds to '
'complete.', strategy, tim)
if res == -1:
status = "trainerror"
elif res == -2:
status = "traincond"
elif res > 0:
status = "train"
count = entry.findreasonvalue(get_strategy_name(strategy))
if count:
spamstatus = count < 0
self._record_action(status, type, ("ok" if spamstatus == spam else "error"), strategy, tim)
else:
self._record_action(status, type, '', strategy, tim)
self._record_action(extint, type, ("ok" if entry.rejected == spam else "error"), '', time.time()-start)
entry.update(rejected=spam)
if delete:
self.delete(req, log_id, True)
def deleteobvious(self, req):
ids = LogEntry.getobvious(self.env)
if ids:
self.delete(req, ids, True)
status = "delete"
for strategy in self.strategies:
count = 0
name = get_strategy_name(strategy)
for r in reasons:
if r[0] == name:
count = as_int(r[1], 0)
break
if count:
spamstatus = count < 0
self._record_action(status, type, ("ok" if spamstatus == rejected else "error"), strategy, 0)
else:
self._record_action(status, type, '', strategy, 0)
self._record_action(status, type, '', '', 0)
LogEntry.purge(self.env, self.purge_age)
if score < self.min_karma:
self.log.debug('Rejecting submission %r by "%s" (%r) because it '
'earned only %d karma points (%d are required).',
abbrev, author, req.remote_addr, score, self.min_karma)
rejects = []
outreasons.sort()
for r in outreasons:
rejects.append(tag.li(r))
msg = tag.div(tag.ul(rejects), class_='message')
self.reject_handler.reject_content(req, msg)
def train(self, req, ids, spam=True, delete=False):
environ = {}
def test_good_karma(self):
req = Mock(environ={}, path_info="/foo", authname="anonymous", remote_addr="127.0.0.1")
DummyStrategy(self.env).configure(5)
FilterSystem(self.env).test(req, "John Doe", [(None, "Test")])
def test_log_reject(self):
req = Mock(environ={}, path_info="/foo", authname="anonymous", remote_addr="127.0.0.1")
DummyStrategy(self.env).configure(-5, "Blacklisted")
try:
FilterSystem(self.env).test(req, "John Doe", [(None, "Test")])
self.fail("Expected RejectContent exception")
except RejectContent, e:
pass
log = list(LogEntry.select(self.env))
self.assertEqual(1, len(log))
entry = log[0]
self.assertEqual("/foo", entry.path)
self.assertEqual("John Doe", entry.author)
self.assertEqual(False, entry.authenticated)
self.assertEqual("127.0.0.1", entry.ipnr)
self.assertEqual("Test", entry.content)
self.assertEqual(True, entry.rejected)
self.assertEqual(-5, entry.karma)
self.assertEqual(["DummyStrategy (-5): Blacklisted"], entry.reasons)
def test_log_accept(self):
req = Mock(environ={}, path_info="/foo", authname="anonymous", remote_addr="127.0.0.1")
DummyStrategy(self.env).configure(5)
FilterSystem(self.env).test(req, "John Doe", [(None, "Test")])