def archive(self, page=1, direction='dsc',
order_by='timestamp', format=None):
"messages archive"
filters = session.get('filter_by', None)
num_items = session.get('msgs_num_items', 50)
if direction == 'dsc':
sort = desc(order_by)
else:
sort = order_by
messages = self._get_archived().order_by(sort)
msgcount = self._get_msg_count(True)
query = UserFilter(Session, c.user, messages, True)
countquery = UserFilter(Session, c.user, msgcount, True)
messages = query.filter()
msgcount = countquery.filter()
if filters:
dynq = DynaQuery(Archive, messages, filters)
dynmsgq = DynaQuery(Archive, msgcount, filters)
messages = dynq.generate()
msgcount = dynmsgq.generate()
c.order_by = order_by
c.direction = direction
msgcount = msgcount.count()
pages = paginate.Page(messages, page=int(page),
items_per_page=num_items,
item_count=msgcount)
if format == 'json':
response.headers['Content-Type'] = 'application/json'
data = convert_to_json(pages,
direction=direction,
order_by=order_by,
section=None)
return data
c.page = pages
return render('/messages/archive.html')
def _get_data(self, format=None, success=None, errors=None):
"Get report data"
filters = session.get('filter_by', [])
query = Session.query(func.max(Message.timestamp).label('oldest'),
func.min(Message.timestamp).label('newest'))
uquery = UserFilter(Session, c.user, query)
query = uquery.filter()
# count = self._get_count()
countq = MsgCount(Session, c.user)
count = countq()
cachekey = u'savedfilters-%s' % c.user.username
sfq = Session.query(SavedFilter)\
.filter(SavedFilter.user == c.user)\
.options(FromCache('sql_cache_short', cachekey))
if self.invalidate:
sfq.invalidate()
savedfilters = sfq.all()
if filters:
dynq = DynaQuery(Message, query, filters)
query = dynq.generate()
dcountq = Session.query(func.count(Message.id).label('count'))
dcountqi = UserFilter(Session, c.user, dcountq)
dcountq = dcountqi.filter()
dyncq = DynaQuery(Message, dcountq, filters)
dcountq = dyncq.generate()
dcount = dcountq.one()
count = dcount.count
cachekey = u'report-aggregates-%s' % c.user.username
query = query.options(FromCache('sql_cache_short', cachekey))
if self.invalidate:
query.invalidate()
data = query.all()
saved_filters = [processfilters(filt, filters)
for filt in savedfilters]
if format is None:
return data, count, filters, saved_filters
else:
if format == 'json':
data = data[0]
filterdict = dict(FILTER_ITEMS)
filterbydict = dict(FILTER_BY)
active_filters = [dict(
filter_field=unicode(filterdict[filt['field']]),
filter_by=unicode(filterbydict[filt['filter']]),
filter_value=unicode(filt['value']))
for filt in filters]
try:
newest = data.newest.strftime("%Y-%m-%d %H:%M")
oldest = data.oldest.strftime("%Y-%m-%d %H:%M")
except AttributeError:
newest = ''
oldest = ''
datadict = dict(count=count, newest=newest, oldest=oldest)
jsondata = dict(success=success, data=datadict, errors=errors,
active_filters=active_filters,
saved_filters=saved_filters)
return jsondata
def archive(self, page=1, direction='dsc',
order_by='timestamp', format=None):
"messages archive"
filters = session.get('filter_by', None)
num_items = session.get('msgs_num_items', 50)
if direction == 'dsc':
sort = desc(order_by)
else:
sort = order_by
messages = self._get_archived().order_by(sort)
msgcount = self._get_msg_count(True)
query = UserFilter(Session, c.user, messages, True)
countquery = UserFilter(Session, c.user, msgcount, True)
messages = query.filter()
msgcount = countquery.filter()
if filters:
dynq = DynaQuery(Archive, messages, filters)
dynmsgq = DynaQuery(Archive, msgcount, filters)
messages = dynq.generate()
msgcount = dynmsgq.generate()
c.order_by = order_by
c.direction = direction
msgcount = msgcount.count()
pages = paginate.Page(messages, page=int(page),
items_per_page=num_items,
item_count=msgcount)
if format == 'json':
response.headers['Content-Type'] = 'application/json'
data = convert_to_json(pages,
direction=direction,
order_by=order_by,
section=None)
return data
c.page = pages
return render('/messages/archive.html')
def listing(self, page=1, direction='dsc', order_by='timestamp',
format=None):
"message listing"
filters = session.get('filter_by', None)
num_items = session.get('msgs_num_items', 50)
if direction == 'dsc':
sort = desc(order_by)
else:
sort = order_by
messages = get_messages().order_by(sort)
query = UserFilter(Session, c.user, messages)
messages = query.filter()
if filters:
msgcount = get_msg_count()
countquery = UserFilter(Session, c.user, msgcount)
msgcount = countquery.filter()
dynq = DynaQuery(Message, messages, filters)
dynmsgq = DynaQuery(Message, msgcount, filters)
messages = dynq.generate()
msgcount = dynmsgq.generate()
msgcount = msgcount.count()
else:
countquery = MsgCount(Session, c.user)
msgcount = countquery.count()
c.list_all = True
c.order_by = order_by
c.direction = direction
pages = paginate.Page(messages, page=int(page),
items_per_page=num_items,
item_count=msgcount)
if format == 'json':
response.headers['Content-Type'] = 'application/json'
data = convert_to_json(pages,
direction=direction,
order_by=order_by,
section=None)
return data
c.page = pages
return self.render('/messages/listing.html')
def listing(self,
page=1,
direction='dsc',
order_by='timestamp',
format=None):
"message listing"
filters = session.get('filter_by', None)
num_items = session.get('msgs_num_items', 50)
if direction == 'dsc':
sort = desc(order_by)
else:
sort = order_by
messages = get_messages().order_by(sort)
query = UserFilter(Session, c.user, messages)
messages = query.filter()
if filters:
msgcount = get_msg_count()
countquery = UserFilter(Session, c.user, msgcount)
msgcount = countquery.filter()
dynq = DynaQuery(Message, messages, filters)
dynmsgq = DynaQuery(Message, msgcount, filters)
messages = dynq.generate()
msgcount = dynmsgq.generate()
msgcount = msgcount.count()
else:
countquery = MsgCount(Session, c.user)
msgcount = countquery.count()
c.list_all = True
c.order_by = order_by
c.direction = direction
pages = paginate.Page(messages,
page=int(page),
items_per_page=num_items,
item_count=msgcount)
if format == 'json':
response.headers['Content-Type'] = 'application/json'
data = convert_to_json(pages,
direction=direction,
order_by=order_by,
section=None)
return data
c.page = pages
return self.render('/messages/listing.html')
def search(self, format=None):
"Search for messages"
q = request.GET.get('q', None)
if q is None:
redirect(url(controller='messages', action='listing'))
index = 'messages, messagesdelta, messages_rt'
action = request.GET.get('a', 'listing')
if not action in ['listing', 'quarantine', 'archive']:
action = 'listing'
if action == 'archive':
index = 'archive archivedelta'
try:
page = int(request.GET.get('page', 1))
except ValueError:
page = 1
num_items = session.get('msgs_search_num_results', 50)
conn = SphinxClient()
conn.SetMatchMode(SPH_MATCH_EXTENDED2)
#conn.SetSortMode(SPH_SORT_EXTENDED, "timestamp DESC")
if action == 'quarantine':
conn.SetFilter('isquarantined', [True,])
if page == 1:
conn.SetLimits(0, num_items, 500)
else:
offset = (page - 1) * num_items
conn.SetLimits(offset, num_items, 500)
if not c.user.is_superadmin:
filter_sphinx(Session, c.user, conn)
else:
conn.SetSelect('timestamp')
q = clean_sphinx_q(q)
results = conn.Query(q, index)
q = restore_sphinx_q(q)
if results and results['matches']:
#import pprint
#pprint.pprint(results)
ids = [hit['id'] for hit in results['matches']]
filters = session.get('filter_by', None)
if action == 'archive':
messages = self._get_archived().filter(
Archive.id.in_(ids))
query = UserFilter(Session, c.user, messages, True)
messages = query.filter()
if filters:
dynq = DynaQuery(Message, messages, filters)
messages = dynq.generate()
else:
messages = self._get_messages().filter(
Message.id.in_(ids))
query = UserFilter(Session, c.user, messages)
messages = query.filter()
if filters:
dynq = DynaQuery(Message, messages, filters)
messages = dynq.generate()
total_found = results['total']
search_time = results['time']
messages = messages.order_by(desc('timestamp'))
else:
print '=' * 100
print conn.GetLastError()
messages = []
results = dict(matches=[], total=0)
total_found = 0
search_time = 0
pages = paginator(dict(page=page, results_per_page=num_items,
total=results['total'],
items=len(results['matches']),
q=q))
if format == 'json':
response.headers['Content-Type'] = 'application/json'
data = dict(action=action,
total_found=total_found,
search_time=search_time,
paginator=pages,
items=[jsonify_msg_list(msg) for msg in messages])
return json.dumps(data)
c.messages = messages
c.action = action
c.total_found = total_found
c.search_time = search_time
c.page = pages
return render('/messages/searchresults.html')
def quarantine(self, page=1, direction='dsc', order_by='timestamp',
section=None, format=None):
"quarantined messages"
filters = session.get('filter_by', None)
num_items = session.get('msgs_num_items', 50)
if direction == 'dsc':
sort = desc(order_by)
else:
sort = order_by
messages = self._get_messages().filter(
Message.isquarantined == 1).order_by(sort)
msgcount = self._get_msg_count().filter(
Message.isquarantined == 1)
query = UserFilter(Session, c.user, messages)
countquery = UserFilter(Session, c.user, msgcount)
messages = query.filter()
msgcount = countquery.filter()
if section:
if section == 'spam':
messages = messages.filter(Message.spam == 1)
msgcount = messages.filter(Message.spam == 1)
else:
messages = messages.filter(Message.spam == 0)
msgcount = messages.filter(Message.spam == 0)
if filters:
dynq = DynaQuery(Message, messages, filters)
dynmsgq = DynaQuery(Message, msgcount, filters)
messages = dynq.generate()
msgcount = dynmsgq.generate()
c.order_by = order_by
c.direction = direction
c.section = section
msgcount = msgcount.count()
c.form = BulkReleaseForm(request.POST, csrf_context=session)
if request.POST:
choices = session.get('bulk_items', [])
else:
pages = paginate.Page(messages, page=int(page),
items_per_page=num_items,
item_count=msgcount)
choices = [(str(message.id), message.id)
for message in pages.items]
session['bulk_items'] = choices
session.save()
c.form.message_id.choices = choices
if request.POST and c.form.validate() and choices:
msgs = Session.query(Message.id,
Message.messageid,
Message.from_address,
Message.timestamp, Message.to_address,
Message.hostname)\
.filter(Message.id.in_(c.form.message_id.data))
query = UserFilter(Session, c.user, msgs)
msgs = query.filter()
localtmz = config.get('baruwa.timezone', 'Africa/Johannesburg')
formvals = (dict(release=c.form.release.data,
learn=c.form.learn.data,
salearn_as=c.form.learnas.data,
todelete=c.form.delete.data,
use_alt=c.form.usealt.data,
altrecipients=c.form.altrecipients.data,
message_id=msg.messageid,
from_address=msg.from_address,
date=convert_date(msg.timestamp, localtmz).strftime('%Y%m%d'),
to_address=msg.to_address,
hostname=msg.hostname,
mid=msg.id)
for msg in msgs)
taskset = TaskSet(tasks=(
process_quarantined_msg.subtask(
args=[formval],
options=dict(queue=formval['hostname'])
) for formval in formvals
)
)
if formvals:
try:
task = taskset.apply_async()
task.save(dbbackend)
session['bulk_items'] = []
if not 'taskids' in session:
session['taskids'] = []
session['taskids'].append(task.taskset_id)
session['bulkprocess-count'] = 1
session.save()
redirect(url('messages-bulk-process',
taskid=task.taskset_id))
except QueueNotFound:
flash_alert(_('The messages could not processed'
', try again later'))
elif request.POST and not c.form.validate():
flash_alert(_(u', '.join([unicode(c.form.errors[err][0])
for err in c.form.errors])))
pages = paginate.Page(messages, page=int(page),
items_per_page=num_items,
item_count=msgcount)
if format == 'json':
response.headers['Content-Type'] = 'application/json'
data = convert_to_json(pages,
direction=direction,
order_by=order_by,
section=section)
return data
c.page = pages
return render('/messages/quarantine.html')
def display(self, reportid, format=None):
"Display a report"
c.report_title = REPORTS[reportid]['title']
filters = session.get('filter_by', [])
if reportid in ['1', '2', '3', '4', '5', '6', '7', '8', '10']:
rquery = ReportQuery(c.user, reportid, filters)
query = rquery()
cachekey = u'reportquery-%s-%s' % (c.user.username, reportid)
query = query.options(FromCache('sql_cache_short', cachekey))
data = query[:10]
if format == 'png':
return self._generate_png(data, reportid)
if format == 'csv':
info = REPORTDL_MSG % dict(r=c.report_title, f='csv')
audit_log(c.user.username, 1, info, request.host,
request.remote_addr, datetime.now())
return self._generate_csv(data, reportid)
jsondata = [
dict(tooltip=getattr(item, 'address'),
y=getattr(item, REPORTS[reportid]['sort']),
stroke='black',
color=PIE_COLORS[index],
size=getattr(item, 'size'))
for index, item in enumerate(data)
]
template = '/reports/piereport.html'
if reportid == '9':
query = sa_scores(Session, c.user)
if filters:
dynq = DynaQuery(Message, query, filters)
query = dynq.generate()
cachekey = u'sascores-%s' % c.user.username
query = query.options(FromCache('sql_cache_short', cachekey))
data = query.all()
if format == 'json':
scores = []
counts = []
for row in data:
scores.append(
dict(value=int(row.score), text=str(row.score)))
counts.append(
dict(y=int(row.count),
tooltip=(_('Score ') + str(row.score) + ': ' +
str(row.count))))
jsondata = dict(scores=scores, count=counts)
elif format == 'png':
return self._generate_png(data, reportid)
else:
jsondata = {}
jsondata['labels'] = [{
'value': index + 1,
'text': str(item.score)
} for index, item in enumerate(data)]
jsondata['scores'] = [item.count for item in data]
template = '/reports/barreport.html'
if reportid == '10':
if format == 'json':
data = [[
item.address.strip(),
get_hostname(item.address.strip()),
country_flag(item.address.strip()), item.count,
format_byte_size(item.size)
] for item in data]
template = '/reports/relays.html'
if reportid == '11':
query = message_totals(Session, c.user)
if filters:
dynq = DynaQuery(Message, query, filters)
query = dynq.generate()
cachekey = u'msgtotals-%s' % c.user.username
query = query.options(FromCache('sql_cache_short', cachekey))
data = query.all()
if format == 'png':
return self._generate_png(data, reportid)
elif format == 'json':
dates = []
mail_total = []
spam_total = []
size_total = []
virus_total = []
for row in data:
dates.append(str(row.date))
mail_total.append(int(row.mail_total))
spam_total.append(int(row.spam_total))
virus_total.append(int(row.virus_total))
size_total.append(int(row.total_size))
jsondata = dict(
dates=[
dict(value=index + 1, text=date)
for index, date in enumerate(dates)
],
mail=[
dict(y=total,
tooltip=(_('Mail totals on ') + dates[index] +
': ' + str(total)))
for index, total in enumerate(mail_total)
],
spam=[
dict(y=total,
tooltip=(_('Spam totals on ') + dates[index] +
': ' + str(total)))
for index, total in enumerate(spam_total)
],
virii=[
dict(y=total,
tooltip=(_('Virus totals on ') + dates[index] +
': ' + str(total)))
for index, total in enumerate(virus_total)
],
volume=size_total,
mail_total=sum(mail_total),
spam_total=sum(spam_total),
virus_total=sum(virus_total),
volume_total=sum(size_total))
try:
vpct = "%.1f" % (
(1.0 * sum(virus_total) / sum(mail_total)) * 100)
spct = "%.1f" % (
(1.0 * sum(spam_total) / sum(mail_total)) * 100)
except ZeroDivisionError:
vpct = "0.0"
spct = "0.0"
jsondata['vpct'] = vpct
jsondata['spct'] = spct
data = [
dict(date=str(row.date),
mail_total=row.mail_total,
spam_total=row.spam_total,
virus_total=row.virus_total,
size_total=format_byte_size(row.total_size),
virus_percent="%.1f" %
((1.0 * int(row.virus_total) / int(row.mail_total)) *
100),
spam_percent="%.1f" %
((1.0 * int(row.spam_total) / int(row.mail_total)) *
100)) for row in data
]
elif format == 'csv':
info = REPORTDL_MSG % dict(r=c.report_title, f='csv')
audit_log(c.user.username, 1, info, request.host,
request.remote_addr, datetime.now())
return self._generate_csv(data, reportid)
else:
jsondata = dict(mail=[],
spam=[],
virus=[],
volume=[],
labels=[])
for index, item in enumerate(data):
jsondata['spam'].append(item.spam_total)
jsondata['mail'].append(item.mail_total)
jsondata['virus'].append(item.virus_total)
jsondata['volume'].append(item.total_size)
jsondata['labels'].append(
dict(text=str(item.date), value=index))
template = '/reports/listing.html'
if format == 'json':
response.headers['Content-Type'] = JSON_HEADER
return json.dumps(dict(items=list(data), pie_data=jsondata))
if format == 'pdf' and reportid != '9':
info = REPORTDL_MSG % dict(r=c.report_title, f='pdf')
audit_log(c.user.username, 1, info, request.host,
request.remote_addr, datetime.now())
return self._generate_pdf(data, reportid)
c.reportid = reportid
c.chart_data = json.dumps(jsondata)
c.top_items = data
c.active_filters = filters
c.saved_filters = []
c.FILTER_BY = FILTER_BY
c.FILTER_ITEMS = FILTER_ITEMS
c.form = FilterForm(request.POST, csrf_context=session)
info = REPORTVIEW_MSG % dict(r=c.report_title)
audit_log(c.user.username, 1, info, request.host, request.remote_addr,
datetime.now())
return render(template)
def _get_data(self, format=None, success=None, errors=None):
"Get report data"
filters = session.get('filter_by', [])
query = Session.query(
func.max(Message.timestamp).label('oldest'),
func.min(Message.timestamp).label('newest'))
uquery = UserFilter(Session, c.user, query)
query = uquery.filter()
# count = self._get_count()
countq = MsgCount(Session, c.user)
count = countq()
cachekey = u'savedfilters-%s' % c.user.username
sfq = Session.query(SavedFilter)\
.filter(SavedFilter.user == c.user)\
.options(FromCache('sql_cache_short', cachekey))
if self.invalidate:
sfq.invalidate()
savedfilters = sfq.all()
if filters:
dynq = DynaQuery(Message, query, filters)
query = dynq.generate()
dcountq = Session.query(func.count(Message.id).label('count'))
dcountqi = UserFilter(Session, c.user, dcountq)
dcountq = dcountqi.filter()
dyncq = DynaQuery(Message, dcountq, filters)
dcountq = dyncq.generate()
dcount = dcountq.one()
count = dcount.count
cachekey = u'report-aggregates-%s' % c.user.username
query = query.options(FromCache('sql_cache_short', cachekey))
if self.invalidate:
query.invalidate()
data = query.all()
saved_filters = [
processfilters(filt, filters) for filt in savedfilters
]
if format is None:
return data, count, filters, saved_filters
else:
if format == 'json':
data = data[0]
filterdict = dict(FILTER_ITEMS)
filterbydict = dict(FILTER_BY)
active_filters = [
dict(filter_field=filterdict[filt['field']],
filter_by=filterbydict[filt['filter']],
filter_value=filt['value']) for filt in filters
]
try:
newest = data.newest.strftime("%Y-%m-%d %H:%M")
oldest = data.oldest.strftime("%Y-%m-%d %H:%M")
except AttributeError:
newest = ''
oldest = ''
datadict = dict(count=count, newest=newest, oldest=oldest)
jsondata = dict(success=success,
data=datadict,
errors=errors,
active_filters=active_filters,
saved_filters=saved_filters)
return jsondata
def search(self, format=None):
"Search for messages"
q = request.GET.get('q', None)
if q is None:
redirect(url(controller='messages', action='listing'))
index = 'messages, messages_rt'
action = request.GET.get('a', 'listing')
if not action in ['listing', 'quarantine', 'archive']:
action = 'listing'
if action == 'archive':
index = 'archive'
try:
page = int(request.GET.get('page', 1))
except ValueError:
page = 1
num_items = session.get('msgs_search_num_results', 50)
conn = SphinxClient()
conn.SetMatchMode(SPH_MATCH_EXTENDED2)
#conn.SetSortMode(SPH_SORT_EXTENDED, "timestamp DESC")
if action == 'quarantine':
conn.SetFilter('isquarantined', [True,])
if page == 1:
conn.SetLimits(0, num_items, 500)
else:
offset = (page - 1) * num_items
conn.SetLimits(offset, num_items, 500)
if not c.user.is_superadmin:
filter_sphinx(Session, c.user, conn)
else:
conn.SetSelect('timestamp')
q = clean_sphinx_q(q)
results = conn.Query(q, index)
q = restore_sphinx_q(q)
if results and results['matches']:
#import pprint
#pprint.pprint(results)
ids = [hit['id'] for hit in results['matches']]
filters = session.get('filter_by', None)
if index == 'archive':
messages = self._get_archived().filter(
Archive.id.in_(ids))
query = UserFilter(Session, c.user, messages, True)
messages = query.filter()
if filters:
dynq = DynaQuery(Message, messages, filters)
messages = dynq.generate()
else:
messages = self._get_messages().filter(
Message.id.in_(ids))
query = UserFilter(Session, c.user, messages)
messages = query.filter()
if filters:
dynq = DynaQuery(Message, messages, filters)
messages = dynq.generate()
total_found = results['total']
search_time = results['time']
messages = messages.order_by(desc('timestamp'))
else:
print '=' * 100
print conn.GetLastError()
messages = []
results = dict(matches=[], total=0)
total_found = 0
search_time = 0
pages = paginator(dict(page=page, results_per_page=num_items,
total=results['total'],
items=len(results['matches']),
q=q))
if format == 'json':
response.headers['Content-Type'] = 'application/json'
data = dict(action=action,
total_found=total_found,
search_time=search_time,
paginator=pages,
items=[jsonify_msg_list(msg) for msg in messages])
return json.dumps(data)
c.messages = messages
c.action = action
c.total_found = total_found
c.search_time = search_time
c.page = pages
return render('/messages/searchresults.html')
def quarantine(self, page=1, direction='dsc', order_by='timestamp',
section=None, format=None):
"quarantined messages"
filters = session.get('filter_by', None)
num_items = session.get('msgs_num_items', 50)
if direction == 'dsc':
sort = desc(order_by)
else:
sort = order_by
messages = self._get_messages().filter(
Message.isquarantined == 1).order_by(sort)
msgcount = self._get_msg_count().filter(
Message.isquarantined == 1)
query = UserFilter(Session, c.user, messages)
countquery = UserFilter(Session, c.user, msgcount)
messages = query.filter()
msgcount = countquery.filter()
if section:
if section == 'spam':
messages = messages.filter(Message.spam == 1)
msgcount = messages.filter(Message.spam == 1)
else:
messages = messages.filter(Message.spam == 0)
msgcount = messages.filter(Message.spam == 0)
if filters:
dynq = DynaQuery(Message, messages, filters)
dynmsgq = DynaQuery(Message, msgcount, filters)
messages = dynq.generate()
msgcount = dynmsgq.generate()
c.order_by = order_by
c.direction = direction
c.section = section
msgcount = msgcount.count()
c.form = BulkReleaseForm(request.POST, csrf_context=session)
if request.POST:
choices = session.get('bulk_items', [])
else:
pages = paginate.Page(messages, page=int(page),
items_per_page=num_items,
item_count=msgcount)
choices = [(str(message.id), message.id)
for message in pages.items]
session['bulk_items'] = choices
session.save()
c.form.message_id.choices = choices
if request.POST and c.form.validate() and choices:
msgs = Session.query(Message.id,
Message.messageid,
Message.from_address,
Message.date, Message.to_address,
Message.hostname)\
.filter(Message.id.in_(c.form.message_id.data))
query = UserFilter(Session, c.user, msgs)
msgs = query.filter()
formvals = (dict(release=c.form.release.data,
learn=c.form.learn.data,
salearn_as=c.form.learnas.data,
todelete=c.form.delete.data,
use_alt=c.form.usealt.data,
altrecipients=c.form.altrecipients.data,
message_id=msg.messageid,
from_address=msg.from_address,
date=str(msg.date), to_address=msg.to_address,
hostname=msg.hostname,
mid=msg.id)
for msg in msgs)
taskset = TaskSet(tasks=(
process_quarantined_msg.subtask(
args=[formval],
options=dict(queue=formval['hostname'])
) for formval in formvals
)
)
if formvals:
try:
task = taskset.apply_async()
task.save(dbbackend)
session['bulk_items'] = []
if not 'taskids' in session:
session['taskids'] = []
session['taskids'].append(task.taskset_id)
session['bulkprocess-count'] = 1
session.save()
redirect(url('messages-bulk-process',
taskid=task.taskset_id))
except QueueNotFound:
flash_alert(_('The messages could not processed'
', try again later'))
elif request.POST and not c.form.validate():
flash_alert(_(', '.join([c.form.errors[err][0]
for err in c.form.errors])))
pages = paginate.Page(messages, page=int(page),
items_per_page=num_items,
item_count=msgcount)
if format == 'json':
response.headers['Content-Type'] = 'application/json'
data = convert_to_json(pages,
direction=direction,
order_by=order_by,
section=section)
return data
c.page = pages
return render('/messages/quarantine.html')
def display(self, reportid, format=None):
"Display a report"
try:
c.report_title = REPORTS[reportid]['title']
except KeyError:
abort(404)
filters = session.get('filter_by', [])
if reportid in ['1', '2', '3', '4', '5', '6', '7', '8', '10']:
rquery = ReportQuery(c.user, reportid, filters)
query = rquery()
cachekey = u'reportquery-%s-%s' % (c.user.username, reportid)
query = query.options(FromCache('sql_cache_short', cachekey))
data = query[:10]
if format == 'png':
return self._generate_png(data, reportid)
if format == 'csv':
info = REPORTDL_MSG % dict(r=c.report_title, f='csv')
audit_log(c.user.username,
1, unicode(info), request.host,
request.remote_addr, arrow.utcnow().datetime)
return self._generate_csv(data, reportid)
jsondata = [dict(tooltip=getattr(item, 'address'),
y=getattr(item, REPORTS[reportid]['sort']),
stroke='black', color=PIE_COLORS[index],
size=getattr(item, 'size'))
for index, item in enumerate(data)]
template = '/reports/piereport.html'
if reportid == '9':
query = sa_scores(Session, c.user)
if filters:
dynq = DynaQuery(Message, query, filters)
query = dynq.generate()
cachekey = u'sascores-%s' % c.user.username
query = query.options(FromCache('sql_cache_short', cachekey))
data = query.all()
if format == 'json':
scores = []
counts = []
for row in data:
scores.append(dict(value=int(row.score),
text=str(row.score)))
counts.append(dict(y=int(row.count), tooltip=(_('Score ') +
str(row.score) + ': ' + str(row.count))))
jsondata = dict(scores=scores, count=counts)
elif format == 'png':
return self._generate_png(data, reportid)
else:
jsondata = {}
jsondata['labels'] = [{'value': index + 1,
'text': str(item.score)}
for index, item in enumerate(data)]
jsondata['scores'] = [item.count for item in data]
template = '/reports/barreport.html'
if reportid == '10':
if format == 'json':
data = [[item.address.strip(),
get_hostname(item.address.strip()),
country_flag(item.address.strip()),
item.count, item.size]
for item in data]
template = '/reports/relays.html'
if reportid == '11':
query = message_totals(Session, c.user)
if filters:
dynq = DynaQuery(Message, query, filters)
query = dynq.generate()
cachekey = u'msgtotals-%s' % c.user.username
query = query.options(FromCache('sql_cache_short', cachekey))
data = query.all()
if format == 'png':
return self._generate_png(data, reportid)
elif format == 'json':
dates = []
mail_total = []
spam_total = []
size_total = []
virus_total = []
for row in data:
dates.append(str(row.ldate))
mail_total.append(int(row.mail_total))
spam_total.append(int(row.spam_total))
virus_total.append(int(row.virus_total))
size_total.append(int(row.total_size))
jsondata = dict(dates=[dict(value=index + 1, text=date)
for index, date in enumerate(dates)],
mail=[dict(y=total,
tooltip=(_('Mail totals on ') +
dates[index] + ': ' + str(total)))
for index, total in enumerate(mail_total)],
spam=[dict(y=total,
tooltip=(_('Spam totals on ') +
dates[index] + ': ' + str(total)))
for index, total in enumerate(spam_total)],
virii=[dict(y=total,
tooltip=(_('Virus totals on ') +
dates[index] + ': ' + str(total)))
for index, total in enumerate(virus_total)],
volume=size_total, mail_total=sum(mail_total),
spam_total=sum(spam_total),
virus_total=sum(virus_total),
volume_total=sum(size_total))
try:
vpct = "%.1f" % ((1.0 * sum(virus_total) /
sum(mail_total)) * 100)
spct = "%.1f" % ((1.0 * sum(spam_total) /
sum(mail_total)) * 100)
except ZeroDivisionError:
vpct = "0.0"
spct = "0.0"
jsondata['vpct'] = vpct
jsondata['spct'] = spct
data = [dict(date=str(row.ldate),
mail_total=row.mail_total,
spam_total=row.spam_total,
virus_total=row.virus_total,
size_total=format_byte_size(row.total_size),
virus_percent="%.1f" % ((1.0 * int(row.virus_total) /
int(row.mail_total)) * 100),
spam_percent="%.1f" % ((1.0 * int(row.spam_total) /
int(row.mail_total)) * 100)) for row in data]
elif format == 'csv':
info = REPORTDL_MSG % dict(r=c.report_title, f='csv')
audit_log(c.user.username,
1, unicode(info), request.host,
request.remote_addr, arrow.utcnow().datetime)
return self._generate_csv(data, reportid)
else:
jsondata = dict(mail=[],
spam=[],
virus=[],
volume=[],
labels=[])
for index, item in enumerate(data):
jsondata['spam'].append(item.spam_total)
jsondata['mail'].append(item.mail_total)
jsondata['virus'].append(item.virus_total)
jsondata['volume'].append(item.total_size)
jsondata['labels'].append(dict(text=str(item.ldate),
value=index))
template = '/reports/listing.html'
if format == 'json':
response.headers['Content-Type'] = JSON_HEADER
return json.dumps(dict(items=list(data), pie_data=jsondata))
if format == 'pdf' and reportid != '9':
info = REPORTDL_MSG % dict(r=c.report_title, f='pdf')
audit_log(c.user.username,
1, unicode(info), request.host,
request.remote_addr, arrow.utcnow().datetime)
return self._generate_pdf(data, reportid)
c.reportid = reportid
c.chart_data = json.dumps(jsondata)
c.top_items = data
c.active_filters = filters
c.saved_filters = []
c.FILTER_BY = FILTER_BY
c.FILTER_ITEMS = FILTER_ITEMS
c.form = FilterForm(request.POST, csrf_context=session)
info = REPORTVIEW_MSG % dict(r=c.report_title)
audit_log(c.user.username,
1, unicode(info), request.host,
request.remote_addr, arrow.utcnow().datetime)
return self.render(template)
def search(self, format=None):
"Search for messages"
qry = request.GET.get('q', None)
if qry is None:
redirect(url(controller='messages', action='listing'))
index = 'messages, messagesdelta, messages_rt'
action = request.GET.get('a', 'listing')
if action not in ['listing', 'quarantine', 'archive']:
action = 'listing'
if action == 'archive':
index = 'archive archivedelta'
try:
page = int(request.GET.get('page', 1))
except ValueError:
page = 1
num_items = session.get('msgs_search_num_results', 50)
conn = SphinxClient()
sphinxopts = extract_sphinx_opts(config['sphinx.url'])
conn.SetServer(sphinxopts.get('host', '127.0.0.1'))
conn.SetMatchMode(SPH_MATCH_EXTENDED2)
if action == 'quarantine':
conn.SetFilter('isquarantined', [
True,
])
if page == 1:
conn.SetLimits(0, num_items, 500)
else:
offset = (page - 1) * num_items
conn.SetLimits(offset, num_items, 500)
if not c.user.is_superadmin:
filter_sphinx(Session, c.user, conn)
else:
conn.SetSelect('timestamp')
qry = clean_sphinx_q(qry)
try:
results = conn.Query(qry, index)
except (socket.timeout, struct.error):
redirect(request.path_qs)
qry = restore_sphinx_q(qry)
if results and results['matches']:
ids = [hit['id'] for hit in results['matches']]
filters = session.get('filter_by', None)
if action == 'archive':
messages = get_archived().filter(Archive.id.in_(ids))
query = UserFilter(Session, c.user, messages, True)
messages = query.filter()
if filters:
dynq = DynaQuery(Message, messages, filters)
messages = dynq.generate()
else:
messages = get_messages().filter(Message.id.in_(ids))
query = UserFilter(Session, c.user, messages)
messages = query.filter()
if filters:
dynq = DynaQuery(Message, messages, filters)
messages = dynq.generate()
total_found = results['total']
search_time = results['time']
messages = messages.order_by(desc('timestamp'))
else:
messages = []
results = dict(matches=[], total=0)
total_found = 0
search_time = 0
pages = paginator(
dict(page=page,
results_per_page=num_items,
total=results['total'],
items=len(results['matches']),
q=qry))
if format == 'json':
response.headers['Content-Type'] = 'application/json'
data = dict(action=action,
total_found=total_found,
search_time=search_time,
paginator=pages,
items=[jsonify_msg_list(msg) for msg in messages])
return json.dumps(data)
c.messages = messages
c.action = action
c.total_found = total_found
c.search_time = search_time
c.page = pages
return self.render('/messages/searchresults.html')
def quarantine(self,
page=1,
direction='dsc',
order_by='timestamp',
section=None,
format=None):
"quarantined messages"
filters = session.get('filter_by', None)
num_items = session.get('msgs_num_items', 50)
if direction == 'dsc':
sort = desc(order_by)
else:
sort = order_by
messages = get_messages().filter(
Message.isquarantined == 1).order_by(sort)
msgcount = get_msg_count().filter(Message.isquarantined == 1)
query = UserFilter(Session, c.user, messages)
countquery = UserFilter(Session, c.user, msgcount)
messages = query.filter()
msgcount = countquery.filter()
if section:
if section == 'spam':
messages = messages.filter(Message.spam == 1)
msgcount = messages.filter(Message.spam == 1)
else:
messages = messages.filter(Message.spam == 0)
msgcount = messages.filter(Message.spam == 0)
if filters:
dynq = DynaQuery(Message, messages, filters)
dynmsgq = DynaQuery(Message, msgcount, filters)
messages = dynq.generate()
msgcount = dynmsgq.generate()
c.order_by = order_by
c.direction = direction
c.section = section
msgcount = msgcount.count()
c.form = BulkReleaseForm(request.POST, csrf_context=session)
if request.method == 'POST':
choices = session.get('bulk_items', [])
else:
pages = paginate.Page(messages,
page=int(page),
items_per_page=num_items,
item_count=msgcount)
choices = [(str(message.id), message.id)
for message in pages.items]
session['bulk_items'] = choices
session.save()
c.form.message_id.choices = choices
if request.method == 'POST' and c.form.validate() and choices:
msgs = Session.query(Message.id,
Message.messageid,
Message.from_address,
Message.timestamp, Message.to_address,
Message.hostname, Message.msgfiles)\
.filter(Message.id.in_(c.form.message_id.data))
query = UserFilter(Session, c.user, msgs)
msgs = query.filter()
localtmz = config.get('baruwa.timezone', 'Africa/Johannesburg')
formvals = (dict(release=c.form.release.data,
learn=c.form.learn.data,
salearn_as=c.form.learnas.data,
todelete=c.form.delete.data,
use_alt=c.form.usealt.data,
altrecipients=c.form.altrecipients.data,
message_id=msg.messageid,
from_address=msg.from_address,
date=convert_date(msg.timestamp,
localtmz).strftime('%Y%m%d'),
msgfiles=msg.msgfiles,
to_address=msg.to_address,
hostname=msg.hostname,
mid=msg.id,
num=num_items) for msg in msgs)
if formvals:
try:
subtasks = [
process_quarantined_msg.subtask(
args=[formval],
options=dict(
routing_key=system_hostname() if asbool(
config.get('ms.quarantine.shared', 'false')
) else formval['hostname']))
for formval in formvals
]
task = group(subtasks).apply_async()
task.save(backend=RBACKEND)
session['bulk_items'] = []
if 'taskids' not in session:
session['taskids'] = []
session['taskids'].append(task.id)
session['bulkprocess-count'] = 1
session.save()
redirect(url('messages-bulk-process', taskid=task.id))
except (QueueNotFound, OperationalError, IndexError):
flash_alert(
_('The messages could not processed'
', try again later'))
elif request.method == 'POST' and not c.form.validate():
try:
flash_alert(
_(u', '.join([
unicode(c.form.errors[err][0]) for err in c.form.errors
])))
except IndexError:
flash_alert(
_('The messages could not processed'
', an Unknown error occured.'))
pages = paginate.Page(messages,
page=int(page),
items_per_page=num_items,
item_count=msgcount)
if format == 'json':
response.headers['Content-Type'] = 'application/json'
data = convert_to_json(pages,
direction=direction,
order_by=order_by,
section=section)
return data
c.page = pages
return self.render('/messages/quarantine.html')