def index(self, format=None):
"return recent messages"
num_items = session.get('msgs_num_items', 50)
query = self._get_messagez().order_by(desc('timestamp'))
if ('X-Last-Timestamp' in request.headers and
request.headers['X-Last-Timestamp']):
tstmp = request.headers.get('X-Last-Timestamp')
query = query.filter(Message.timestamp > tstmp)
uquery = UserFilter(Session, c.user, query)
query = uquery.filter()
items = query[:num_items]
if format == 'json':
response.headers['Content-Type'] = 'application/json'
msgs = [item.json for item in items]
tmp = dict(
totals=c.baruwa_totals,
inbound=c.baruwa_inbound,
outbound=c.baruwa_outbound,
items=msgs,
num_items=num_items
)
if c.user.is_admin:
tmp['status'] = c.baruwa_status
return json.dumps(tmp)
c.messages = items
c.num_items = num_items
return render('/messages/index.html')
def archive(self, page=1, direction='dsc',
order_by='timestamp', format=None):
"messages archive"
filters = session.get('filter_by', None)
num_items = session.get('msgs_num_items', 50)
if direction == 'dsc':
sort = desc(order_by)
else:
sort = order_by
messages = self._get_archived().order_by(sort)
msgcount = self._get_msg_count(True)
query = UserFilter(Session, c.user, messages, True)
countquery = UserFilter(Session, c.user, msgcount, True)
messages = query.filter()
msgcount = countquery.filter()
if filters:
dynq = DynaQuery(Archive, messages, filters)
dynmsgq = DynaQuery(Archive, msgcount, filters)
messages = dynq.generate()
msgcount = dynmsgq.generate()
c.order_by = order_by
c.direction = direction
msgcount = msgcount.count()
pages = paginate.Page(messages, page=int(page),
items_per_page=num_items,
item_count=msgcount)
if format == 'json':
response.headers['Content-Type'] = 'application/json'
data = convert_to_json(pages,
direction=direction,
order_by=order_by,
section=None)
return data
c.page = pages
return render('/messages/archive.html')
def index(self, format=None):
"return recent messages"
num_items = session.get('msgs_num_items', 50)
query = self._get_messagez().order_by(desc('timestamp'))
if ('X-Last-Timestamp' in request.headers and
request.headers['X-Last-Timestamp']):
tstmp = request.headers.get('X-Last-Timestamp')
query = query.filter(Message.timestamp > tstmp)
uquery = UserFilter(Session, c.user, query)
query = uquery.filter()
items = query[:num_items]
if format == 'json':
response.headers['Content-Type'] = 'application/json'
msgs = [item.json for item in items]
tmp = dict(
totals=c.baruwa_totals,
inbound=c.baruwa_inbound,
outbound=c.baruwa_outbound,
items=msgs,
num_items=num_items
)
if c.user.is_admin:
tmp['status'] = c.baruwa_status
return json.dumps(tmp)
c.messages = items
c.num_items = num_items
return render('/messages/index.html')
def _get_data(self, format=None, success=None, errors=None):
"Get report data"
filters = session.get('filter_by', [])
query = Session.query(func.max(Message.timestamp).label('oldest'),
func.min(Message.timestamp).label('newest'))
uquery = UserFilter(Session, c.user, query)
query = uquery.filter()
# count = self._get_count()
countq = MsgCount(Session, c.user)
count = countq()
cachekey = u'savedfilters-%s' % c.user.username
sfq = Session.query(SavedFilter)\
.filter(SavedFilter.user == c.user)\
.options(FromCache('sql_cache_short', cachekey))
if self.invalidate:
sfq.invalidate()
savedfilters = sfq.all()
if filters:
dynq = DynaQuery(Message, query, filters)
query = dynq.generate()
dcountq = Session.query(func.count(Message.id).label('count'))
dcountqi = UserFilter(Session, c.user, dcountq)
dcountq = dcountqi.filter()
dyncq = DynaQuery(Message, dcountq, filters)
dcountq = dyncq.generate()
dcount = dcountq.one()
count = dcount.count
cachekey = u'report-aggregates-%s' % c.user.username
query = query.options(FromCache('sql_cache_short', cachekey))
if self.invalidate:
query.invalidate()
data = query.all()
saved_filters = [processfilters(filt, filters)
for filt in savedfilters]
if format is None:
return data, count, filters, saved_filters
else:
if format == 'json':
data = data[0]
filterdict = dict(FILTER_ITEMS)
filterbydict = dict(FILTER_BY)
active_filters = [dict(
filter_field=unicode(filterdict[filt['field']]),
filter_by=unicode(filterbydict[filt['filter']]),
filter_value=unicode(filt['value']))
for filt in filters]
try:
newest = data.newest.strftime("%Y-%m-%d %H:%M")
oldest = data.oldest.strftime("%Y-%m-%d %H:%M")
except AttributeError:
newest = ''
oldest = ''
datadict = dict(count=count, newest=newest, oldest=oldest)
jsondata = dict(success=success, data=datadict, errors=errors,
active_filters=active_filters,
saved_filters=saved_filters)
return jsondata
def _get_message(self, messageid):
"utility to return message"
try:
cachekey = u"msgid-%s" % messageid
q = Session.query(Message).filter(Message.id == messageid)
if not c.user.is_superadmin:
uquery = UserFilter(Session, c.user, q)
q = uquery.filter()
q = q.options(FromCache('sql_cache_long', cachekey))
if self.invalidate:
q.invalidate()
message = q.one()
except (NoResultFound, MultipleResultsFound):
message = None
return message
def _get_archive(self, messageid):
"utility to return message"
try:
cachekey = u"archiveid-%s" % messageid
q = Session.query(Archive).filter(Archive.id == messageid)
if c.user.account_type != 1:
uquery = UserFilter(Session, c.user, q)
q = uquery.filter()
q = q.options(FromCache('sql_cache_long', cachekey))
if self.invalidate:
q.invalidate()
message = q.one()
except (NoResultFound, MultipleResultsFound):
message = None
return message
def _get_message(self, messageid):
"utility to return message"
try:
cachekey = u"msgid-%s" % messageid
q = Session.query(Message).filter(Message.id == messageid)
if not c.user.is_superadmin:
uquery = UserFilter(Session, c.user, q)
q = uquery.filter()
q = q.options(FromCache('sql_cache_long', cachekey))
if self.invalidate:
q.invalidate()
message = q.one()
except (NoResultFound, MultipleResultsFound):
message = None
return message
def _get_archive(self, messageid):
"utility to return message"
try:
cachekey = u"archiveid-%s" % messageid
q = Session.query(Archive).filter(Archive.id == messageid)
if c.user.account_type != 1:
uquery = UserFilter(Session, c.user, q)
q = uquery.filter()
q = q.options(FromCache('sql_cache_long', cachekey))
if self.invalidate:
q.invalidate()
message = q.one()
except (NoResultFound, MultipleResultsFound):
message = None
return message
def mailq_detail(self, queueid):
"View a queued message's details"
query = Session.query(MailQueueItem)
uquery = UserFilter(Session, c.user, query, model=MailQueueItem)
query = uquery.filter()
try:
mailqitem = query.filter(MailQueueItem.id == queueid).one()
except NoResultFound:
#abort(404)
flash_alert(_('The requested queued message was not found.'))
redirect(url('mailq-status'))
c.mailqitem = mailqitem
c.form = MailQueueProcessForm(request.POST, csrf_context=session)
session['queue_choices'] = [(queueid, queueid),]
session.save()
return render('/status/detail.html')
def mailq_detail(self, queueid):
"View a queued message's details"
query = Session.query(MailQueueItem)
uquery = UserFilter(Session, c.user, query, model=MailQueueItem)
query = uquery.filter()
try:
mailqitem = query.filter(MailQueueItem.id == queueid).one()
except NoResultFound:
#abort(404)
flash_alert(_('The requested queued message was not found.'))
redirect(url('mailq-status'))
c.mailqitem = mailqitem
c.form = MailQueueProcessForm(request.POST, csrf_context=session)
session['queue_choices'] = [(queueid, queueid),]
session.save()
return render('/status/detail.html')
def mailq(self,
serverid=None,
queue='inbound',
page=1,
direction='dsc',
order_by='timestamp'):
"Display mailqueue"
server = None
if not serverid is None:
server = self._get_server(serverid)
if not server:
abort(404)
c.queue = queue
c.server = server
c.direction = direction
c.order_by = desc(order_by) if direction == 'dsc' else order_by
if queue == 'inbound':
qdirection = 1
else:
qdirection = 2
num_items = session.get('mailq_num_items', 10)
query = Session.query(MailQueueItem).filter(
MailQueueItem.direction == qdirection).order_by(order_by)
if server:
query = query.filter(MailQueueItem.hostname == server.hostname)
uquery = UserFilter(Session, c.user, query, model=MailQueueItem)
query = uquery.filter()
c.form = MailQueueProcessForm(request.POST, csrf_context=session)
pages = paginate.Page(query, page=int(page), items_per_page=num_items)
choices = [(str(item.id), item.id) for item in pages.items]
session['queue_choices'] = choices
session.save()
c.page = paginate.Page(query, page=int(page), items_per_page=num_items)
return render('/status/mailq.html')
def archive(self, page=1, direction='dsc',
order_by='timestamp', format=None):
"messages archive"
filters = session.get('filter_by', None)
num_items = session.get('msgs_num_items', 50)
if direction == 'dsc':
sort = desc(order_by)
else:
sort = order_by
messages = self._get_archived().order_by(sort)
msgcount = self._get_msg_count(True)
query = UserFilter(Session, c.user, messages, True)
countquery = UserFilter(Session, c.user, msgcount, True)
messages = query.filter()
msgcount = countquery.filter()
if filters:
dynq = DynaQuery(Archive, messages, filters)
dynmsgq = DynaQuery(Archive, msgcount, filters)
messages = dynq.generate()
msgcount = dynmsgq.generate()
c.order_by = order_by
c.direction = direction
msgcount = msgcount.count()
pages = paginate.Page(messages, page=int(page),
items_per_page=num_items,
item_count=msgcount)
if format == 'json':
response.headers['Content-Type'] = 'application/json'
data = convert_to_json(pages,
direction=direction,
order_by=order_by,
section=None)
return data
c.page = pages
return render('/messages/archive.html')
def message_totals_report(model, num_of_days):
"Message totals report"
query = Session.query(Message.date,
func.count(Message.date).label('mail_total'),
func.sum(case([(Message.virusinfected > 0, 1)],
else_=0)).label('virus_total'),
func.sum(case([(and_(Message.virusinfected ==
0, Message.spam > 0), 1)], else_=0))\
.label('spam_total'), func.sum(Message.size)\
.label('total_size')).group_by(Message.date)\
.order_by(desc(Message.date))
if isinstance(model, User):
uquery = UserFilter(Session, model, query)
query = uquery.filter()
else:
query = query.filter(Domain.name == model.name)
if int(num_of_days) > 0:
numofdays = datetime.timedelta(days=num_of_days)
startdate = now().date() - numofdays
query = query.filter(Message.timestamp > str(startdate))
data = query.all()
return data
def mailq(self, serverid=None, queue='inbound', page=1, direction='dsc',
order_by='timestamp'):
"Display mailqueue"
server = None
if not serverid is None:
server = self._get_server(serverid)
if not server:
abort(404)
c.queue = queue
c.server = server
c.direction = direction
c.order_by = desc(order_by) if direction == 'dsc' else order_by
if queue == 'inbound':
qdirection = 1
else:
qdirection = 2
num_items = session.get('mailq_num_items', 10)
query = Session.query(MailQueueItem).filter(
MailQueueItem.direction == qdirection).order_by(order_by)
if server:
query = query.filter(MailQueueItem.hostname == server.hostname)
uquery = UserFilter(Session, c.user, query, model=MailQueueItem)
query = uquery.filter()
c.form = MailQueueProcessForm(request.POST, csrf_context=session)
pages = paginate.Page(query, page=int(page), items_per_page=num_items)
choices = [(str(item.id), item.id) for item in pages.items]
session['queue_choices'] = choices
session.save()
c.page = paginate.Page(query, page=int(page),
items_per_page=num_items)
return render('/status/mailq.html')
def listing(self,
page=1,
direction='dsc',
order_by='timestamp',
format=None):
"message listing"
filters = session.get('filter_by', None)
num_items = session.get('msgs_num_items', 50)
if direction == 'dsc':
sort = desc(order_by)
else:
sort = order_by
messages = get_messages().order_by(sort)
query = UserFilter(Session, c.user, messages)
messages = query.filter()
if filters:
msgcount = get_msg_count()
countquery = UserFilter(Session, c.user, msgcount)
msgcount = countquery.filter()
dynq = DynaQuery(Message, messages, filters)
dynmsgq = DynaQuery(Message, msgcount, filters)
messages = dynq.generate()
msgcount = dynmsgq.generate()
msgcount = msgcount.count()
else:
countquery = MsgCount(Session, c.user)
msgcount = countquery.count()
c.list_all = True
c.order_by = order_by
c.direction = direction
pages = paginate.Page(messages,
page=int(page),
items_per_page=num_items,
item_count=msgcount)
if format == 'json':
response.headers['Content-Type'] = 'application/json'
data = convert_to_json(pages,
direction=direction,
order_by=order_by,
section=None)
return data
c.page = pages
return self.render('/messages/listing.html')
def listing(self, page=1, direction='dsc', order_by='timestamp',
format=None):
"message listing"
filters = session.get('filter_by', None)
num_items = session.get('msgs_num_items', 50)
if direction == 'dsc':
sort = desc(order_by)
else:
sort = order_by
messages = get_messages().order_by(sort)
query = UserFilter(Session, c.user, messages)
messages = query.filter()
if filters:
msgcount = get_msg_count()
countquery = UserFilter(Session, c.user, msgcount)
msgcount = countquery.filter()
dynq = DynaQuery(Message, messages, filters)
dynmsgq = DynaQuery(Message, msgcount, filters)
messages = dynq.generate()
msgcount = dynmsgq.generate()
msgcount = msgcount.count()
else:
countquery = MsgCount(Session, c.user)
msgcount = countquery.count()
c.list_all = True
c.order_by = order_by
c.direction = direction
pages = paginate.Page(messages, page=int(page),
items_per_page=num_items,
item_count=msgcount)
if format == 'json':
response.headers['Content-Type'] = 'application/json'
data = convert_to_json(pages,
direction=direction,
order_by=order_by,
section=None)
return data
c.page = pages
return self.render('/messages/listing.html')
def command(self):
"run command"
self.init()
import baruwa
here = os.path.dirname(
os.path.dirname(os.path.abspath(baruwa.__file__))
)
path = os.path.join(here, 'baruwa', 'templates')
logo = os.path.join(here, 'baruwa', 'public', 'imgs', 'logo.png')
localedir = os.path.join(here, 'baruwa', 'i18n')
cache_dir = os.path.join(self.conf['cache_dir'], 'templates')
pkgname = 'baruwa'
if not os.path.exists(logo):
print sys.STDERR ("The logo image: %s does not exist" % logo)
sys.exit(2)
with open(logo) as handle:
logo = base64.b64encode(handle.read())
mako_lookup = TemplateLookup(
directories=[path],
error_handler=handle_mako_error,
module_directory=cache_dir,
input_encoding='utf-8',
default_filters=['escape'],
output_encoding='utf-8',
encoding_errors='replace',
imports=['from webhelpers.html import escape']
)
mailer = Mailer(get_conf_options(self.conf))
mailer.start()
users = Session.query(User)\
.filter(User.active == True)\
.filter(User.send_report == True)
previous_records = Session\
.query(Release.messageid)\
.order_by(desc('timestamp'))
for user in users:
messages = Session.query(Message.id, Message.timestamp,
Message.from_address, Message.to_address,
Message.subject,
case([(and_(Message.spam > 0,
Message.virusinfected == 0,
Message.nameinfected == 0,
Message.otherinfected == 0,
), True)],
else_=False).label('spam'),
Message.to_domain)\
.filter(Message.isquarantined > 0)\
.filter(or_(Message.spam > 0,
Message.nameinfected > 0,
Message.otherinfected > 0))\
.order_by(desc('timestamp'))
query = UserFilter(Session, user, messages)
messages = query.filter()
if int(self.options.days) > 0:
a_day = datetime.timedelta(days=self.options.days)
startdate = now().date() - a_day
messages = messages.filter(Message.timestamp > str(startdate))
messages = messages.filter(~Message.id.in_(previous_records))
messages = messages[:25]
if messages:
lang = 'en'
host_urls = dict([(domain.name, domain.site_url)
for domain in user.domains
if domain.status == True])
langs = [domain.language for domain in user.domains
if domain.language != 'en']
if langs:
lang = langs.pop(0)
translator = set_lang(lang, pkgname, localedir)
_ = translator.ugettext
def make_release_records(spam):
"map function"
uuid = gen_uuid(user)
spam.uuid = uuid
return Release(uuid=uuid, messageid=spam.id)
if user.is_peleb:
torelease = [make_release_records(spam)
for spam in messages]
template = mako_lookup.get_template('/email/quarantine.html')
html = template.render(messages=messages,
host_urls=host_urls, url=url_for,
default_url=self.conf['baruwa.default.url'])
template = mako_lookup.get_template('/email/quarantine.txt')
text = template.render(messages=messages)
displayname = "%s %s" % (user.firstname, user.lastname)
email = Msg(author=[(_('Baruwa Reports'),
self.conf['baruwa.reports.sender'])],
to=[(displayname, user.email)],
subject=_('Baruwa quarantine report'))
email.plain = text
email.rich = html
email.attach('logo.png',
data=logo,
maintype='image',
subtype='png',
inline=True)
mailer.send(email)
if 'torelease' in locals():
Session.add_all(torelease)
Session.commit()
mailer.stop()
def command(self):
"send"
self.init()
import baruwa
pkgname = 'baruwa'
here = os.path.dirname(
os.path.dirname(os.path.abspath(baruwa.__file__)))
path = os.path.join(here, 'baruwa', 'templates')
logo = os.path.join(here, 'baruwa', 'public', 'imgs', 'logo.png')
localedir = os.path.join(here, 'baruwa', 'i18n')
cache_dir = os.path.join(self.conf['cache_dir'], 'templates')
mako_lookup = TemplateLookup(
directories=[path],
error_handler=handle_mako_error,
module_directory=cache_dir,
input_encoding='utf-8',
default_filters=['escape'],
output_encoding='utf-8',
encoding_errors='replace',
imports=['from webhelpers.html import escape'])
mailer = Mailer(get_conf_options(self.conf))
mailer.start()
users = Session\
.query(User)\
.filter(User.active == True)\
.filter(User.send_report == True).all()
#localedir = os.path.join(self.conf['here'], 'baruwa', 'i18n')
for user in users:
host_url = self.conf['baruwa.default.url']
sentry = 0
language = 'en'
if user.is_domain_admin:
orgs = [group.id for group in user.organizations]
domains = Session\
.query(Domain.site_url, Domain.language)\
.join(domain_owners)\
.filter(Domain.status == True)\
.filter(domain_owners.c.organization_id.in_(orgs))\
.all()
if domains:
host_url, language = domains.pop(0)
if user.is_peleb:
domains = [(domain.site_url, domain.language)
for domain in user.domains if domain.status == True]
if domains:
host_url, language = domains.pop(0)
if language == 'en' and 'domains' in locals() and domains:
while domains:
_, language = domains.pop(0)
if language != 'en':
break
translator = set_lang(language, pkgname, localedir)
_ = translator.ugettext
reports = {
'1': {
'address': 'from_address',
'sort': 'count',
'title': _('Top Senders by Quantity')
},
'2': {
'address': 'from_address',
'sort': 'size',
'title': _('Top Senders by Volume')
},
'3': {
'address': 'from_domain',
'sort': 'count',
'title': _('Top Sender Domains by Quantity')
},
'4': {
'address': 'from_domain',
'sort': 'size',
'title': _('Top Sender Domains by Volume')
},
'5': {
'address': 'to_address',
'sort': 'count',
'title': _('Top Recipients by Quantity')
},
'6': {
'address': 'to_address',
'sort': 'size',
'title': _('Top Recipients by Volume')
},
'7': {
'address': 'to_domain',
'sort': 'count',
'title': _('Top Recipient Domains By Quantity')
},
'8': {
'address': 'to_domain',
'sort': 'size',
'title': _('Top Recipient Domains By Volume')
},
'9': {
'address': '',
'sort': '',
'title': _('Spam Score distribution')
},
'10': {
'address': 'clientip',
'sort': 'count',
'title': _('Top mail hosts by quantity')
},
'11': {
'address': '',
'sort': '',
'title': _('Total messages [ After SMTP ]')
}
}
pieheadings = ('', _('Address'), _('Count'), _('Volume'), '')
totalsheaders = dict(date=_('Date'),
mail=_('Mail totals'),
spam=_('Spam totals'),
virus=_('Virus totals'),
volume=_('Mail volume'),
totals=_('Totals'))
pdfcreator = PDFReport(logo, _('Baruwa mail report'))
for reportid in ['1', '2', '3', '4', '5', '6', '7', '8', '10']:
sortby = reports[reportid]['sort']
if user.account_type == 3 and reportid in ['7', '8']:
data = None
else:
query = ReportQuery(user, reportid)
if int(self.options.days) > 0:
a_day = datetime.timedelta(days=self.options.days)
startdate = datetime.date.today() - a_day
query = query.get().filter(
Message.timestamp > str(startdate))
data = query[:10]
else:
data = query.get()[:10]
if data:
sentry += 1
pdfcreator.add(data, reports[reportid]['title'],
pieheadings, sortby)
query = Session.query(Message.date,
func.count(Message.date).label('mail_total'),
func.sum(case([(Message.virusinfected > 0, 1)],
else_=0)).label('virus_total'),
func.sum(case([(and_(Message.virusinfected ==
0, Message.spam > 0), 1)], else_=0))\
.label('spam_total'), func.sum(Message.size)\
.label('total_size')).group_by(Message.date)\
.order_by(desc(Message.date))
uquery = UserFilter(Session, user, query)
query = uquery.filter()
data = query.all()
if data:
if not sentry:
sentry += 1
pdfcreator.add(data,
_('Message Totals'),
totalsheaders,
chart='bar')
if sentry:
template = mako_lookup.get_template('/email/pdfreports.txt')
text = template.render(user=user, url=host_url)
displayname = '%s %s' % (user.firstname or '', user.lastname
or '')
email = Msg(author=[(_('Baruwa Reports'),
self.conf['baruwa.reports.sender'])],
to=[(displayname, user.email)],
subject=_('Baruwa usage report'))
email.plain = text
pdf_file = base64.b64encode(pdfcreator.build())
email.attach('baruwa-reports.pdf',
data=pdf_file,
maintype='application/pdf',
subtype='application/x-pdf')
try:
mailer.send(email)
except (TransportFailedException, MessageFailedException), err:
print >> sys.stderr, ("Error sending to: %s, Error: %s" %
(user.email, err))
def mailq_preview(self, queueid, attachid=None, imgid=None, allowimgs=None):
"preview a queued message"
query = Session.query(MailQueueItem)
uquery = UserFilter(Session, c.user, query, model=MailQueueItem)
query = uquery.filter()
try:
mailqitem = query.filter(MailQueueItem.id == queueid).one()
except NoResultFound:
flash_alert(_('The requested queued message was not found.'))
redirect(url('mailq-status'))
try:
task = preview_queued_msg.apply_async(args=[mailqitem.messageid,
mailqitem.direction, attachid, imgid],
queue=mailqitem.hostname)
task.wait(30)
if task.result:
if imgid:
response.content_type = task.result['content_type']
if task.result and 'img' in task.result:
info = QUEUEDOWNLOAD_MSG % dict(m=mailqitem.messageid,
a=task.result['name'])
audit_log(c.user.username,
1, info, request.host,
request.remote_addr, now())
return base64.decodestring(task.result['img'])
abort(404)
if attachid:
info = QUEUEDOWNLOAD_MSG % dict(m=mailqitem.messageid,
a=task.result['name'])
audit_log(c.user.username,
1, info, request.host,
request.remote_addr, now())
response.content_type = task.result['mimetype']
dispos = 'attachment; filename="%s"' % task.result['name']
response.headers['Content-Disposition'] = str(dispos)
content_len = len(task.result['attachment'])
response.headers['Content-Length'] = content_len
response.headers['Pragma'] = 'public'
response.headers['Cache-Control'] = 'max-age=0'
return base64.decodestring(task.result['attachment'])
for part in task.result['parts']:
if part['type'] == 'html':
html = fromstring(part['content'])
for element, attribute, link, pos in iterlinks(html):
if not link.startswith('cid:'):
if not allowimgs and attribute == 'src':
element.attrib['src'] = '%simgs/blocked.gif' % media_url()
element.attrib['title'] = link
flash(_('This message contains external images, which have been blocked. ') +
literal(link_to(_('Display images'),
url('queue-preview-with-imgs', queueid=queueid), class_='uline')))
else:
imgname = link.replace('cid:', '')
element.attrib['src'] = url('queue-preview-img',
imgid=imgname.replace('/', '__xoxo__'),
queueid=queueid)
part['content'] = tostring(html)
c.message = task.result
info = QUEUEPREVIEW_MSG % dict(m=mailqitem.messageid)
audit_log(c.user.username,
1, info, request.host,
request.remote_addr, now())
else:
raise TimeoutError
except (TimeoutError, QueueNotFound):
flash_alert(_('The message could not be processed'))
redirect(url('mailq-status'))
c.queueid = queueid
c.messageid = mailqitem.messageid
return render('/status/preview.html')
def _get_data(self, format=None, success=None, errors=None):
"Get report data"
filters = session.get('filter_by', [])
query = Session.query(
func.max(Message.timestamp).label('oldest'),
func.min(Message.timestamp).label('newest'))
uquery = UserFilter(Session, c.user, query)
query = uquery.filter()
# count = self._get_count()
countq = MsgCount(Session, c.user)
count = countq()
cachekey = u'savedfilters-%s' % c.user.username
sfq = Session.query(SavedFilter)\
.filter(SavedFilter.user == c.user)\
.options(FromCache('sql_cache_short', cachekey))
if self.invalidate:
sfq.invalidate()
savedfilters = sfq.all()
if filters:
dynq = DynaQuery(Message, query, filters)
query = dynq.generate()
dcountq = Session.query(func.count(Message.id).label('count'))
dcountqi = UserFilter(Session, c.user, dcountq)
dcountq = dcountqi.filter()
dyncq = DynaQuery(Message, dcountq, filters)
dcountq = dyncq.generate()
dcount = dcountq.one()
count = dcount.count
cachekey = u'report-aggregates-%s' % c.user.username
query = query.options(FromCache('sql_cache_short', cachekey))
if self.invalidate:
query.invalidate()
data = query.all()
saved_filters = [
processfilters(filt, filters) for filt in savedfilters
]
if format is None:
return data, count, filters, saved_filters
else:
if format == 'json':
data = data[0]
filterdict = dict(FILTER_ITEMS)
filterbydict = dict(FILTER_BY)
active_filters = [
dict(filter_field=filterdict[filt['field']],
filter_by=filterbydict[filt['filter']],
filter_value=filt['value']) for filt in filters
]
try:
newest = data.newest.strftime("%Y-%m-%d %H:%M")
oldest = data.oldest.strftime("%Y-%m-%d %H:%M")
except AttributeError:
newest = ''
oldest = ''
datadict = dict(count=count, newest=newest, oldest=oldest)
jsondata = dict(success=success,
data=datadict,
errors=errors,
active_filters=active_filters,
saved_filters=saved_filters)
return jsondata
def mailq_preview(self, queueid, attachid=None, imgid=None, allowimgs=None):
"preview a queued message"
query = Session.query(MailQueueItem)
uquery = UserFilter(Session, c.user, query, model=MailQueueItem)
query = uquery.filter()
try:
mailqitem = query.filter(MailQueueItem.id == queueid).one()
except NoResultFound:
flash_alert(_('The requested queued message was not found.'))
redirect(url('mailq-status'))
try:
task = preview_queued_msg.apply_async(args=[mailqitem.messageid,
mailqitem.direction, attachid, imgid],
queue=mailqitem.hostname)
task.wait(30)
if task.result:
if imgid:
response.content_type = task.result['content_type']
if task.result and 'img' in task.result:
info = QUEUEDOWNLOAD_MSG % dict(m=mailqitem.messageid,
a=task.result['name'])
audit_log(c.user.username,
1, info, request.host,
request.remote_addr, datetime.now())
return base64.decodestring(task.result['img'])
abort(404)
if attachid:
info = QUEUEDOWNLOAD_MSG % dict(m=mailqitem.messageid,
a=task.result['name'])
audit_log(c.user.username,
1, info, request.host,
request.remote_addr, datetime.now())
response.content_type = task.result['mimetype']
dispos = 'attachment; filename="%s"' % task.result['name']
response.headers['Content-Disposition'] = dispos
content_len = len(task.result['attachment'])
response.headers['Content-Length'] = content_len
response.headers['Pragma'] = 'public'
response.headers['Cache-Control'] = 'max-age=0'
return base64.decodestring(task.result['attachment'])
for part in task.result['parts']:
if part['type'] == 'html':
html = fromstring(part['content'])
for element, attribute, link, pos in iterlinks(html):
if not link.startswith('cid:'):
if not allowimgs and attribute == 'src':
element.attrib['src'] = '%simgs/blocked.gif' % media_url()
element.attrib['title'] = link
flash(_('This message contains external images, which have been blocked. ') +
literal(link_to(_('Display images'),
url('queue-preview-with-imgs', queueid=queueid), class_='uline')))
else:
imgname = link.replace('cid:', '')
element.attrib['src'] = url('queue-preview-img',
imgid=imgname.replace('/', '__xoxo__'),
queueid=queueid)
part['content'] = tostring(html)
c.message = task.result
info = QUEUEPREVIEW_MSG % dict(m=mailqitem.messageid)
audit_log(c.user.username,
1, info, request.host,
request.remote_addr, datetime.now())
else:
raise TimeoutError
except (TimeoutError, QueueNotFound):
flash_alert(_('The message could not be processed'))
redirect(url('mailq-status'))
c.queueid = queueid
c.messageid = mailqitem.messageid
return render('/status/preview.html')
def command(self):
"run command"
self.init()
import baruwa
here = os.path.dirname(
os.path.dirname(os.path.abspath(baruwa.__file__)))
path = os.path.join(here, 'baruwa', 'templates')
logo = os.path.join(here, 'baruwa', 'public', 'imgs', 'logo.png')
localedir = os.path.join(here, 'baruwa', 'i18n')
cache_dir = os.path.join(self.conf['cache_dir'], 'templates')
pkgname = 'baruwa'
if not os.path.exists(logo):
print sys.STDERR("The logo image: %s does not exist" % logo)
sys.exit(2)
with open(logo) as handle:
logo = base64.b64encode(handle.read())
mako_lookup = TemplateLookup(
directories=[path],
error_handler=handle_mako_error,
module_directory=cache_dir,
input_encoding='utf-8',
default_filters=['escape'],
output_encoding='utf-8',
encoding_errors='replace',
imports=['from webhelpers.html import escape'])
mailer = Mailer(get_conf_options(self.conf))
mailer.start()
users = Session.query(User)\
.filter(User.active == True)\
.filter(User.send_report == True)
previous_records = Session\
.query(Release.messageid)\
.order_by(desc('timestamp'))
for user in users:
messages = Session.query(Message.id, Message.timestamp,
Message.from_address, Message.to_address,
Message.subject,
case([(and_(Message.spam > 0,
Message.virusinfected == 0,
Message.nameinfected == 0,
Message.otherinfected == 0,
), True)],
else_=False).label('spam'),
Message.to_domain)\
.filter(Message.isquarantined > 0)\
.filter(or_(Message.spam > 0,
Message.nameinfected > 0,
Message.otherinfected > 0))\
.order_by(desc('timestamp'))
query = UserFilter(Session, user, messages)
messages = query.filter()
if int(self.options.days) > 0:
a_day = datetime.timedelta(days=self.options.days)
startdate = now().date() - a_day
messages = messages.filter(Message.timestamp > str(startdate))
messages = messages.filter(~Message.id.in_(previous_records))
messages = messages[:25]
if messages:
lang = 'en'
host_urls = dict([(domain.name, domain.site_url)
for domain in user.domains
if domain.status == True])
langs = [
domain.language for domain in user.domains
if domain.language != 'en'
]
if langs:
lang = langs.pop(0)
translator = set_lang(lang, pkgname, localedir)
_ = translator.ugettext
def make_release_records(spam):
"map function"
uuid = gen_uuid(user)
spam.uuid = uuid
return Release(uuid=uuid, messageid=spam.id)
if user.is_peleb:
torelease = [
make_release_records(spam) for spam in messages
]
template = mako_lookup.get_template('/email/quarantine.html')
html = template.render(
messages=messages,
host_urls=host_urls,
url=url_for,
default_url=self.conf['baruwa.default.url'])
template = mako_lookup.get_template('/email/quarantine.txt')
text = template.render(messages=messages)
displayname = "%s %s" % (user.firstname, user.lastname)
email = Msg(author=[(_('Baruwa Reports'),
self.conf['baruwa.reports.sender'])],
to=[(displayname, user.email)],
subject=_('Baruwa quarantine report'))
email.plain = text
email.rich = html
email.attach('logo.png',
data=logo,
maintype='image',
subtype='png',
inline=True)
mailer.send(email)
if 'torelease' in locals():
Session.add_all(torelease)
Session.commit()
mailer.stop()
def mailq_preview(self, queueid, attachid=None, imgid=None,
allowimgs=None, richformat=None):
"preview a queued message"
query = Session.query(MailQueueItem)
uquery = UserFilter(Session, c.user, query, model=MailQueueItem)
query = uquery.filter()
try:
mailqitem = query.filter(MailQueueItem.id == queueid).one()
except NoResultFound:
flash_alert(_('The requested queued message was not found.'))
redirect(url('mailq-status'))
try:
task = preview_queued_msg.apply_async(args=[mailqitem.messageid,
mailqitem.direction, attachid, imgid],
routing_key=mailqitem.hostname)
task.wait(30)
if task.result:
if imgid:
response.content_type = task.result['content_type']
if task.result and 'img' in task.result:
info = QUEUEDOWNLOAD_MSG % dict(m=mailqitem.messageid,
a=task.result['name'])
audit_log(c.user.username,
1, unicode(info), request.host,
request.remote_addr, arrow.utcnow().datetime)
return base64.decodestring(task.result['img'])
abort(404)
if attachid:
info = QUEUEDOWNLOAD_MSG % dict(m=mailqitem.messageid,
a=task.result['name'])
audit_log(c.user.username,
1, unicode(info), request.host,
request.remote_addr, arrow.utcnow().datetime)
response.content_type = task.result['mimetype']
dispos = 'attachment; filename="%s"' % task.result['name']
response.headers['Content-Disposition'] = str(dispos)
content_len = len(task.result['attachment'])
response.headers['Content-Length'] = content_len
response.headers['Pragma'] = 'public'
response.headers['Cache-Control'] = 'max-age=0'
return base64.decodestring(task.result['attachment'])
for part in task.result['parts']:
if part['type'] == 'text/html':
local_rf = (not task.result['is_multipart']
or richformat)
part['content'] = img_fixups(part['content'],
queueid, allowimgs,
local_rf)
c.message = task.result
info = QUEUEPREVIEW_MSG % dict(m=mailqitem.messageid)
audit_log(c.user.username,
1, unicode(info), request.host,
request.remote_addr, arrow.utcnow().datetime)
else:
raise TimeoutError
except (TimeoutError, QueueNotFound), error:
msg = _('The message could not be processed')
flash_alert(msg)
msg = _('The message could not be processed: %s') % error
log.info(msg)
redirect(url('mailq-status'))
def command(self):
"send"
self.init()
import baruwa
pkgname = "baruwa"
here = os.path.dirname(os.path.dirname(os.path.abspath(baruwa.__file__)))
path = os.path.join(here, "baruwa", "templates")
logo = os.path.join(here, "baruwa", "public", "imgs", "logo.png")
localedir = os.path.join(here, "baruwa", "i18n")
cache_dir = os.path.join(self.conf["cache_dir"], "templates")
mako_lookup = TemplateLookup(
directories=[path],
error_handler=handle_mako_error,
module_directory=cache_dir,
input_encoding="utf-8",
default_filters=["escape"],
output_encoding="utf-8",
encoding_errors="replace",
imports=["from webhelpers.html import escape"],
)
mailer = Mailer(get_conf_options(self.conf))
mailer.start()
users = Session.query(User).filter(User.active == True).filter(User.send_report == True).all()
# localedir = os.path.join(self.conf['here'], 'baruwa', 'i18n')
for user in users:
host_url = self.conf["baruwa.default.url"]
sentry = 0
language = "en"
if user.is_domain_admin:
orgs = [group.id for group in user.organizations]
domains = (
Session.query(Domain.site_url, Domain.language)
.join(domain_owners)
.filter(Domain.status == True)
.filter(domain_owners.c.organization_id.in_(orgs))
.all()
)
if domains:
host_url, language = domains.pop(0)
if user.is_peleb:
domains = [(domain.site_url, domain.language) for domain in user.domains if domain.status == True]
if domains:
host_url, language = domains.pop(0)
if language == "en" and "domains" in locals() and domains:
while domains:
_, language = domains.pop(0)
if language != "en":
break
translator = set_lang(language, pkgname, localedir)
_ = translator.ugettext
reports = {
"1": {"address": "from_address", "sort": "count", "title": _("Top Senders by Quantity")},
"2": {"address": "from_address", "sort": "size", "title": _("Top Senders by Volume")},
"3": {"address": "from_domain", "sort": "count", "title": _("Top Sender Domains by Quantity")},
"4": {"address": "from_domain", "sort": "size", "title": _("Top Sender Domains by Volume")},
"5": {"address": "to_address", "sort": "count", "title": _("Top Recipients by Quantity")},
"6": {"address": "to_address", "sort": "size", "title": _("Top Recipients by Volume")},
"7": {"address": "to_domain", "sort": "count", "title": _("Top Recipient Domains By Quantity")},
"8": {"address": "to_domain", "sort": "size", "title": _("Top Recipient Domains By Volume")},
"9": {"address": "", "sort": "", "title": _("Spam Score distribution")},
"10": {"address": "clientip", "sort": "count", "title": _("Top mail hosts by quantity")},
"11": {"address": "", "sort": "", "title": _("Total messages [ After SMTP ]")},
}
pieheadings = ("", _("Address"), _("Count"), _("Volume"), "")
totalsheaders = dict(
date=_("Date"),
mail=_("Mail totals"),
spam=_("Spam totals"),
virus=_("Virus totals"),
volume=_("Mail volume"),
totals=_("Totals"),
)
pdfcreator = PDFReport(logo, _("Baruwa mail report"))
for reportid in ["1", "2", "3", "4", "5", "6", "7", "8", "10"]:
sortby = reports[reportid]["sort"]
if user.account_type == 3 and reportid in ["7", "8"]:
data = None
else:
query = ReportQuery(user, reportid)
if int(self.options.days) > 0:
a_day = datetime.timedelta(days=self.options.days)
startdate = now().date() - a_day
query = query.get().filter(Message.timestamp > str(startdate))
data = query[:10]
else:
data = query.get()[:10]
if data:
sentry += 1
pdfcreator.add(data, reports[reportid]["title"], pieheadings, sortby)
query = (
Session.query(
Message.date,
func.count(Message.date).label("mail_total"),
func.sum(case([(Message.virusinfected > 0, 1)], else_=0)).label("virus_total"),
func.sum(case([(and_(Message.virusinfected == 0, Message.spam > 0), 1)], else_=0)).label(
"spam_total"
),
func.sum(Message.size).label("total_size"),
)
.group_by(Message.date)
.order_by(desc(Message.date))
)
uquery = UserFilter(Session, user, query)
query = uquery.filter()
data = query.all()
if data:
if not sentry:
sentry += 1
pdfcreator.add(data, _("Message Totals"), totalsheaders, chart="bar")
if sentry:
template = mako_lookup.get_template("/email/pdfreports.txt")
text = template.render(user=user, url=host_url)
displayname = "%s %s" % (user.firstname or "", user.lastname or "")
email = Msg(
author=[(_("Baruwa Reports"), self.conf["baruwa.reports.sender"])],
to=[(displayname, user.email)],
subject=_("Baruwa usage report"),
)
email.plain = text
pdf_file = pdfcreator.build()
email.attach("baruwa-reports.pdf", data=pdf_file, maintype="application", subtype="pdf")
try:
mailer.send(email)
except (TransportFailedException, MessageFailedException), err:
print >>sys.stderr, ("Error sending to: %s, Error: %s" % (user.email, err))
def search(self, format=None):
"Search for messages"
q = request.GET.get('q', None)
if q is None:
redirect(url(controller='messages', action='listing'))
index = 'messages, messages_rt'
action = request.GET.get('a', 'listing')
if not action in ['listing', 'quarantine', 'archive']:
action = 'listing'
if action == 'archive':
index = 'archive'
try:
page = int(request.GET.get('page', 1))
except ValueError:
page = 1
num_items = session.get('msgs_search_num_results', 50)
conn = SphinxClient()
conn.SetMatchMode(SPH_MATCH_EXTENDED2)
#conn.SetSortMode(SPH_SORT_EXTENDED, "timestamp DESC")
if action == 'quarantine':
conn.SetFilter('isquarantined', [True,])
if page == 1:
conn.SetLimits(0, num_items, 500)
else:
offset = (page - 1) * num_items
conn.SetLimits(offset, num_items, 500)
if not c.user.is_superadmin:
filter_sphinx(Session, c.user, conn)
else:
conn.SetSelect('timestamp')
q = clean_sphinx_q(q)
results = conn.Query(q, index)
q = restore_sphinx_q(q)
if results and results['matches']:
#import pprint
#pprint.pprint(results)
ids = [hit['id'] for hit in results['matches']]
filters = session.get('filter_by', None)
if index == 'archive':
messages = self._get_archived().filter(
Archive.id.in_(ids))
query = UserFilter(Session, c.user, messages, True)
messages = query.filter()
if filters:
dynq = DynaQuery(Message, messages, filters)
messages = dynq.generate()
else:
messages = self._get_messages().filter(
Message.id.in_(ids))
query = UserFilter(Session, c.user, messages)
messages = query.filter()
if filters:
dynq = DynaQuery(Message, messages, filters)
messages = dynq.generate()
total_found = results['total']
search_time = results['time']
messages = messages.order_by(desc('timestamp'))
else:
print '=' * 100
print conn.GetLastError()
messages = []
results = dict(matches=[], total=0)
total_found = 0
search_time = 0
pages = paginator(dict(page=page, results_per_page=num_items,
total=results['total'],
items=len(results['matches']),
q=q))
if format == 'json':
response.headers['Content-Type'] = 'application/json'
data = dict(action=action,
total_found=total_found,
search_time=search_time,
paginator=pages,
items=[jsonify_msg_list(msg) for msg in messages])
return json.dumps(data)
c.messages = messages
c.action = action
c.total_found = total_found
c.search_time = search_time
c.page = pages
return render('/messages/searchresults.html')
def search(self, format=None):
"Search for messages"
qry = request.GET.get('q', None)
if qry is None:
redirect(url(controller='messages', action='listing'))
index = 'messages, messagesdelta, messages_rt'
action = request.GET.get('a', 'listing')
if action not in ['listing', 'quarantine', 'archive']:
action = 'listing'
if action == 'archive':
index = 'archive archivedelta'
try:
page = int(request.GET.get('page', 1))
except ValueError:
page = 1
num_items = session.get('msgs_search_num_results', 50)
conn = SphinxClient()
sphinxopts = extract_sphinx_opts(config['sphinx.url'])
conn.SetServer(sphinxopts.get('host', '127.0.0.1'))
conn.SetMatchMode(SPH_MATCH_EXTENDED2)
if action == 'quarantine':
conn.SetFilter('isquarantined', [
True,
])
if page == 1:
conn.SetLimits(0, num_items, 500)
else:
offset = (page - 1) * num_items
conn.SetLimits(offset, num_items, 500)
if not c.user.is_superadmin:
filter_sphinx(Session, c.user, conn)
else:
conn.SetSelect('timestamp')
qry = clean_sphinx_q(qry)
try:
results = conn.Query(qry, index)
except (socket.timeout, struct.error):
redirect(request.path_qs)
qry = restore_sphinx_q(qry)
if results and results['matches']:
ids = [hit['id'] for hit in results['matches']]
filters = session.get('filter_by', None)
if action == 'archive':
messages = get_archived().filter(Archive.id.in_(ids))
query = UserFilter(Session, c.user, messages, True)
messages = query.filter()
if filters:
dynq = DynaQuery(Message, messages, filters)
messages = dynq.generate()
else:
messages = get_messages().filter(Message.id.in_(ids))
query = UserFilter(Session, c.user, messages)
messages = query.filter()
if filters:
dynq = DynaQuery(Message, messages, filters)
messages = dynq.generate()
total_found = results['total']
search_time = results['time']
messages = messages.order_by(desc('timestamp'))
else:
messages = []
results = dict(matches=[], total=0)
total_found = 0
search_time = 0
pages = paginator(
dict(page=page,
results_per_page=num_items,
total=results['total'],
items=len(results['matches']),
q=qry))
if format == 'json':
response.headers['Content-Type'] = 'application/json'
data = dict(action=action,
total_found=total_found,
search_time=search_time,
paginator=pages,
items=[jsonify_msg_list(msg) for msg in messages])
return json.dumps(data)
c.messages = messages
c.action = action
c.total_found = total_found
c.search_time = search_time
c.page = pages
return self.render('/messages/searchresults.html')
def command(self):
"send"
self.init()
import baruwa
pkgname = 'baruwa'
here = os.path.dirname(
os.path.dirname(os.path.abspath(baruwa.__file__))
)
path = os.path.join(here, 'baruwa', 'templates')
logo = os.path.join(here, 'baruwa', 'public', 'imgs', 'logo.png')
localedir = os.path.join(here, 'baruwa', 'i18n')
cache_dir = os.path.join(self.conf['cache_dir'], 'templates')
mako_lookup = TemplateLookup(
directories=[path],
error_handler=handle_mako_error,
module_directory=cache_dir,
input_encoding='utf-8',
default_filters=['escape'],
output_encoding='utf-8',
encoding_errors='replace',
imports=['from webhelpers.html import escape']
)
mailer = Mailer(get_conf_options(self.conf))
mailer.start()
users = Session\
.query(User)\
.filter(User.active == True)\
.filter(User.send_report == True).all()
#localedir = os.path.join(self.conf['here'], 'baruwa', 'i18n')
for user in users:
host_url = self.conf['baruwa.default.url']
sentry = 0
language = 'en'
if user.is_domain_admin:
orgs = [group.id for group in user.organizations]
domains = Session\
.query(Domain.site_url, Domain.language)\
.join(domain_owners)\
.filter(Domain.status == True)\
.filter(domain_owners.c.organization_id.in_(orgs))\
.all()
if domains:
host_url, language = domains.pop(0)
if user.is_peleb:
domains = [(domain.site_url, domain.language)
for domain in user.domains
if domain.status == True]
if domains:
host_url, language = domains.pop(0)
if language == 'en' and 'domains' in locals() and domains:
while domains:
_, language = domains.pop(0)
if language != 'en':
break
translator = set_lang(language, pkgname, localedir)
_ = translator.ugettext
reports = {
'1': {'address': 'from_address', 'sort': 'count',
'title': _('Top Senders by Quantity')},
'2': {'address': 'from_address', 'sort': 'size',
'title': _('Top Senders by Volume')},
'3': {'address': 'from_domain', 'sort': 'count',
'title': _('Top Sender Domains by Quantity')},
'4': {'address': 'from_domain', 'sort': 'size',
'title': _('Top Sender Domains by Volume')},
'5': {'address': 'to_address', 'sort': 'count',
'title': _('Top Recipients by Quantity')},
'6': {'address': 'to_address', 'sort': 'size',
'title': _('Top Recipients by Volume')},
'7': {'address': 'to_domain', 'sort': 'count',
'title': _('Top Recipient Domains By Quantity')},
'8': {'address': 'to_domain', 'sort': 'size',
'title': _('Top Recipient Domains By Volume')},
'9': {'address': '', 'sort': '',
'title': _('Spam Score distribution')},
'10': {'address': 'clientip', 'sort': 'count',
'title': _('Top mail hosts by quantity')},
'11': {'address': '', 'sort': '',
'title': _('Total messages [ After SMTP ]')}
}
pieheadings = ('', _('Address'), _('Count'), _('Volume'), '')
totalsheaders = dict(date=_('Date'), mail=_('Mail totals'),
spam=_('Spam totals'), virus=_('Virus totals'),
volume=_('Mail volume'), totals=_('Totals'))
pdfcreator = PDFReport(logo, _('Baruwa mail report'))
for reportid in ['1', '2', '3', '4', '5', '6', '7', '8', '10']:
sortby = reports[reportid]['sort']
if user.account_type == 3 and reportid in ['7', '8']:
data = None
else:
query = ReportQuery(user, reportid)
if int(self.options.days) > 0:
a_day = datetime.timedelta(days=self.options.days)
startdate = datetime.date.today() - a_day
query = query.get().filter(Message.timestamp >
str(startdate))
data = query[:10]
else:
data = query.get()[:10]
if data:
sentry += 1
pdfcreator.add(data, reports[reportid]['title'],
pieheadings, sortby)
query = Session.query(Message.date,
func.count(Message.date).label('mail_total'),
func.sum(case([(Message.virusinfected > 0, 1)],
else_=0)).label('virus_total'),
func.sum(case([(and_(Message.virusinfected ==
0, Message.spam > 0), 1)], else_=0))\
.label('spam_total'), func.sum(Message.size)\
.label('total_size')).group_by(Message.date)\
.order_by(desc(Message.date))
uquery = UserFilter(Session, user, query)
query = uquery.filter()
data = query.all()
if data:
if not sentry:
sentry += 1
pdfcreator.add(data, _('Message Totals'), totalsheaders,
chart='bar')
if sentry:
template = mako_lookup.get_template('/email/pdfreports.txt')
text = template.render(user=user, url=host_url)
displayname = '%s %s' % (user.firstname or '',
user.lastname or '')
email = Msg(author=[(_('Baruwa Reports'),
self.conf['baruwa.reports.sender'])],
to=[(displayname, user.email)],
subject=_('Baruwa usage report'))
email.plain = text
pdf_file = base64.b64encode(pdfcreator.build())
email.attach('baruwa-reports.pdf',
data=pdf_file,
maintype='application/pdf',
subtype='application/x-pdf')
try:
mailer.send(email)
except (TransportFailedException, MessageFailedException), err:
print >> sys.stderr, ("Error sending to: %s, Error: %s" %
(user.email, err))
def quarantine(self,
page=1,
direction='dsc',
order_by='timestamp',
section=None,
format=None):
"quarantined messages"
filters = session.get('filter_by', None)
num_items = session.get('msgs_num_items', 50)
if direction == 'dsc':
sort = desc(order_by)
else:
sort = order_by
messages = get_messages().filter(
Message.isquarantined == 1).order_by(sort)
msgcount = get_msg_count().filter(Message.isquarantined == 1)
query = UserFilter(Session, c.user, messages)
countquery = UserFilter(Session, c.user, msgcount)
messages = query.filter()
msgcount = countquery.filter()
if section:
if section == 'spam':
messages = messages.filter(Message.spam == 1)
msgcount = messages.filter(Message.spam == 1)
else:
messages = messages.filter(Message.spam == 0)
msgcount = messages.filter(Message.spam == 0)
if filters:
dynq = DynaQuery(Message, messages, filters)
dynmsgq = DynaQuery(Message, msgcount, filters)
messages = dynq.generate()
msgcount = dynmsgq.generate()
c.order_by = order_by
c.direction = direction
c.section = section
msgcount = msgcount.count()
c.form = BulkReleaseForm(request.POST, csrf_context=session)
if request.method == 'POST':
choices = session.get('bulk_items', [])
else:
pages = paginate.Page(messages,
page=int(page),
items_per_page=num_items,
item_count=msgcount)
choices = [(str(message.id), message.id)
for message in pages.items]
session['bulk_items'] = choices
session.save()
c.form.message_id.choices = choices
if request.method == 'POST' and c.form.validate() and choices:
msgs = Session.query(Message.id,
Message.messageid,
Message.from_address,
Message.timestamp, Message.to_address,
Message.hostname, Message.msgfiles)\
.filter(Message.id.in_(c.form.message_id.data))
query = UserFilter(Session, c.user, msgs)
msgs = query.filter()
localtmz = config.get('baruwa.timezone', 'Africa/Johannesburg')
formvals = (dict(release=c.form.release.data,
learn=c.form.learn.data,
salearn_as=c.form.learnas.data,
todelete=c.form.delete.data,
use_alt=c.form.usealt.data,
altrecipients=c.form.altrecipients.data,
message_id=msg.messageid,
from_address=msg.from_address,
date=convert_date(msg.timestamp,
localtmz).strftime('%Y%m%d'),
msgfiles=msg.msgfiles,
to_address=msg.to_address,
hostname=msg.hostname,
mid=msg.id,
num=num_items) for msg in msgs)
if formvals:
try:
subtasks = [
process_quarantined_msg.subtask(
args=[formval],
options=dict(
routing_key=system_hostname() if asbool(
config.get('ms.quarantine.shared', 'false')
) else formval['hostname']))
for formval in formvals
]
task = group(subtasks).apply_async()
task.save(backend=RBACKEND)
session['bulk_items'] = []
if 'taskids' not in session:
session['taskids'] = []
session['taskids'].append(task.id)
session['bulkprocess-count'] = 1
session.save()
redirect(url('messages-bulk-process', taskid=task.id))
except (QueueNotFound, OperationalError, IndexError):
flash_alert(
_('The messages could not processed'
', try again later'))
elif request.method == 'POST' and not c.form.validate():
try:
flash_alert(
_(u', '.join([
unicode(c.form.errors[err][0]) for err in c.form.errors
])))
except IndexError:
flash_alert(
_('The messages could not processed'
', an Unknown error occured.'))
pages = paginate.Page(messages,
page=int(page),
items_per_page=num_items,
item_count=msgcount)
if format == 'json':
response.headers['Content-Type'] = 'application/json'
data = convert_to_json(pages,
direction=direction,
order_by=order_by,
section=section)
return data
c.page = pages
return self.render('/messages/quarantine.html')
def quarantine(self, page=1, direction='dsc', order_by='timestamp',
section=None, format=None):
"quarantined messages"
filters = session.get('filter_by', None)
num_items = session.get('msgs_num_items', 50)
if direction == 'dsc':
sort = desc(order_by)
else:
sort = order_by
messages = self._get_messages().filter(
Message.isquarantined == 1).order_by(sort)
msgcount = self._get_msg_count().filter(
Message.isquarantined == 1)
query = UserFilter(Session, c.user, messages)
countquery = UserFilter(Session, c.user, msgcount)
messages = query.filter()
msgcount = countquery.filter()
if section:
if section == 'spam':
messages = messages.filter(Message.spam == 1)
msgcount = messages.filter(Message.spam == 1)
else:
messages = messages.filter(Message.spam == 0)
msgcount = messages.filter(Message.spam == 0)
if filters:
dynq = DynaQuery(Message, messages, filters)
dynmsgq = DynaQuery(Message, msgcount, filters)
messages = dynq.generate()
msgcount = dynmsgq.generate()
c.order_by = order_by
c.direction = direction
c.section = section
msgcount = msgcount.count()
c.form = BulkReleaseForm(request.POST, csrf_context=session)
if request.POST:
choices = session.get('bulk_items', [])
else:
pages = paginate.Page(messages, page=int(page),
items_per_page=num_items,
item_count=msgcount)
choices = [(str(message.id), message.id)
for message in pages.items]
session['bulk_items'] = choices
session.save()
c.form.message_id.choices = choices
if request.POST and c.form.validate() and choices:
msgs = Session.query(Message.id,
Message.messageid,
Message.from_address,
Message.timestamp, Message.to_address,
Message.hostname)\
.filter(Message.id.in_(c.form.message_id.data))
query = UserFilter(Session, c.user, msgs)
msgs = query.filter()
localtmz = config.get('baruwa.timezone', 'Africa/Johannesburg')
formvals = (dict(release=c.form.release.data,
learn=c.form.learn.data,
salearn_as=c.form.learnas.data,
todelete=c.form.delete.data,
use_alt=c.form.usealt.data,
altrecipients=c.form.altrecipients.data,
message_id=msg.messageid,
from_address=msg.from_address,
date=convert_date(msg.timestamp, localtmz).strftime('%Y%m%d'),
to_address=msg.to_address,
hostname=msg.hostname,
mid=msg.id)
for msg in msgs)
taskset = TaskSet(tasks=(
process_quarantined_msg.subtask(
args=[formval],
options=dict(queue=formval['hostname'])
) for formval in formvals
)
)
if formvals:
try:
task = taskset.apply_async()
task.save(dbbackend)
session['bulk_items'] = []
if not 'taskids' in session:
session['taskids'] = []
session['taskids'].append(task.taskset_id)
session['bulkprocess-count'] = 1
session.save()
redirect(url('messages-bulk-process',
taskid=task.taskset_id))
except QueueNotFound:
flash_alert(_('The messages could not processed'
', try again later'))
elif request.POST and not c.form.validate():
flash_alert(_(u', '.join([unicode(c.form.errors[err][0])
for err in c.form.errors])))
pages = paginate.Page(messages, page=int(page),
items_per_page=num_items,
item_count=msgcount)
if format == 'json':
response.headers['Content-Type'] = 'application/json'
data = convert_to_json(pages,
direction=direction,
order_by=order_by,
section=section)
return data
c.page = pages
return render('/messages/quarantine.html')
def quarantine(self, page=1, direction='dsc', order_by='timestamp',
section=None, format=None):
"quarantined messages"
filters = session.get('filter_by', None)
num_items = session.get('msgs_num_items', 50)
if direction == 'dsc':
sort = desc(order_by)
else:
sort = order_by
messages = self._get_messages().filter(
Message.isquarantined == 1).order_by(sort)
msgcount = self._get_msg_count().filter(
Message.isquarantined == 1)
query = UserFilter(Session, c.user, messages)
countquery = UserFilter(Session, c.user, msgcount)
messages = query.filter()
msgcount = countquery.filter()
if section:
if section == 'spam':
messages = messages.filter(Message.spam == 1)
msgcount = messages.filter(Message.spam == 1)
else:
messages = messages.filter(Message.spam == 0)
msgcount = messages.filter(Message.spam == 0)
if filters:
dynq = DynaQuery(Message, messages, filters)
dynmsgq = DynaQuery(Message, msgcount, filters)
messages = dynq.generate()
msgcount = dynmsgq.generate()
c.order_by = order_by
c.direction = direction
c.section = section
msgcount = msgcount.count()
c.form = BulkReleaseForm(request.POST, csrf_context=session)
if request.POST:
choices = session.get('bulk_items', [])
else:
pages = paginate.Page(messages, page=int(page),
items_per_page=num_items,
item_count=msgcount)
choices = [(str(message.id), message.id)
for message in pages.items]
session['bulk_items'] = choices
session.save()
c.form.message_id.choices = choices
if request.POST and c.form.validate() and choices:
msgs = Session.query(Message.id,
Message.messageid,
Message.from_address,
Message.date, Message.to_address,
Message.hostname)\
.filter(Message.id.in_(c.form.message_id.data))
query = UserFilter(Session, c.user, msgs)
msgs = query.filter()
formvals = (dict(release=c.form.release.data,
learn=c.form.learn.data,
salearn_as=c.form.learnas.data,
todelete=c.form.delete.data,
use_alt=c.form.usealt.data,
altrecipients=c.form.altrecipients.data,
message_id=msg.messageid,
from_address=msg.from_address,
date=str(msg.date), to_address=msg.to_address,
hostname=msg.hostname,
mid=msg.id)
for msg in msgs)
taskset = TaskSet(tasks=(
process_quarantined_msg.subtask(
args=[formval],
options=dict(queue=formval['hostname'])
) for formval in formvals
)
)
if formvals:
try:
task = taskset.apply_async()
task.save(dbbackend)
session['bulk_items'] = []
if not 'taskids' in session:
session['taskids'] = []
session['taskids'].append(task.taskset_id)
session['bulkprocess-count'] = 1
session.save()
redirect(url('messages-bulk-process',
taskid=task.taskset_id))
except QueueNotFound:
flash_alert(_('The messages could not processed'
', try again later'))
elif request.POST and not c.form.validate():
flash_alert(_(', '.join([c.form.errors[err][0]
for err in c.form.errors])))
pages = paginate.Page(messages, page=int(page),
items_per_page=num_items,
item_count=msgcount)
if format == 'json':
response.headers['Content-Type'] = 'application/json'
data = convert_to_json(pages,
direction=direction,
order_by=order_by,
section=section)
return data
c.page = pages
return render('/messages/quarantine.html')
def search(self, format=None):
"Search for messages"
q = request.GET.get('q', None)
if q is None:
redirect(url(controller='messages', action='listing'))
index = 'messages, messagesdelta, messages_rt'
action = request.GET.get('a', 'listing')
if not action in ['listing', 'quarantine', 'archive']:
action = 'listing'
if action == 'archive':
index = 'archive archivedelta'
try:
page = int(request.GET.get('page', 1))
except ValueError:
page = 1
num_items = session.get('msgs_search_num_results', 50)
conn = SphinxClient()
conn.SetMatchMode(SPH_MATCH_EXTENDED2)
#conn.SetSortMode(SPH_SORT_EXTENDED, "timestamp DESC")
if action == 'quarantine':
conn.SetFilter('isquarantined', [True,])
if page == 1:
conn.SetLimits(0, num_items, 500)
else:
offset = (page - 1) * num_items
conn.SetLimits(offset, num_items, 500)
if not c.user.is_superadmin:
filter_sphinx(Session, c.user, conn)
else:
conn.SetSelect('timestamp')
q = clean_sphinx_q(q)
results = conn.Query(q, index)
q = restore_sphinx_q(q)
if results and results['matches']:
#import pprint
#pprint.pprint(results)
ids = [hit['id'] for hit in results['matches']]
filters = session.get('filter_by', None)
if action == 'archive':
messages = self._get_archived().filter(
Archive.id.in_(ids))
query = UserFilter(Session, c.user, messages, True)
messages = query.filter()
if filters:
dynq = DynaQuery(Message, messages, filters)
messages = dynq.generate()
else:
messages = self._get_messages().filter(
Message.id.in_(ids))
query = UserFilter(Session, c.user, messages)
messages = query.filter()
if filters:
dynq = DynaQuery(Message, messages, filters)
messages = dynq.generate()
total_found = results['total']
search_time = results['time']
messages = messages.order_by(desc('timestamp'))
else:
print '=' * 100
print conn.GetLastError()
messages = []
results = dict(matches=[], total=0)
total_found = 0
search_time = 0
pages = paginator(dict(page=page, results_per_page=num_items,
total=results['total'],
items=len(results['matches']),
q=q))
if format == 'json':
response.headers['Content-Type'] = 'application/json'
data = dict(action=action,
total_found=total_found,
search_time=search_time,
paginator=pages,
items=[jsonify_msg_list(msg) for msg in messages])
return json.dumps(data)
c.messages = messages
c.action = action
c.total_found = total_found
c.search_time = search_time
c.page = pages
return render('/messages/searchresults.html')
def mailq_preview(self,
queueid,
attachid=None,
imgid=None,
allowimgs=None,
richformat=None):
"preview a queued message"
query = Session.query(MailQueueItem)
uquery = UserFilter(Session, c.user, query, model=MailQueueItem)
query = uquery.filter()
try:
mailqitem = query.filter(MailQueueItem.id == queueid).one()
except NoResultFound:
flash_alert(_('The requested queued message was not found.'))
redirect(url('mailq-status'))
try:
task = preview_queued_msg.apply_async(
args=[
mailqitem.messageid, mailqitem.direction, attachid, imgid
],
routing_key=mailqitem.hostname)
task.wait(30)
if task.result:
if imgid:
response.content_type = task.result['content_type']
if task.result and 'img' in task.result:
info = QUEUEDOWNLOAD_MSG % dict(m=mailqitem.messageid,
a=task.result['name'])
audit_log(c.user.username, 1, unicode(info),
request.host, request.remote_addr,
arrow.utcnow().datetime)
return base64.decodestring(task.result['img'])
abort(404)
if attachid:
info = QUEUEDOWNLOAD_MSG % dict(m=mailqitem.messageid,
a=task.result['name'])
audit_log(c.user.username, 1, unicode(info), request.host,
request.remote_addr,
arrow.utcnow().datetime)
response.content_type = task.result['mimetype']
dispos = 'attachment; filename="%s"' % task.result['name']
response.headers['Content-Disposition'] = str(dispos)
content_len = len(task.result['attachment'])
response.headers['Content-Length'] = content_len
response.headers['Pragma'] = 'public'
response.headers['Cache-Control'] = 'max-age=0'
return base64.decodestring(task.result['attachment'])
for part in task.result['parts']:
if part['type'] == 'text/html':
local_rf = (not task.result['is_multipart']
or richformat)
part['content'] = img_fixups(part['content'], queueid,
allowimgs, local_rf)
c.message = task.result
info = QUEUEPREVIEW_MSG % dict(m=mailqitem.messageid)
audit_log(c.user.username, 1, unicode(info), request.host,
request.remote_addr,
arrow.utcnow().datetime)
else:
raise TimeoutError
except (TimeoutError, QueueNotFound), error:
msg = _('The message could not be processed')
flash_alert(msg)
msg = _('The message could not be processed: %s') % error
log.info(msg)
redirect(url('mailq-status'))