Пример #1
0
 def archive(self, page=1, direction='dsc',
             order_by='timestamp', format=None):
     "messages archive"
     filters = session.get('filter_by', None)
     num_items = session.get('msgs_num_items', 50)
     if direction == 'dsc':
         sort = desc(order_by)
     else:
         sort = order_by
     messages = self._get_archived().order_by(sort)
     msgcount = self._get_msg_count(True)
     query = UserFilter(Session, c.user, messages, True)
     countquery = UserFilter(Session, c.user, msgcount, True)
     messages = query.filter()
     msgcount = countquery.filter()
     if filters:
         dynq = DynaQuery(Archive, messages, filters)
         dynmsgq = DynaQuery(Archive, msgcount, filters)
         messages = dynq.generate()
         msgcount = dynmsgq.generate()
     c.order_by = order_by
     c.direction = direction
     msgcount = msgcount.count()
     pages = paginate.Page(messages, page=int(page),
                             items_per_page=num_items,
                             item_count=msgcount)
     if format == 'json':
         response.headers['Content-Type'] = 'application/json'
         data = convert_to_json(pages,
                                 direction=direction,
                                 order_by=order_by,
                                 section=None)
         return data
     c.page = pages
     return render('/messages/archive.html')
Пример #2
0
 def _get_data(self, format=None, success=None, errors=None):
     "Get report data"
     filters = session.get('filter_by', [])
     query = Session.query(func.max(Message.timestamp).label('oldest'),
                         func.min(Message.timestamp).label('newest'))
     uquery = UserFilter(Session, c.user, query)
     query = uquery.filter()
     # count = self._get_count()
     countq = MsgCount(Session, c.user)
     count = countq()
     cachekey = u'savedfilters-%s' % c.user.username
     sfq = Session.query(SavedFilter)\
             .filter(SavedFilter.user == c.user)\
             .options(FromCache('sql_cache_short', cachekey))
     if self.invalidate:
         sfq.invalidate()
     savedfilters = sfq.all()
     if filters:
         dynq = DynaQuery(Message, query, filters)
         query = dynq.generate()
         dcountq = Session.query(func.count(Message.id).label('count'))
         dcountqi = UserFilter(Session, c.user, dcountq)
         dcountq = dcountqi.filter()
         dyncq = DynaQuery(Message, dcountq, filters)
         dcountq = dyncq.generate()
         dcount = dcountq.one()
         count = dcount.count
     cachekey = u'report-aggregates-%s' % c.user.username
     query = query.options(FromCache('sql_cache_short', cachekey))
     if self.invalidate:
         query.invalidate()
     data = query.all()
     saved_filters = [processfilters(filt, filters)
                     for filt in savedfilters]
     if format is None:
         return data, count, filters, saved_filters
     else:
         if format == 'json':
             data = data[0]
             filterdict = dict(FILTER_ITEMS)
             filterbydict = dict(FILTER_BY)
             active_filters = [dict(
                         filter_field=unicode(filterdict[filt['field']]),
                         filter_by=unicode(filterbydict[filt['filter']]),
                         filter_value=unicode(filt['value']))
                         for filt in filters]
             try:
                 newest = data.newest.strftime("%Y-%m-%d %H:%M")
                 oldest = data.oldest.strftime("%Y-%m-%d %H:%M")
             except AttributeError:
                 newest = ''
                 oldest = ''
             datadict = dict(count=count, newest=newest, oldest=oldest)
             jsondata = dict(success=success, data=datadict, errors=errors,
                             active_filters=active_filters,
                             saved_filters=saved_filters)
             return jsondata
Пример #3
0
 def archive(self, page=1, direction='dsc',
             order_by='timestamp', format=None):
     "messages archive"
     filters = session.get('filter_by', None)
     num_items = session.get('msgs_num_items', 50)
     if direction == 'dsc':
         sort = desc(order_by)
     else:
         sort = order_by
     messages = self._get_archived().order_by(sort)
     msgcount = self._get_msg_count(True)
     query = UserFilter(Session, c.user, messages, True)
     countquery = UserFilter(Session, c.user, msgcount, True)
     messages = query.filter()
     msgcount = countquery.filter()
     if filters:
         dynq = DynaQuery(Archive, messages, filters)
         dynmsgq = DynaQuery(Archive, msgcount, filters)
         messages = dynq.generate()
         msgcount = dynmsgq.generate()
     c.order_by = order_by
     c.direction = direction
     msgcount = msgcount.count()
     pages = paginate.Page(messages, page=int(page),
                             items_per_page=num_items,
                             item_count=msgcount)
     if format == 'json':
         response.headers['Content-Type'] = 'application/json'
         data = convert_to_json(pages,
                                 direction=direction,
                                 order_by=order_by,
                                 section=None)
         return data
     c.page = pages
     return render('/messages/archive.html')
Пример #4
0
 def listing(self, page=1, direction='dsc', order_by='timestamp',
             format=None):
     "message listing"
     filters = session.get('filter_by', None)
     num_items = session.get('msgs_num_items', 50)
     if direction == 'dsc':
         sort = desc(order_by)
     else:
         sort = order_by
     messages = get_messages().order_by(sort)
     query = UserFilter(Session, c.user, messages)
     messages = query.filter()
     if filters:
         msgcount = get_msg_count()
         countquery = UserFilter(Session, c.user, msgcount)
         msgcount = countquery.filter()
         dynq = DynaQuery(Message, messages, filters)
         dynmsgq = DynaQuery(Message, msgcount, filters)
         messages = dynq.generate()
         msgcount = dynmsgq.generate()
         msgcount = msgcount.count()
     else:
         countquery = MsgCount(Session, c.user)
         msgcount = countquery.count()
     c.list_all = True
     c.order_by = order_by
     c.direction = direction
     pages = paginate.Page(messages, page=int(page),
                             items_per_page=num_items,
                             item_count=msgcount)
     if format == 'json':
         response.headers['Content-Type'] = 'application/json'
         data = convert_to_json(pages,
                                 direction=direction,
                                 order_by=order_by,
                                 section=None)
         return data
     c.page = pages
     return self.render('/messages/listing.html')
Пример #5
0
 def listing(self,
             page=1,
             direction='dsc',
             order_by='timestamp',
             format=None):
     "message listing"
     filters = session.get('filter_by', None)
     num_items = session.get('msgs_num_items', 50)
     if direction == 'dsc':
         sort = desc(order_by)
     else:
         sort = order_by
     messages = get_messages().order_by(sort)
     query = UserFilter(Session, c.user, messages)
     messages = query.filter()
     if filters:
         msgcount = get_msg_count()
         countquery = UserFilter(Session, c.user, msgcount)
         msgcount = countquery.filter()
         dynq = DynaQuery(Message, messages, filters)
         dynmsgq = DynaQuery(Message, msgcount, filters)
         messages = dynq.generate()
         msgcount = dynmsgq.generate()
         msgcount = msgcount.count()
     else:
         countquery = MsgCount(Session, c.user)
         msgcount = countquery.count()
     c.list_all = True
     c.order_by = order_by
     c.direction = direction
     pages = paginate.Page(messages,
                           page=int(page),
                           items_per_page=num_items,
                           item_count=msgcount)
     if format == 'json':
         response.headers['Content-Type'] = 'application/json'
         data = convert_to_json(pages,
                                direction=direction,
                                order_by=order_by,
                                section=None)
         return data
     c.page = pages
     return self.render('/messages/listing.html')
Пример #6
0
    def search(self, format=None):
        "Search for messages"
        q = request.GET.get('q', None)
        if q is None:
            redirect(url(controller='messages', action='listing'))
        index = 'messages, messagesdelta, messages_rt'
        action = request.GET.get('a', 'listing')
        if not action in ['listing', 'quarantine', 'archive']:
            action = 'listing'
        if action == 'archive':
            index = 'archive archivedelta'
        try:
            page = int(request.GET.get('page', 1))
        except ValueError:
            page = 1
        num_items = session.get('msgs_search_num_results', 50)
        conn = SphinxClient()
        conn.SetMatchMode(SPH_MATCH_EXTENDED2)
        #conn.SetSortMode(SPH_SORT_EXTENDED, "timestamp DESC")
        if action == 'quarantine':
            conn.SetFilter('isquarantined', [True,])
        if page == 1:
            conn.SetLimits(0, num_items, 500)
        else:
            offset = (page - 1) * num_items
            conn.SetLimits(offset, num_items, 500)
        if not c.user.is_superadmin:
            filter_sphinx(Session, c.user, conn)
        else:
            conn.SetSelect('timestamp')
        q = clean_sphinx_q(q)
        results = conn.Query(q, index)
        q = restore_sphinx_q(q)
        if results and results['matches']:
            #import pprint
            #pprint.pprint(results)
            ids = [hit['id'] for hit in results['matches']]
            filters = session.get('filter_by', None)
            if action == 'archive':
                messages = self._get_archived().filter(
                            Archive.id.in_(ids))
                query = UserFilter(Session, c.user, messages, True)
                messages = query.filter()
                if filters:
                    dynq = DynaQuery(Message, messages, filters)
                    messages = dynq.generate()
            else:
                messages = self._get_messages().filter(
                            Message.id.in_(ids))
                query = UserFilter(Session, c.user, messages)
                messages = query.filter()
                if filters:
                    dynq = DynaQuery(Message, messages, filters)
                    messages = dynq.generate()
            total_found = results['total']
            search_time = results['time']
            messages = messages.order_by(desc('timestamp'))
        else:
            print '=' * 100
            print conn.GetLastError()
            messages = []
            results = dict(matches=[], total=0)
            total_found = 0
            search_time = 0

        pages = paginator(dict(page=page, results_per_page=num_items,
                                total=results['total'],
                                items=len(results['matches']),
                                q=q))

        if format == 'json':
            response.headers['Content-Type'] = 'application/json'
            data = dict(action=action,
                        total_found=total_found,
                        search_time=search_time,
                        paginator=pages,
                        items=[jsonify_msg_list(msg) for msg in messages])
            return json.dumps(data)

        c.messages = messages
        c.action = action
        c.total_found = total_found
        c.search_time = search_time
        c.page = pages
        return render('/messages/searchresults.html')
Пример #7
0
    def quarantine(self, page=1, direction='dsc', order_by='timestamp',
        section=None, format=None):
        "quarantined messages"
        filters = session.get('filter_by', None)
        num_items = session.get('msgs_num_items', 50)
        if direction == 'dsc':
            sort = desc(order_by)
        else:
            sort = order_by
        messages = self._get_messages().filter(
                    Message.isquarantined == 1).order_by(sort)
        msgcount = self._get_msg_count().filter(
                    Message.isquarantined == 1)
        query = UserFilter(Session, c.user, messages)
        countquery = UserFilter(Session, c.user, msgcount)
        messages = query.filter()
        msgcount = countquery.filter()
        if section:
            if section == 'spam':
                messages = messages.filter(Message.spam == 1)
                msgcount = messages.filter(Message.spam == 1)
            else:
                messages = messages.filter(Message.spam == 0)
                msgcount = messages.filter(Message.spam == 0)
        if filters:
            dynq = DynaQuery(Message, messages, filters)
            dynmsgq = DynaQuery(Message, msgcount, filters)
            messages = dynq.generate()
            msgcount = dynmsgq.generate()
        c.order_by = order_by
        c.direction = direction
        c.section = section
        msgcount = msgcount.count()
        c.form = BulkReleaseForm(request.POST, csrf_context=session)
        if request.POST:
            choices = session.get('bulk_items', [])
        else:
            pages = paginate.Page(messages, page=int(page),
                                items_per_page=num_items,
                                item_count=msgcount)
            choices = [(str(message.id), message.id)
                        for message in pages.items]
            session['bulk_items'] = choices
            session.save()
        c.form.message_id.choices = choices
        if request.POST and c.form.validate() and choices:
            msgs = Session.query(Message.id,
                    Message.messageid,
                    Message.from_address,
                    Message.timestamp, Message.to_address,
                    Message.hostname)\
                    .filter(Message.id.in_(c.form.message_id.data))
            query = UserFilter(Session, c.user, msgs)
            msgs = query.filter()
            localtmz = config.get('baruwa.timezone', 'Africa/Johannesburg')
            formvals = (dict(release=c.form.release.data,
                            learn=c.form.learn.data, 
                            salearn_as=c.form.learnas.data,
                            todelete=c.form.delete.data,
                            use_alt=c.form.usealt.data,
                            altrecipients=c.form.altrecipients.data,
                            message_id=msg.messageid,
                            from_address=msg.from_address,
                            date=convert_date(msg.timestamp, localtmz).strftime('%Y%m%d'),
                            to_address=msg.to_address,
                            hostname=msg.hostname,
                            mid=msg.id)
                        for msg in msgs)

            taskset = TaskSet(tasks=(
                                process_quarantined_msg.subtask(
                                    args=[formval],
                                    options=dict(queue=formval['hostname'])
                                ) for formval in formvals
                                )
                            )
            if formvals:
                try:
                    task = taskset.apply_async()
                    task.save(dbbackend)
                    session['bulk_items'] = []
                    if not 'taskids' in session:
                        session['taskids'] = []
                    session['taskids'].append(task.taskset_id)
                    session['bulkprocess-count'] = 1
                    session.save()
                    redirect(url('messages-bulk-process',
                            taskid=task.taskset_id))
                except QueueNotFound:
                    flash_alert(_('The messages could not processed'
                                ', try again later'))
        elif request.POST and not c.form.validate():
            flash_alert(_(u', '.join([unicode(c.form.errors[err][0])
                                    for err in c.form.errors])))
        pages = paginate.Page(messages, page=int(page),
                                items_per_page=num_items,
                                item_count=msgcount)
        if format == 'json':
            response.headers['Content-Type'] = 'application/json'
            data = convert_to_json(pages,
                                    direction=direction,
                                    order_by=order_by,
                                    section=section)
            return data
        c.page = pages
        return render('/messages/quarantine.html')
Пример #8
0
 def display(self, reportid, format=None):
     "Display a report"
     c.report_title = REPORTS[reportid]['title']
     filters = session.get('filter_by', [])
     if reportid in ['1', '2', '3', '4', '5', '6', '7', '8', '10']:
         rquery = ReportQuery(c.user, reportid, filters)
         query = rquery()
         cachekey = u'reportquery-%s-%s' % (c.user.username, reportid)
         query = query.options(FromCache('sql_cache_short', cachekey))
         data = query[:10]
         if format == 'png':
             return self._generate_png(data, reportid)
         if format == 'csv':
             info = REPORTDL_MSG % dict(r=c.report_title, f='csv')
             audit_log(c.user.username, 1, info, request.host,
                       request.remote_addr, datetime.now())
             return self._generate_csv(data, reportid)
         jsondata = [
             dict(tooltip=getattr(item, 'address'),
                  y=getattr(item, REPORTS[reportid]['sort']),
                  stroke='black',
                  color=PIE_COLORS[index],
                  size=getattr(item, 'size'))
             for index, item in enumerate(data)
         ]
         template = '/reports/piereport.html'
     if reportid == '9':
         query = sa_scores(Session, c.user)
         if filters:
             dynq = DynaQuery(Message, query, filters)
             query = dynq.generate()
         cachekey = u'sascores-%s' % c.user.username
         query = query.options(FromCache('sql_cache_short', cachekey))
         data = query.all()
         if format == 'json':
             scores = []
             counts = []
             for row in data:
                 scores.append(
                     dict(value=int(row.score), text=str(row.score)))
                 counts.append(
                     dict(y=int(row.count),
                          tooltip=(_('Score ') + str(row.score) + ': ' +
                                   str(row.count))))
             jsondata = dict(scores=scores, count=counts)
         elif format == 'png':
             return self._generate_png(data, reportid)
         else:
             jsondata = {}
             jsondata['labels'] = [{
                 'value': index + 1,
                 'text': str(item.score)
             } for index, item in enumerate(data)]
             jsondata['scores'] = [item.count for item in data]
             template = '/reports/barreport.html'
     if reportid == '10':
         if format == 'json':
             data = [[
                 item.address.strip(),
                 get_hostname(item.address.strip()),
                 country_flag(item.address.strip()), item.count,
                 format_byte_size(item.size)
             ] for item in data]
         template = '/reports/relays.html'
     if reportid == '11':
         query = message_totals(Session, c.user)
         if filters:
             dynq = DynaQuery(Message, query, filters)
             query = dynq.generate()
         cachekey = u'msgtotals-%s' % c.user.username
         query = query.options(FromCache('sql_cache_short', cachekey))
         data = query.all()
         if format == 'png':
             return self._generate_png(data, reportid)
         elif format == 'json':
             dates = []
             mail_total = []
             spam_total = []
             size_total = []
             virus_total = []
             for row in data:
                 dates.append(str(row.date))
                 mail_total.append(int(row.mail_total))
                 spam_total.append(int(row.spam_total))
                 virus_total.append(int(row.virus_total))
                 size_total.append(int(row.total_size))
             jsondata = dict(
                 dates=[
                     dict(value=index + 1, text=date)
                     for index, date in enumerate(dates)
                 ],
                 mail=[
                     dict(y=total,
                          tooltip=(_('Mail totals on ') + dates[index] +
                                   ': ' + str(total)))
                     for index, total in enumerate(mail_total)
                 ],
                 spam=[
                     dict(y=total,
                          tooltip=(_('Spam totals on ') + dates[index] +
                                   ': ' + str(total)))
                     for index, total in enumerate(spam_total)
                 ],
                 virii=[
                     dict(y=total,
                          tooltip=(_('Virus totals on ') + dates[index] +
                                   ': ' + str(total)))
                     for index, total in enumerate(virus_total)
                 ],
                 volume=size_total,
                 mail_total=sum(mail_total),
                 spam_total=sum(spam_total),
                 virus_total=sum(virus_total),
                 volume_total=sum(size_total))
             try:
                 vpct = "%.1f" % (
                     (1.0 * sum(virus_total) / sum(mail_total)) * 100)
                 spct = "%.1f" % (
                     (1.0 * sum(spam_total) / sum(mail_total)) * 100)
             except ZeroDivisionError:
                 vpct = "0.0"
                 spct = "0.0"
             jsondata['vpct'] = vpct
             jsondata['spct'] = spct
             data = [
                 dict(date=str(row.date),
                      mail_total=row.mail_total,
                      spam_total=row.spam_total,
                      virus_total=row.virus_total,
                      size_total=format_byte_size(row.total_size),
                      virus_percent="%.1f" %
                      ((1.0 * int(row.virus_total) / int(row.mail_total)) *
                       100),
                      spam_percent="%.1f" %
                      ((1.0 * int(row.spam_total) / int(row.mail_total)) *
                       100)) for row in data
             ]
         elif format == 'csv':
             info = REPORTDL_MSG % dict(r=c.report_title, f='csv')
             audit_log(c.user.username, 1, info, request.host,
                       request.remote_addr, datetime.now())
             return self._generate_csv(data, reportid)
         else:
             jsondata = dict(mail=[],
                             spam=[],
                             virus=[],
                             volume=[],
                             labels=[])
             for index, item in enumerate(data):
                 jsondata['spam'].append(item.spam_total)
                 jsondata['mail'].append(item.mail_total)
                 jsondata['virus'].append(item.virus_total)
                 jsondata['volume'].append(item.total_size)
                 jsondata['labels'].append(
                     dict(text=str(item.date), value=index))
             template = '/reports/listing.html'
     if format == 'json':
         response.headers['Content-Type'] = JSON_HEADER
         return json.dumps(dict(items=list(data), pie_data=jsondata))
     if format == 'pdf' and reportid != '9':
         info = REPORTDL_MSG % dict(r=c.report_title, f='pdf')
         audit_log(c.user.username, 1, info, request.host,
                   request.remote_addr, datetime.now())
         return self._generate_pdf(data, reportid)
     c.reportid = reportid
     c.chart_data = json.dumps(jsondata)
     c.top_items = data
     c.active_filters = filters
     c.saved_filters = []
     c.FILTER_BY = FILTER_BY
     c.FILTER_ITEMS = FILTER_ITEMS
     c.form = FilterForm(request.POST, csrf_context=session)
     info = REPORTVIEW_MSG % dict(r=c.report_title)
     audit_log(c.user.username, 1, info, request.host, request.remote_addr,
               datetime.now())
     return render(template)
Пример #9
0
 def _get_data(self, format=None, success=None, errors=None):
     "Get report data"
     filters = session.get('filter_by', [])
     query = Session.query(
         func.max(Message.timestamp).label('oldest'),
         func.min(Message.timestamp).label('newest'))
     uquery = UserFilter(Session, c.user, query)
     query = uquery.filter()
     # count = self._get_count()
     countq = MsgCount(Session, c.user)
     count = countq()
     cachekey = u'savedfilters-%s' % c.user.username
     sfq = Session.query(SavedFilter)\
             .filter(SavedFilter.user == c.user)\
             .options(FromCache('sql_cache_short', cachekey))
     if self.invalidate:
         sfq.invalidate()
     savedfilters = sfq.all()
     if filters:
         dynq = DynaQuery(Message, query, filters)
         query = dynq.generate()
         dcountq = Session.query(func.count(Message.id).label('count'))
         dcountqi = UserFilter(Session, c.user, dcountq)
         dcountq = dcountqi.filter()
         dyncq = DynaQuery(Message, dcountq, filters)
         dcountq = dyncq.generate()
         dcount = dcountq.one()
         count = dcount.count
     cachekey = u'report-aggregates-%s' % c.user.username
     query = query.options(FromCache('sql_cache_short', cachekey))
     if self.invalidate:
         query.invalidate()
     data = query.all()
     saved_filters = [
         processfilters(filt, filters) for filt in savedfilters
     ]
     if format is None:
         return data, count, filters, saved_filters
     else:
         if format == 'json':
             data = data[0]
             filterdict = dict(FILTER_ITEMS)
             filterbydict = dict(FILTER_BY)
             active_filters = [
                 dict(filter_field=filterdict[filt['field']],
                      filter_by=filterbydict[filt['filter']],
                      filter_value=filt['value']) for filt in filters
             ]
             try:
                 newest = data.newest.strftime("%Y-%m-%d %H:%M")
                 oldest = data.oldest.strftime("%Y-%m-%d %H:%M")
             except AttributeError:
                 newest = ''
                 oldest = ''
             datadict = dict(count=count, newest=newest, oldest=oldest)
             jsondata = dict(success=success,
                             data=datadict,
                             errors=errors,
                             active_filters=active_filters,
                             saved_filters=saved_filters)
             return jsondata
Пример #10
0
    def search(self, format=None):
        "Search for messages"
        q = request.GET.get('q', None)
        if q is None:
            redirect(url(controller='messages', action='listing'))
        index = 'messages, messages_rt'
        action = request.GET.get('a', 'listing')
        if not action in ['listing', 'quarantine', 'archive']:
            action = 'listing'
        if action == 'archive':
            index = 'archive'
        try:
            page = int(request.GET.get('page', 1))
        except ValueError:
            page = 1
        num_items = session.get('msgs_search_num_results', 50)
        conn = SphinxClient()
        conn.SetMatchMode(SPH_MATCH_EXTENDED2)
        #conn.SetSortMode(SPH_SORT_EXTENDED, "timestamp DESC")
        if action == 'quarantine':
            conn.SetFilter('isquarantined', [True,])
        if page == 1:
            conn.SetLimits(0, num_items, 500)
        else:
            offset = (page - 1) * num_items
            conn.SetLimits(offset, num_items, 500)
        if not c.user.is_superadmin:
            filter_sphinx(Session, c.user, conn)
        else:
            conn.SetSelect('timestamp')
        q = clean_sphinx_q(q)
        results = conn.Query(q, index)
        q = restore_sphinx_q(q)
        if results and results['matches']:
            #import pprint
            #pprint.pprint(results)
            ids = [hit['id'] for hit in results['matches']]
            filters = session.get('filter_by', None)
            if index == 'archive':
                messages = self._get_archived().filter(
                            Archive.id.in_(ids))
                query = UserFilter(Session, c.user, messages, True)
                messages = query.filter()
                if filters:
                    dynq = DynaQuery(Message, messages, filters)
                    messages = dynq.generate()
            else:
                messages = self._get_messages().filter(
                            Message.id.in_(ids))
                query = UserFilter(Session, c.user, messages)
                messages = query.filter()
                if filters:
                    dynq = DynaQuery(Message, messages, filters)
                    messages = dynq.generate()
            total_found = results['total']
            search_time = results['time']
            messages = messages.order_by(desc('timestamp'))
        else:
            print '=' * 100
            print conn.GetLastError()
            messages = []
            results = dict(matches=[], total=0)
            total_found = 0
            search_time = 0

        pages = paginator(dict(page=page, results_per_page=num_items,
                                total=results['total'],
                                items=len(results['matches']),
                                q=q))

        if format == 'json':
            response.headers['Content-Type'] = 'application/json'
            data = dict(action=action,
                        total_found=total_found,
                        search_time=search_time,
                        paginator=pages,
                        items=[jsonify_msg_list(msg) for msg in messages])
            return json.dumps(data)

        c.messages = messages
        c.action = action
        c.total_found = total_found
        c.search_time = search_time
        c.page = pages
        return render('/messages/searchresults.html')
Пример #11
0
    def quarantine(self, page=1, direction='dsc', order_by='timestamp',
        section=None, format=None):
        "quarantined messages"
        filters = session.get('filter_by', None)
        num_items = session.get('msgs_num_items', 50)
        if direction == 'dsc':
            sort = desc(order_by)
        else:
            sort = order_by
        messages = self._get_messages().filter(
                    Message.isquarantined == 1).order_by(sort)
        msgcount = self._get_msg_count().filter(
                    Message.isquarantined == 1)
        query = UserFilter(Session, c.user, messages)
        countquery = UserFilter(Session, c.user, msgcount)
        messages = query.filter()
        msgcount = countquery.filter()
        if section:
            if section == 'spam':
                messages = messages.filter(Message.spam == 1)
                msgcount = messages.filter(Message.spam == 1)
            else:
                messages = messages.filter(Message.spam == 0)
                msgcount = messages.filter(Message.spam == 0)
        if filters:
            dynq = DynaQuery(Message, messages, filters)
            dynmsgq = DynaQuery(Message, msgcount, filters)
            messages = dynq.generate()
            msgcount = dynmsgq.generate()
        c.order_by = order_by
        c.direction = direction
        c.section = section
        msgcount = msgcount.count()
        c.form = BulkReleaseForm(request.POST, csrf_context=session)
        if request.POST:
            choices = session.get('bulk_items', [])
        else:
            pages = paginate.Page(messages, page=int(page),
                                items_per_page=num_items,
                                item_count=msgcount)
            choices = [(str(message.id), message.id)
                        for message in pages.items]
            session['bulk_items'] = choices
            session.save()
        c.form.message_id.choices = choices
        if request.POST and c.form.validate() and choices:
            msgs = Session.query(Message.id,
                    Message.messageid,
                    Message.from_address,
                    Message.date, Message.to_address,
                    Message.hostname)\
                    .filter(Message.id.in_(c.form.message_id.data))
            query = UserFilter(Session, c.user, msgs)
            msgs = query.filter()
            formvals = (dict(release=c.form.release.data,
                            learn=c.form.learn.data, 
                            salearn_as=c.form.learnas.data,
                            todelete=c.form.delete.data,
                            use_alt=c.form.usealt.data,
                            altrecipients=c.form.altrecipients.data,
                            message_id=msg.messageid,
                            from_address=msg.from_address,
                            date=str(msg.date), to_address=msg.to_address,
                            hostname=msg.hostname,
                            mid=msg.id)
                        for msg in msgs)

            taskset = TaskSet(tasks=(
                                process_quarantined_msg.subtask(
                                    args=[formval],
                                    options=dict(queue=formval['hostname'])
                                ) for formval in formvals
                                )
                            )
            if formvals:
                try:
                    task = taskset.apply_async()
                    task.save(dbbackend)
                    session['bulk_items'] = []
                    if not 'taskids' in session:
                        session['taskids'] = []
                    session['taskids'].append(task.taskset_id)
                    session['bulkprocess-count'] = 1
                    session.save()
                    redirect(url('messages-bulk-process',
                            taskid=task.taskset_id))
                except QueueNotFound:
                    flash_alert(_('The messages could not processed'
                                ', try again later'))
        elif request.POST and not c.form.validate():
            flash_alert(_(', '.join([c.form.errors[err][0]
                                    for err in c.form.errors])))
        pages = paginate.Page(messages, page=int(page),
                                items_per_page=num_items,
                                item_count=msgcount)
        if format == 'json':
            response.headers['Content-Type'] = 'application/json'
            data = convert_to_json(pages,
                                    direction=direction,
                                    order_by=order_by,
                                    section=section)
            return data
        c.page = pages
        return render('/messages/quarantine.html')
Пример #12
0
    def display(self, reportid, format=None):
        "Display a report"
        try:
            c.report_title = REPORTS[reportid]['title']
        except KeyError:
            abort(404)

        filters = session.get('filter_by', [])
        if reportid in ['1', '2', '3', '4', '5', '6', '7', '8', '10']:
            rquery = ReportQuery(c.user, reportid, filters)
            query = rquery()
            cachekey = u'reportquery-%s-%s' % (c.user.username, reportid)
            query = query.options(FromCache('sql_cache_short', cachekey))
            data = query[:10]
            if format == 'png':
                return self._generate_png(data, reportid)
            if format == 'csv':
                info = REPORTDL_MSG % dict(r=c.report_title, f='csv')
                audit_log(c.user.username,
                        1, unicode(info), request.host,
                        request.remote_addr, arrow.utcnow().datetime)
                return self._generate_csv(data, reportid)
            jsondata = [dict(tooltip=getattr(item, 'address'),
                        y=getattr(item, REPORTS[reportid]['sort']),
                        stroke='black', color=PIE_COLORS[index],
                        size=getattr(item, 'size'))
                        for index, item in enumerate(data)]
            template = '/reports/piereport.html'
        if reportid == '9':
            query = sa_scores(Session, c.user)
            if filters:
                dynq = DynaQuery(Message, query, filters)
                query = dynq.generate()
            cachekey = u'sascores-%s' % c.user.username
            query = query.options(FromCache('sql_cache_short', cachekey))
            data = query.all()
            if format == 'json':
                scores = []
                counts = []
                for row in data:
                    scores.append(dict(value=int(row.score),
                                text=str(row.score)))
                    counts.append(dict(y=int(row.count), tooltip=(_('Score ') +
                                str(row.score) + ': ' + str(row.count))))
                jsondata = dict(scores=scores, count=counts)
            elif format == 'png':
                return self._generate_png(data, reportid)
            else:
                jsondata = {}
                jsondata['labels'] = [{'value': index + 1,
                                    'text': str(item.score)}
                                    for index, item in enumerate(data)]
                jsondata['scores'] = [item.count for item in data]
                template = '/reports/barreport.html'
        if reportid == '10':
            if format == 'json':
                data = [[item.address.strip(),
                        get_hostname(item.address.strip()),
                        country_flag(item.address.strip()),
                        item.count, item.size]
                        for item in data]
            template = '/reports/relays.html'
        if reportid == '11':
            query = message_totals(Session, c.user)
            if filters:
                dynq = DynaQuery(Message, query, filters)
                query = dynq.generate()
            cachekey = u'msgtotals-%s' % c.user.username
            query = query.options(FromCache('sql_cache_short', cachekey))
            data = query.all()
            if format == 'png':
                return self._generate_png(data, reportid)
            elif format == 'json':
                dates = []
                mail_total = []
                spam_total = []
                size_total = []
                virus_total = []
                for row in data:
                    dates.append(str(row.ldate))
                    mail_total.append(int(row.mail_total))
                    spam_total.append(int(row.spam_total))
                    virus_total.append(int(row.virus_total))
                    size_total.append(int(row.total_size))
                jsondata = dict(dates=[dict(value=index + 1, text=date)
                                for index, date in enumerate(dates)],
                            mail=[dict(y=total,
                                tooltip=(_('Mail totals on ') +
                                dates[index] + ': ' + str(total)))
                                for index, total in enumerate(mail_total)],
                            spam=[dict(y=total,
                                tooltip=(_('Spam totals on ') +
                                dates[index] + ': ' + str(total)))
                                for index, total in enumerate(spam_total)],
                            virii=[dict(y=total,
                                tooltip=(_('Virus totals on ') +
                                dates[index] + ': ' + str(total)))
                                for index, total in enumerate(virus_total)],
                            volume=size_total, mail_total=sum(mail_total),
                            spam_total=sum(spam_total),
                            virus_total=sum(virus_total),
                            volume_total=sum(size_total))
                try:
                    vpct = "%.1f" % ((1.0 * sum(virus_total) /
                                    sum(mail_total)) * 100)
                    spct = "%.1f" % ((1.0 * sum(spam_total) /
                                    sum(mail_total)) * 100)
                except ZeroDivisionError:
                    vpct = "0.0"
                    spct = "0.0"
                jsondata['vpct'] = vpct
                jsondata['spct'] = spct
                data = [dict(date=str(row.ldate),
                        mail_total=row.mail_total,
                        spam_total=row.spam_total,
                        virus_total=row.virus_total,
                        size_total=format_byte_size(row.total_size),
                        virus_percent="%.1f" % ((1.0 * int(row.virus_total) /
                        int(row.mail_total)) * 100),
                        spam_percent="%.1f" % ((1.0 * int(row.spam_total) /
                        int(row.mail_total)) * 100)) for row in data]
            elif format == 'csv':
                info = REPORTDL_MSG % dict(r=c.report_title, f='csv')
                audit_log(c.user.username,
                        1, unicode(info), request.host,
                        request.remote_addr, arrow.utcnow().datetime)
                return self._generate_csv(data, reportid)
            else:
                jsondata = dict(mail=[],
                                spam=[],
                                virus=[],
                                volume=[],
                                labels=[])
                for index, item in enumerate(data):
                    jsondata['spam'].append(item.spam_total)
                    jsondata['mail'].append(item.mail_total)
                    jsondata['virus'].append(item.virus_total)
                    jsondata['volume'].append(item.total_size)
                    jsondata['labels'].append(dict(text=str(item.ldate),
                                            value=index))
                template = '/reports/listing.html'
        if format == 'json':
            response.headers['Content-Type'] = JSON_HEADER
            return json.dumps(dict(items=list(data), pie_data=jsondata))
        if format == 'pdf' and reportid != '9':
            info = REPORTDL_MSG % dict(r=c.report_title, f='pdf')
            audit_log(c.user.username,
                    1, unicode(info), request.host,
                    request.remote_addr, arrow.utcnow().datetime)
            return self._generate_pdf(data, reportid)
        c.reportid = reportid
        c.chart_data = json.dumps(jsondata)
        c.top_items = data
        c.active_filters = filters
        c.saved_filters = []
        c.FILTER_BY = FILTER_BY
        c.FILTER_ITEMS = FILTER_ITEMS
        c.form = FilterForm(request.POST, csrf_context=session)
        info = REPORTVIEW_MSG % dict(r=c.report_title)
        audit_log(c.user.username,
                1, unicode(info), request.host,
                request.remote_addr, arrow.utcnow().datetime)
        return self.render(template)
Пример #13
0
    def search(self, format=None):
        "Search for messages"
        qry = request.GET.get('q', None)
        if qry is None:
            redirect(url(controller='messages', action='listing'))
        index = 'messages, messagesdelta, messages_rt'
        action = request.GET.get('a', 'listing')
        if action not in ['listing', 'quarantine', 'archive']:
            action = 'listing'
        if action == 'archive':
            index = 'archive archivedelta'
        try:
            page = int(request.GET.get('page', 1))
        except ValueError:
            page = 1
        num_items = session.get('msgs_search_num_results', 50)
        conn = SphinxClient()
        sphinxopts = extract_sphinx_opts(config['sphinx.url'])
        conn.SetServer(sphinxopts.get('host', '127.0.0.1'))
        conn.SetMatchMode(SPH_MATCH_EXTENDED2)
        if action == 'quarantine':
            conn.SetFilter('isquarantined', [
                True,
            ])
        if page == 1:
            conn.SetLimits(0, num_items, 500)
        else:
            offset = (page - 1) * num_items
            conn.SetLimits(offset, num_items, 500)
        if not c.user.is_superadmin:
            filter_sphinx(Session, c.user, conn)
        else:
            conn.SetSelect('timestamp')
        qry = clean_sphinx_q(qry)
        try:
            results = conn.Query(qry, index)
        except (socket.timeout, struct.error):
            redirect(request.path_qs)
        qry = restore_sphinx_q(qry)
        if results and results['matches']:
            ids = [hit['id'] for hit in results['matches']]
            filters = session.get('filter_by', None)
            if action == 'archive':
                messages = get_archived().filter(Archive.id.in_(ids))
                query = UserFilter(Session, c.user, messages, True)
                messages = query.filter()
                if filters:
                    dynq = DynaQuery(Message, messages, filters)
                    messages = dynq.generate()
            else:
                messages = get_messages().filter(Message.id.in_(ids))
                query = UserFilter(Session, c.user, messages)
                messages = query.filter()
                if filters:
                    dynq = DynaQuery(Message, messages, filters)
                    messages = dynq.generate()
            total_found = results['total']
            search_time = results['time']
            messages = messages.order_by(desc('timestamp'))
        else:
            messages = []
            results = dict(matches=[], total=0)
            total_found = 0
            search_time = 0

        pages = paginator(
            dict(page=page,
                 results_per_page=num_items,
                 total=results['total'],
                 items=len(results['matches']),
                 q=qry))

        if format == 'json':
            response.headers['Content-Type'] = 'application/json'
            data = dict(action=action,
                        total_found=total_found,
                        search_time=search_time,
                        paginator=pages,
                        items=[jsonify_msg_list(msg) for msg in messages])
            return json.dumps(data)

        c.messages = messages
        c.action = action
        c.total_found = total_found
        c.search_time = search_time
        c.page = pages
        return self.render('/messages/searchresults.html')
Пример #14
0
    def quarantine(self,
                   page=1,
                   direction='dsc',
                   order_by='timestamp',
                   section=None,
                   format=None):
        "quarantined messages"
        filters = session.get('filter_by', None)
        num_items = session.get('msgs_num_items', 50)
        if direction == 'dsc':
            sort = desc(order_by)
        else:
            sort = order_by
        messages = get_messages().filter(
            Message.isquarantined == 1).order_by(sort)
        msgcount = get_msg_count().filter(Message.isquarantined == 1)
        query = UserFilter(Session, c.user, messages)
        countquery = UserFilter(Session, c.user, msgcount)
        messages = query.filter()
        msgcount = countquery.filter()
        if section:
            if section == 'spam':
                messages = messages.filter(Message.spam == 1)
                msgcount = messages.filter(Message.spam == 1)
            else:
                messages = messages.filter(Message.spam == 0)
                msgcount = messages.filter(Message.spam == 0)
        if filters:
            dynq = DynaQuery(Message, messages, filters)
            dynmsgq = DynaQuery(Message, msgcount, filters)
            messages = dynq.generate()
            msgcount = dynmsgq.generate()
        c.order_by = order_by
        c.direction = direction
        c.section = section
        msgcount = msgcount.count()
        c.form = BulkReleaseForm(request.POST, csrf_context=session)
        if request.method == 'POST':
            choices = session.get('bulk_items', [])
        else:
            pages = paginate.Page(messages,
                                  page=int(page),
                                  items_per_page=num_items,
                                  item_count=msgcount)
            choices = [(str(message.id), message.id)
                       for message in pages.items]
            session['bulk_items'] = choices
            session.save()
        c.form.message_id.choices = choices
        if request.method == 'POST' and c.form.validate() and choices:
            msgs = Session.query(Message.id,
                    Message.messageid,
                    Message.from_address,
                    Message.timestamp, Message.to_address,
                    Message.hostname, Message.msgfiles)\
                    .filter(Message.id.in_(c.form.message_id.data))
            query = UserFilter(Session, c.user, msgs)
            msgs = query.filter()
            localtmz = config.get('baruwa.timezone', 'Africa/Johannesburg')
            formvals = (dict(release=c.form.release.data,
                             learn=c.form.learn.data,
                             salearn_as=c.form.learnas.data,
                             todelete=c.form.delete.data,
                             use_alt=c.form.usealt.data,
                             altrecipients=c.form.altrecipients.data,
                             message_id=msg.messageid,
                             from_address=msg.from_address,
                             date=convert_date(msg.timestamp,
                                               localtmz).strftime('%Y%m%d'),
                             msgfiles=msg.msgfiles,
                             to_address=msg.to_address,
                             hostname=msg.hostname,
                             mid=msg.id,
                             num=num_items) for msg in msgs)

            if formvals:
                try:
                    subtasks = [
                        process_quarantined_msg.subtask(
                            args=[formval],
                            options=dict(
                                routing_key=system_hostname() if asbool(
                                    config.get('ms.quarantine.shared', 'false')
                                ) else formval['hostname']))
                        for formval in formvals
                    ]
                    task = group(subtasks).apply_async()
                    task.save(backend=RBACKEND)
                    session['bulk_items'] = []
                    if 'taskids' not in session:
                        session['taskids'] = []
                    session['taskids'].append(task.id)
                    session['bulkprocess-count'] = 1
                    session.save()
                    redirect(url('messages-bulk-process', taskid=task.id))
                except (QueueNotFound, OperationalError, IndexError):
                    flash_alert(
                        _('The messages could not processed'
                          ', try again later'))
        elif request.method == 'POST' and not c.form.validate():
            try:
                flash_alert(
                    _(u', '.join([
                        unicode(c.form.errors[err][0]) for err in c.form.errors
                    ])))
            except IndexError:
                flash_alert(
                    _('The messages could not processed'
                      ', an Unknown error occured.'))
        pages = paginate.Page(messages,
                              page=int(page),
                              items_per_page=num_items,
                              item_count=msgcount)
        if format == 'json':
            response.headers['Content-Type'] = 'application/json'
            data = convert_to_json(pages,
                                   direction=direction,
                                   order_by=order_by,
                                   section=section)
            return data
        c.page = pages
        return self.render('/messages/quarantine.html')