def process_queued_msgs(msgids, action, direction, *args): "Process queued messages" try: logger = process_queued_msgs.get_logger() eximcmd = get_config_option('Sendmail2') if direction == 2 else 'exim' if 'exim' not in eximcmd: logger.info("Invalid exim command: %s" % eximcmd) return if direction == 1 and action not in ['bounce', 'delete']: logger.info("Invalid action: %s" % action) return exim_user = config.get('baruwa.mail.user', 'exim') queue = EximQueue('sudo -u %s %s' % (exim_user, eximcmd)) func = getattr(queue, action) msgids = [msgid for msgid in msgids if EXIM_MSGID_RE.match(msgid)] func(msgids, *args) for result in queue.results: logger.info("STDOUT: %s" % result) if queue.errors: for errmsg in queue.errors: logger.info("STDERR: %s" % errmsg) hostname = system_hostname() update_queue_stats(hostname) except TypeError, error: logger.info("Invalid input: %s" % error)
def autorelease(self, uuid): "release a message without logging in" releasereq = get_releasereq(uuid) if not releasereq: abort(404) released = False msgid = releasereq.messageid to_address = _('Unknown') errormsg = _('The message has already been released,' ' you can only use the release link once') if releasereq.released is False: try: msg = Session.query(Message)\ .filter(Message.id == releasereq.messageid)\ .one() except (NoResultFound, MultipleResultsFound): abort(404) msgid = msg.messageid to_address = msg.to_address try: if msg.isdangerous and c.user and c.user.is_peleb: raise ValueError localtmz = config.get('baruwa.timezone', 'Africa/Johannesburg') cdte = convert_date(msg.timestamp, localtmz).strftime('%Y%m%d') task = release_message.apply_async( args=[msg.messageid, cdte, msg.from_address, msg.to_address.split(','), msg.msgfiles], routing_key=system_hostname() if asbool(config.get('ms.quarantine.shared', 'false')) else msg.hostname.strip()) task.wait(30) if task.status == 'SUCCESS': result = task.result if result['success']: update_autorelease.apply_async(args=[uuid]) errormsg = result['error'] released = result['success'] except (ValueError, socket.error, TimeoutError, QueueNotFound): released = False errormsg = _('Processing of message failed') log.info(errormsg) result = dict(success=False, error=errormsg) c.messageid = msgid c.errormsg = errormsg c.released = released c.releaseaddress = to_address return self.render('/messages/autorelease.html')
def autorelease(self, uuid): "release a message without logging in" releasereq = get_releasereq(uuid) if not releasereq: abort(404) released = False msgid = releasereq.messageid to_address = _('Unknown') errormsg = _('The message has already been released,' ' you can only use the release link once') if releasereq.released is False: try: msg = Session.query(Message)\ .filter(Message.id == releasereq.messageid)\ .one() except (NoResultFound, MultipleResultsFound): abort(404) msgid = msg.messageid to_address = msg.to_address try: if msg.isdangerous and c.user and c.user.is_peleb: raise ValueError localtmz = config.get('baruwa.timezone', 'Africa/Johannesburg') cdte = convert_date(msg.timestamp, localtmz).strftime('%Y%m%d') task = release_message.apply_async( args=[ msg.messageid, cdte, msg.from_address, msg.to_address.split(','), msg.msgfiles ], routing_key=system_hostname() if asbool( config.get('ms.quarantine.shared', 'false')) else msg.hostname.strip()) task.wait(30) if task.status == 'SUCCESS': result = task.result if result['success']: update_autorelease.apply_async(args=[uuid]) errormsg = result['error'] released = result['success'] except (ValueError, socket.error, TimeoutError, QueueNotFound): released = False errormsg = _('Processing of message failed') log.info(errormsg) result = dict(success=False, error=errormsg) c.messageid = msgid c.errormsg = errormsg c.released = released c.releaseaddress = to_address return self.render('/messages/autorelease.html')
def command(self): "run command" self.init() try: lockfile = os.path.join(self.conf['baruwa.locks.dir'], 'queuestats.lock') with open(lockfile, 'w+') as lock: fcntl.lockf(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) hostname = system_hostname() update_queue_stats(hostname) except IOError: warnings.warn("Queuestats already running !") sys.exit(2)
def command(self): "run command" self.init() try: lockfile = os.path.join(self.conf['baruwa.locks.dir'], 'queuestats.lock') with open(lockfile, 'w+') as lock: fcntl.lockf(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) hostname = system_hostname() update_queue_stats(hostname) except IOError: warnings.warn("Queuestats already running !") sys.exit(2) finally: Session.close()
def command(self): "command" self.init() if asbool(self.conf.get('ms.quarantine.shared')): lock_name = 'cleanquarantine' else: lock_name = 'cleanquarantine-%s' % system_hostname() if acquire_lock(lock_name, self.conf): try: days_to_retain = int( self.conf.get('ms.quarantine.days_to_keep', 0)) quarantine_dir = get_config_option('QuarantineDir') if (quarantine_dir.startswith(('/etc', '/lib', '/home', '/bin', '/sbin', '..'))): return False if ((not os.path.exists(quarantine_dir)) or (days_to_retain == 0)): return False ignore_dirs = ['spam', 'mcp', 'nonspam'] def process_dir(dirs, process_path, direc): "process dirs" if os.path.exists(os.path.join(process_path, direc)): dirs.extend([f for f in os.listdir( os.path.join(process_path, direc))]) dirs = [f for f in os.listdir(quarantine_dir) if os.path.isdir(os.path.join(quarantine_dir, f)) and QDIR.match(f) and should_be_pruned(f, days_to_retain)] dirs.sort() for direc in dirs: process_path = os.path.join(quarantine_dir, direc) ids = [f for f in os.listdir(process_path) if f not in ignore_dirs] for ignore_dir in ignore_dirs: process_dir(ids, process_path, ignore_dir) year, month, day = (int(direc[:4]), int(direc[4:-2]), int(direc[6:])) startdate = datetime.datetime(year, month, day, 00, 00, 00) enddate = datetime.datetime(year, month, day, 23, 59, 59) localzone = make_tz(self.conf['baruwa.timezone']) startdate = localzone.localize(startdate) enddate = localzone.localize(enddate) startdate = pytz.utc.normalize( startdate.astimezone(pytz.utc)) enddate = pytz.utc.normalize(enddate.astimezone(pytz.utc)) sql = Message.__table__.update().where(and_( Message.messageid.in_(ids), Message.timestamp.between(startdate, enddate) )).values(isquarantined=0) Session.bind.execute(sql) if (os.path.isabs(process_path) and (not os.path.islink(process_path))): try: shutil.rmtree(process_path) except shutil.Error: print >> sys.stderr, ("Failed to remove %(path)s" % dict(path=process_path)) else: print >> sys.stderr, ("%(path)s is a symlink skipping" % dict(path=process_path)) finally: Session.close() release_lock(lock_name, self.conf)
def preview(self, msgid, archive=None, attachment=None, img=None, allowimgs=None, richformat=None): """Preview a message stored in the quarantine :param msgid: the database message id :param archive: optional. message archived status :param attachment: optional. request is for an attachmeny :param img: optional request is for an image :param allowimgs: optional allow display of remote images :param richformat: show html format """ if archive: message = self._get_archive(msgid) else: message = self._get_message(msgid) if not message: abort(404) try: if message.isdangerous and c.user.is_peleb: raise ValueError localtmz = config.get('baruwa.timezone', 'Africa/Johannesburg') cdte = convert_date(message.timestamp, localtmz).strftime('%Y%m%d') args = [ message.messageid, cdte, message.msgfiles, attachment, img, allowimgs ] task = preview_msg.apply_async( args=args, routing_key=system_hostname() if asbool( config.get('ms.quarantine.shared', 'false')) else message.hostname.strip()) task.wait(30) if task.result: if img: if message.isdangerous and c.user.is_peleb: abort(404) response.content_type = task.result['content_type'] if task.result and 'img' in task.result: info = MSGDOWNLOAD_MSG % dict(m=message.id, a=task.result['name']) audit_log(c.user.username, 1, unicode(info), request.host, request.remote_addr, arrow.utcnow().datetime) return base64.decodestring(task.result['img']) abort(404) if attachment: info = MSGDOWNLOAD_MSG % dict(m=message.id, a=task.result['name']) audit_log(c.user.username, 1, unicode(info), request.host, request.remote_addr, arrow.utcnow().datetime) response.content_type = task.result['mimetype'] content_disposition = 'attachment; filename="%s"' % \ task.result['name'].encode('ascii', 'replace') response.headers['Content-Disposition'] = \ str(content_disposition) response.headers['Content-Length'] = \ len(task.result['attachment']) response.headers['Pragma'] = 'public' response.headers['Cache-Control'] = 'max-age=0' return base64.decodestring(task.result['attachment']) for part in task.result['parts']: if part['type'] == 'text/html': local_rf = (not task.result['is_multipart'] or richformat) part['content'] = image_fixups(part['content'], msgid, archive, local_rf, allowimgs) c.message = task.result info = MSGPREVIEW_MSG % dict(m=message.id) audit_log(c.user.username, 1, unicode(info), request.host, request.remote_addr, arrow.utcnow().datetime) else: c.message = {} except (socket.error, TimeoutError, QueueNotFound): lmsg = _('The message could not be previewed, try again later') flash_alert(lmsg) log.info(lmsg) whereto = url('message-archive', msgid=msgid) if archive \ else url('message-detail', msgid=msgid) redirect(whereto) except ValueError: lmsg = _('The message/attachments are either prohibited or' ' dangerous. Contact your system admin for assistance') flash_alert(lmsg) log.info(lmsg) whereto = url('message-archive', msgid=msgid) if archive \ else url('message-detail', msgid=msgid) redirect(whereto) c.messageid = message.messageid c.msgid = message.id c.archived = archive c.richformat = richformat c.isdangerous = message.isdangerous # print c.message return self.render('/messages/preview.html')
def quarantine(self, page=1, direction='dsc', order_by='timestamp', section=None, format=None): "quarantined messages" filters = session.get('filter_by', None) num_items = session.get('msgs_num_items', 50) if direction == 'dsc': sort = desc(order_by) else: sort = order_by messages = get_messages().filter( Message.isquarantined == 1).order_by(sort) msgcount = get_msg_count().filter(Message.isquarantined == 1) query = UserFilter(Session, c.user, messages) countquery = UserFilter(Session, c.user, msgcount) messages = query.filter() msgcount = countquery.filter() if section: if section == 'spam': messages = messages.filter(Message.spam == 1) msgcount = messages.filter(Message.spam == 1) else: messages = messages.filter(Message.spam == 0) msgcount = messages.filter(Message.spam == 0) if filters: dynq = DynaQuery(Message, messages, filters) dynmsgq = DynaQuery(Message, msgcount, filters) messages = dynq.generate() msgcount = dynmsgq.generate() c.order_by = order_by c.direction = direction c.section = section msgcount = msgcount.count() c.form = BulkReleaseForm(request.POST, csrf_context=session) if request.method == 'POST': choices = session.get('bulk_items', []) else: pages = paginate.Page(messages, page=int(page), items_per_page=num_items, item_count=msgcount) choices = [(str(message.id), message.id) for message in pages.items] session['bulk_items'] = choices session.save() c.form.message_id.choices = choices if request.method == 'POST' and c.form.validate() and choices: msgs = Session.query(Message.id, Message.messageid, Message.from_address, Message.timestamp, Message.to_address, Message.hostname, Message.msgfiles)\ .filter(Message.id.in_(c.form.message_id.data)) query = UserFilter(Session, c.user, msgs) msgs = query.filter() localtmz = config.get('baruwa.timezone', 'Africa/Johannesburg') formvals = (dict(release=c.form.release.data, learn=c.form.learn.data, salearn_as=c.form.learnas.data, todelete=c.form.delete.data, use_alt=c.form.usealt.data, altrecipients=c.form.altrecipients.data, message_id=msg.messageid, from_address=msg.from_address, date=convert_date(msg.timestamp, localtmz).strftime('%Y%m%d'), msgfiles=msg.msgfiles, to_address=msg.to_address, hostname=msg.hostname, mid=msg.id, num=num_items) for msg in msgs) if formvals: try: subtasks = [ process_quarantined_msg.subtask( args=[formval], options=dict( routing_key=system_hostname() if asbool( config.get('ms.quarantine.shared', 'false') ) else formval['hostname'])) for formval in formvals ] task = group(subtasks).apply_async() task.save(backend=RBACKEND) session['bulk_items'] = [] if 'taskids' not in session: session['taskids'] = [] session['taskids'].append(task.id) session['bulkprocess-count'] = 1 session.save() redirect(url('messages-bulk-process', taskid=task.id)) except (QueueNotFound, OperationalError, IndexError): flash_alert( _('The messages could not processed' ', try again later')) elif request.method == 'POST' and not c.form.validate(): try: flash_alert( _(u', '.join([ unicode(c.form.errors[err][0]) for err in c.form.errors ]))) except IndexError: flash_alert( _('The messages could not processed' ', an Unknown error occured.')) pages = paginate.Page(messages, page=int(page), items_per_page=num_items, item_count=msgcount) if format == 'json': response.headers['Content-Type'] = 'application/json' data = convert_to_json(pages, direction=direction, order_by=order_by, section=section) return data c.page = pages return self.render('/messages/quarantine.html')
def detail(self, msgid, archive=None, format=None): "return message detail" message = self._get_msg(msgid, archive) if not message: abort(404) msgstatus = Session.query(MessageStatus)\ .filter(MessageStatus.messageid == message.messageid)\ .all() show_trail = request.GET.get('t', None) c.form = ReleaseMsgForm(request.POST, csrf_context=session) if request.method == 'POST' and c.form.validate(): localtmz = config.get('baruwa.timezone', 'Africa/Johannesburg') job = dict(release=c.form.release.data, learn=c.form.learn.data, salearn_as=c.form.learnas.data, todelete=c.form.delete.data, use_alt=c.form.usealt.data, altrecipients=c.form.altrecipients.data, message_id=message.messageid, from_address=message.from_address, date=convert_date(message.timestamp, localtmz).strftime('%Y%m%d'), msgfiles=message.msgfiles, to_address=message.to_address, hostname=message.hostname, mid=message.id, issingle=True) try: task = process_quarantined_msg.apply_async( args=[job], routing_key=system_hostname() if asbool( config.get('ms.quarantine.shared', 'false')) else message.hostname) task.wait(30) if task.status == 'SUCCESS': # process response html = process_release_results(c, message, task.result, Session, self.render) self.invalidate = 1 message = self._get_msg(msgid, archive) flash(html) else: html = _('Processing the request failed') flash_alert(html) log.info(html) except (TimeoutError, QueueNotFound): html = _('Processing the request failed') flash_alert(html) log.info(html) if format == 'json': flash.pop_messages() response.headers['Content-Type'] = 'application/json' return json.dumps(dict(result=html)) elif request.method == 'POST' and not c.form.validate(): flash_alert( _(u', '.join([ unicode(c.form.errors[err][0]) for err in c.form.errors ]))) if format == 'json': html = flash.pop_messages() response.headers['Content-Type'] = 'application/json' return json.dumps(dict(result=unicode(html[0]))) if format == 'json': response.headers['Content-Type'] = 'application/json' return json.dumps(message.tojson) c.msg = message c.archived = archive c.show_trail = show_trail c.msgstatus = msgstatus return self.render('/messages/detail.html')
def preview(self, msgid, archive=None, attachment=None, img=None, allowimgs=None, richformat=None): """Preview a message stored in the quarantine :param msgid: the database message id :param archive: optional. message archived status :param attachment: optional. request is for an attachmeny :param img: optional request is for an image :param allowimgs: optional allow display of remote images :param richformat: show html format """ if archive: message = self._get_archive(msgid) else: message = self._get_message(msgid) if not message: abort(404) try: if message.isdangerous and c.user.is_peleb: raise ValueError localtmz = config.get('baruwa.timezone', 'Africa/Johannesburg') cdte = convert_date(message.timestamp, localtmz).strftime('%Y%m%d') args = [message.messageid, cdte, message.msgfiles, attachment, img, allowimgs] task = preview_msg.apply_async(args=args, routing_key=system_hostname() if asbool(config.get('ms.quarantine.shared', 'false')) else message.hostname.strip()) task.wait(30) if task.result: if img: if message.isdangerous and c.user.is_peleb: abort(404) response.content_type = task.result['content_type'] if task.result and 'img' in task.result: info = MSGDOWNLOAD_MSG % dict(m=message.id, a=task.result['name']) audit_log(c.user.username, 1, unicode(info), request.host, request.remote_addr, arrow.utcnow().datetime) return base64.decodestring(task.result['img']) abort(404) if attachment: info = MSGDOWNLOAD_MSG % dict(m=message.id, a=task.result['name']) audit_log(c.user.username, 1, unicode(info), request.host, request.remote_addr, arrow.utcnow().datetime) response.content_type = task.result['mimetype'] content_disposition = 'attachment; filename="%s"' % \ task.result['name'].encode('ascii', 'replace') response.headers['Content-Disposition'] = \ str(content_disposition) response.headers['Content-Length'] = \ len(task.result['attachment']) response.headers['Pragma'] = 'public' response.headers['Cache-Control'] = 'max-age=0' return base64.decodestring(task.result['attachment']) for part in task.result['parts']: if part['type'] == 'text/html': local_rf = (not task.result['is_multipart'] or richformat) part['content'] = image_fixups( part['content'], msgid, archive, local_rf, allowimgs) c.message = task.result info = MSGPREVIEW_MSG % dict(m=message.id) audit_log(c.user.username, 1, unicode(info), request.host, request.remote_addr, arrow.utcnow().datetime) else: c.message = {} except (socket.error, TimeoutError, QueueNotFound): lmsg = _('The message could not be previewed, try again later') flash_alert(lmsg) log.info(lmsg) whereto = url('message-archive', msgid=msgid) if archive \ else url('message-detail', msgid=msgid) redirect(whereto) except ValueError: lmsg = _('The message/attachments are either prohibited or' ' dangerous. Contact your system admin for assistance') flash_alert(lmsg) log.info(lmsg) whereto = url('message-archive', msgid=msgid) if archive \ else url('message-detail', msgid=msgid) redirect(whereto) c.messageid = message.messageid c.msgid = message.id c.archived = archive c.richformat = richformat c.isdangerous = message.isdangerous # print c.message return self.render('/messages/preview.html')
def quarantine(self, page=1, direction='dsc', order_by='timestamp', section=None, format=None): "quarantined messages" filters = session.get('filter_by', None) num_items = session.get('msgs_num_items', 50) if direction == 'dsc': sort = desc(order_by) else: sort = order_by messages = get_messages().filter( Message.isquarantined == 1).order_by(sort) msgcount = get_msg_count().filter( Message.isquarantined == 1) query = UserFilter(Session, c.user, messages) countquery = UserFilter(Session, c.user, msgcount) messages = query.filter() msgcount = countquery.filter() if section: if section == 'spam': messages = messages.filter(Message.spam == 1) msgcount = messages.filter(Message.spam == 1) else: messages = messages.filter(Message.spam == 0) msgcount = messages.filter(Message.spam == 0) if filters: dynq = DynaQuery(Message, messages, filters) dynmsgq = DynaQuery(Message, msgcount, filters) messages = dynq.generate() msgcount = dynmsgq.generate() c.order_by = order_by c.direction = direction c.section = section msgcount = msgcount.count() c.form = BulkReleaseForm(request.POST, csrf_context=session) if request.method == 'POST': choices = session.get('bulk_items', []) else: pages = paginate.Page(messages, page=int(page), items_per_page=num_items, item_count=msgcount) choices = [(str(message.id), message.id) for message in pages.items] session['bulk_items'] = choices session.save() c.form.message_id.choices = choices if request.method == 'POST' and c.form.validate() and choices: msgs = Session.query(Message.id, Message.messageid, Message.from_address, Message.timestamp, Message.to_address, Message.hostname, Message.msgfiles)\ .filter(Message.id.in_(c.form.message_id.data)) query = UserFilter(Session, c.user, msgs) msgs = query.filter() localtmz = config.get('baruwa.timezone', 'Africa/Johannesburg') formvals = (dict(release=c.form.release.data, learn=c.form.learn.data, salearn_as=c.form.learnas.data, todelete=c.form.delete.data, use_alt=c.form.usealt.data, altrecipients=c.form.altrecipients.data, message_id=msg.messageid, from_address=msg.from_address, date=convert_date(msg.timestamp, localtmz) .strftime('%Y%m%d'), msgfiles=msg.msgfiles, to_address=msg.to_address, hostname=msg.hostname, mid=msg.id, num=num_items) for msg in msgs) if formvals: try: subtasks = [process_quarantined_msg.subtask(args=[formval], options=dict(routing_key=system_hostname() if asbool(config.get( 'ms.quarantine.shared', 'false')) else formval['hostname'])) for formval in formvals] task = group(subtasks).apply_async() task.save(backend=RBACKEND) session['bulk_items'] = [] if 'taskids' not in session: session['taskids'] = [] session['taskids'].append(task.id) session['bulkprocess-count'] = 1 session.save() redirect(url('messages-bulk-process', taskid=task.id)) except (QueueNotFound, OperationalError, IndexError): flash_alert(_('The messages could not processed' ', try again later')) elif request.method == 'POST' and not c.form.validate(): try: flash_alert(_(u', '.join([unicode(c.form.errors[err][0]) for err in c.form.errors]))) except IndexError: flash_alert(_('The messages could not processed' ', an Unknown error occured.')) pages = paginate.Page(messages, page=int(page), items_per_page=num_items, item_count=msgcount) if format == 'json': response.headers['Content-Type'] = 'application/json' data = convert_to_json(pages, direction=direction, order_by=order_by, section=section) return data c.page = pages return self.render('/messages/quarantine.html')
def detail(self, msgid, archive=None, format=None): "return message detail" message = self._get_msg(msgid, archive) if not message: abort(404) msgstatus = Session.query(MessageStatus)\ .filter(MessageStatus.messageid == message.messageid)\ .all() show_trail = request.GET.get('t', None) c.form = ReleaseMsgForm(request.POST, csrf_context=session) if request.method == 'POST' and c.form.validate(): localtmz = config.get('baruwa.timezone', 'Africa/Johannesburg') job = dict(release=c.form.release.data, learn=c.form.learn.data, salearn_as=c.form.learnas.data, todelete=c.form.delete.data, use_alt=c.form.usealt.data, altrecipients=c.form.altrecipients.data, message_id=message.messageid, from_address=message.from_address, date=convert_date(message.timestamp, localtmz) .strftime('%Y%m%d'), msgfiles=message.msgfiles, to_address=message.to_address, hostname=message.hostname, mid=message.id, issingle=True) try: task = process_quarantined_msg.apply_async( args=[job], routing_key=system_hostname() if asbool(config.get('ms.quarantine.shared', 'false')) else message.hostname) task.wait(30) if task.status == 'SUCCESS': # process response html = process_release_results( c, message, task.result, Session, self.render) self.invalidate = 1 message = self._get_msg(msgid, archive) flash(html) else: html = _('Processing the request failed') flash_alert(html) log.info(html) except (TimeoutError, QueueNotFound): html = _('Processing the request failed') flash_alert(html) log.info(html) if format == 'json': flash.pop_messages() response.headers['Content-Type'] = 'application/json' return json.dumps(dict(result=html)) elif request.method == 'POST' and not c.form.validate(): flash_alert(_(u', '.join([unicode(c.form.errors[err][0]) for err in c.form.errors]))) if format == 'json': html = flash.pop_messages() response.headers['Content-Type'] = 'application/json' return json.dumps(dict(result=unicode(html[0]))) if format == 'json': response.headers['Content-Type'] = 'application/json' return json.dumps(message.tojson) c.msg = message c.archived = archive c.show_trail = show_trail c.msgstatus = msgstatus return self.render('/messages/detail.html')