def update_queue_stats(hostname): "Update queue stats" inqdir = get_config_option('IncomingQueueDir') outqdir = get_config_option('OutgoingQueueDir') allids, inqueue = process_queue(inqdir, 1) tmpids, outqueue = process_queue(outqdir, 2) allids.extend(tmpids) dbids = [item.messageid for item in Session.query(MailQueueItem.messageid) .filter(MailQueueItem.hostname == hostname.decode('utf-8')).all()] remids = [item for item in dbids if item not in allids] preids = [item for item in dbids if item not in remids] if remids: print >> sys.stderr, ("== Deleting %(items)d queue " "items from DB ==" % dict(items=len(remids))) Session.query(MailQueueItem)\ .filter(MailQueueItem.messageid.in_(remids))\ .delete(synchronize_session='fetch') Session.commit() populate_db(inqueue, inqdir, 1, preids) populate_db(outqueue, outqdir, 2, preids)
def update_queue_stats(hostname): "Update queue stats" inqdir = get_config_option('IncomingQueueDir') outqdir = get_config_option('OutgoingQueueDir') allids, inqueue = process_queue(inqdir, 1) tmpids, outqueue = process_queue(outqdir, 2) allids.extend(tmpids) dbids = [item.messageid for item in Session.query(MailQueueItem.messageid)\ .filter(MailQueueItem.hostname == hostname)\ .all()] remids = [item for item in dbids if not item in allids] preids = [item for item in dbids if not item in remids] if remids: print >> sys.stderr, ("== Deleting %(items)d queue " "items from DB ==" % dict(items=len(remids))) Session.query(MailQueueItem)\ .filter(MailQueueItem.messageid.in_(remids))\ .delete(synchronize_session='fetch') Session.commit() populate_db(inqueue, inqdir, 1, preids) populate_db(outqueue, outqdir, 2, preids)
def search_queue(msgid, direction): "search queue" if direction == 1: qdir = get_config_option('Incoming Queue Dir') else: qdir = get_config_option('Outgoing Queue Dir') header = os.path.join(qdir, '%s-H' % msgid) data = os.path.join(qdir, '%s-D' % msgid) if os.path.exists(header) and os.path.exists(data): return header return None
def process_queued_msgs(msgids, action, direction, *args): "Process queued messages" try: logger = process_queued_msgs.get_logger() eximcmd = get_config_option('Sendmail2') if direction == 2 else 'exim' if 'exim' not in eximcmd: logger.info("Invalid exim command: %s" % eximcmd) return if direction == 1 and action not in ['bounce', 'delete']: logger.info("Invalid action: %s" % action) return exim_user = config.get('baruwa.mail.user', 'exim') queue = EximQueue('sudo -u %s %s' % (exim_user, eximcmd)) func = getattr(queue, action) msgids = [msgid for msgid in msgids if EXIM_MSGID_RE.match(msgid)] func(msgids, *args) for result in queue.results: logger.info("STDOUT: %s" % result) if queue.errors: for errmsg in queue.errors: logger.info("STDERR: %s" % errmsg) hostname = system_hostname() update_queue_stats(hostname) except TypeError, error: logger.info("Invalid input: %s" % error)
def main(argv): "Main function" parser = OptionParser() parser.add_option('-c', '--config', dest="settingsfile", help="Baruwa configuration file", default='/etc/baruwa/production.ini') parser.add_option('-e', '--disable-exim-messageid', dest="eximid", help="Disable the filtering of exim message id's", action="store_false", default=True) parser.add_option('-d', '--delete-ophans', dest="delmsg", help="Delete ophaned messages", action="store_true", default=False) options, _ = parser.parse_args(argv) basepath = os.path.dirname(os.path.dirname(__file__)) configfile = os.path.join(basepath, options.settingsfile) if not os.path.exists(configfile): print parser.print_help() print "The config file %s does not exist" % configfile sys.exit(2) config = load_config(configfile) sqlalchemyurl = config.get('app:main', 'sqlalchemy.url', vars=dict(here=basepath)) engine = create_engine(sqlalchemyurl) Session = sessionmaker(bind=engine) session = Session() quarantine = get_config_option('QuarantineDir') count = 0 for (dirname, _, files) in os.walk(quarantine): for mail in files: if mail == 'message': mail = os.path.dirname(dirname) if mail.startswith('.'): continue if options.eximid and not EXIM_MSGID_RE.match(mail): continue count += 1 msg = session.query(Message.id)\ .filter(Message.messageid == mail)\ .all() if not msg: msg = session.query(Archive.id)\ .filter(Archive.messageid == mail)\ .all() if not msg: print "%(m)s not found" % dict(m=mail) filename = os.path.join(dirname, mail) if options.delmsg: os.unlink(filename) print '-' * 100 print '%(c)d messages found' % dict(c=count)
def command(self): "command" self.init() days_to_retain = int(self.conf.get('ms.quarantine.days_to_keep', 0)) quarantine_dir = get_config_option('QuarantineDir') if (quarantine_dir.startswith('/etc') or quarantine_dir.startswith('/lib') or quarantine_dir.startswith('/home') or quarantine_dir.startswith('/bin') or quarantine_dir.startswith('..')): return False if (not os.path.exists(quarantine_dir)) or (days_to_retain == 0): return False ignore_dirs = ['spam', 'mcp', 'nonspam'] def process_dir(dirs, process_path, direc): "process dirs" if os.path.exists(os.path.join(process_path, direc)): dirs.extend( [f for f in os.listdir(os.path.join(process_path, direc))]) dirs = [ f for f in os.listdir(quarantine_dir) if os.path.isdir(os.path.join(quarantine_dir, f)) and QDIR.match(f) and should_be_pruned(f, days_to_retain) ] dirs.sort() for direc in dirs: process_path = os.path.join(quarantine_dir, direc) ids = [f for f in os.listdir(process_path) if f not in ignore_dirs] for ignore_dir in ignore_dirs: process_dir(ids, process_path, ignore_dir) sql = Message.__table__.update()\ .where(Message.messageid.in_(ids))\ .values(isquarantined=0) Session.bind.execute(sql) if (os.path.isabs(process_path) and (not os.path.islink(process_path))): try: shutil.rmtree(process_path) except shutil.Error: print >> sys.stderr, ("Failed to remove %(path)s" % dict(path=process_path)) else: print >> sys.stderr, ("%(path)s is a symlink skipping" % dict(path=process_path))
def search_quarantine(date, message_id, msgfiles): """search_quarantine""" if msgfiles: if ':' in msgfiles: filedir = msgfiles.split(':')[0] file_path = os.path.join(filedir, 'message') isdir = True else: file_path = msgfiles isdir = False if os.path.isdir(file_path): file_path = os.path.join(file_path, 'message') isdir = True return file_path, isdir else: qdir = get_config_option('Quarantine Dir') date = "%s" % date date = date.replace('-', '') return get_message_path(qdir, date, message_id)
def process_queued_msgs(msgids, action, direction, *args): "Process queued messages" try: logger = process_queued_msgs.get_logger() eximcmd = get_config_option('Sendmail2') if direction == 2 else 'exim' if not 'exim' in eximcmd: logger.info(_("Invalid exim command: %s") % eximcmd) return if direction == 1 and action not in ['bounce', 'delete']: logger.info(_("Invalid action: %s") % action) return queue = EximQueue('sudo -u exim ' + eximcmd) func = getattr(queue, action) msgids = [msgid for msgid in msgids if EXIM_MSGID_RE.match(msgid)] func(msgids, *args) for result in queue.results: logger.info(_("STDOUT: %s") % result) if queue.errors: for errmsg in queue.errors: logger.info(_("STDERR: %s") % errmsg) update_queue_stats() except TypeError, error: logger.info(_("Invalid input: %s") % error)
def search_quarantine(date, message_id): """search_quarantine""" qdir = get_config_option('Quarantine Dir') date = "%s" % date date = date.replace('-', '') return get_message_path(qdir, date, message_id)
sqlalchemyurl = config.get('app:main', 'sqlalchemy.url', vars=dict(here=basepath)) sphinxurl = config.get('app:main', 'sphinx.url', vars=dict(here=basepath)) engine = create_engine(sqlalchemyurl) # sphinxengine = create_engine(sphinxurl) Session = sessionmaker(bind=engine) # SphinxSession = sessionmaker(bind=sphinxengine) session = Session() # sphinxsession = SphinxSession() mboxdir = options.mboxdir #count = 0 messages = [] archived = [] servername = hostname() msquarantine = get_config_option('QuarantineDir') cutoff = parse('01-01-05') qdirs = ["spam", "nonspam"] for (dirname, dirs, files) in os.walk(mboxdir): #print_ '*' * 100 print 'Processing: %(d)s' % dict(d=dirname) for mail in files: if mail.startswith('.'): continue count = 0 filename = os.path.join(dirname, mail) print "Processing mailbox: %s" % mail for message in mailbox.mbox(filename): try: msgdatetime = parse(message['date'], ignoretz=True) msgtimestamp = msgdatetime.strftime("%Y-%m-%d %H:%M:%S")
def main(argv): "main function" try: conn = SpamdConnection(socket='/var/run/spamassassin/spamd.sock') usage = """ usage: %prog [options] options: -c --config "configuration file" -i --inputdir "mbox input directory" -o --outputdir "mbox output directory" """ parser = OptionParser(usage) parser.add_option('-c', '--config', dest="settingsfile", help="Baruwa configuration file", default='/etc/baruwa/production.ini') parser.add_option('-i', '--inputdir', dest="mboxdir", help="Mbox directory") parser.add_option('-o', '--outputdir', dest="outputdir", help="Output directory") options, _ = parser.parse_args(argv) if not options.mboxdir: print usage print "Please specify the directory with the mbox files" sys.exit(2) if not options.outputdir: print usage print "Please specify the directory to store processed files" sys.exit(2) basepath = os.path.dirname(os.path.dirname(__file__)) configfile = os.path.join(basepath, options.settingsfile) if not os.path.exists(configfile): print parser.print_help() print "The config file %s does not exist" % configfile sys.exit(2) config = load_config(configfile) sqlalchemyurl = config.get('app:main', 'sqlalchemy.url', vars=dict(here=basepath)) sphinxurl = config.get('app:main', 'sphinx.url', vars=dict(here=basepath)) engine = create_engine(sqlalchemyurl) # sphinxengine = create_engine(sphinxurl) Session = sessionmaker(bind=engine) # SphinxSession = sessionmaker(bind=sphinxengine) session = Session() # sphinxsession = SphinxSession() mboxdir = options.mboxdir # count = 0 messages = [] archived = [] servername = hostname() msquarantine = get_config_option('QuarantineDir') cutoff = parse('01-01-05') qdirs = ["spam", "nonspam"] for (dirname, _, files) in os.walk(mboxdir): # print_ '*' * 100 print 'Processing: %(d)s' % dict(d=dirname) for mail in files: if mail.startswith('.'): continue count = 0 filename = os.path.join(dirname, mail) print "Processing mailbox: %s" % mail for message in mailbox.mbox(filename): try: msgdatetime = parse(message['date'], ignoretz=True) msgdate = msgdatetime.strftime("%Y-%m-%d") # msgtime = msgdatetime.strftime("%H:%M:%S") dirdate = msgdate.replace('-', '') # quarantinedir = os.path.join(basepath, 'data', # 'quarantine', dirdate) quarantinedir = os.path.join(msquarantine, dirdate) if not os.path.exists(quarantinedir): os.mkdir(quarantinedir) os.mkdir(os.path.join(quarantinedir, 'spam')) os.mkdir(os.path.join(quarantinedir, 'nonspam')) messageid = parseaddr(message['message-id'])[1] messageid = messageid.replace('/', '.') # messagepath = os.path.join(quarantinedir, messageid) exists = False for message_kind in qdirs: messagepath = os.path.join(quarantinedir, message_kind, messageid) if os.path.exists(messagepath): exists = True break if exists: # print_ "Skipping message with id: %s" % messageid continue count += 1 message_kind = 'nonspam' received = message.get_all('Received') or [] fromip = get_hostname(received) fromaddr = parseaddr(message['from'])[1] toaddr = parseaddr(message['to'])[1] fromdomain = fromaddr.split('@')[1] todomain = toaddr.split('@')[1] msgheaders = convert_headers(message.items()) if msgdatetime.date() < cutoff.date(): msg = Archive(messageid=messageid) else: msg = Message(messageid=messageid) msg.actions = "deliver" msg.clientip = fromip msg.from_address = fromaddr.lower() msg.from_domain = fromdomain.lower() msg.to_address = toaddr.lower() msg.to_domain = todomain.lower() msg.hostname = servername msg.timestamp = msgdatetime msg.date = msgdatetime.date() msg.time = msgdatetime.time() msg.subject = get_header(message['subject']) msg.headers = msgheaders conn.addheader('User', 'andrew') conn.check(SYMBOLS, message.as_string()) isspam, msg.sascore = conn.getspamstatus() msg.spam = int(isspam) msg.spamreport = generate_sareport(msg.spam, msg.sascore, conn.response_message, session) msg.size = len(message.as_string()) msg.isquarantined = 1 if msg.spam: msg.actions = "store" message_kind = 'spam' msg.scaned = 1 if msg.sascore > 10: msg.highspam = 1 if msgdatetime.date() < cutoff.date(): messages.append(msg) else: archived.append(msg) messagepath = os.path.join(quarantinedir, message_kind, messageid) msghandle = open(messagepath, 'w') msghandle.write(message.as_string()) msghandle.close() # print_ "Processed: %s" % messageid if (count % 100) == 0: flush2db(session, messages, archived, count, sphinxurl) else: print_(" Processed: %(c)d" % dict(c=count)) except (IndexError, ValueError, AttributeError, IOError): pass outfile = os.path.join(options.outputdir, mail) shutil.move(filename, outfile) flush2db(session, messages, archived, count, sphinxurl) except KeyboardInterrupt: if 'session' in locals() and (messages or archived): flush2db(session, messages, archived, count, sphinxurl) print "\nExiting..."
def main(argv): "main function" try: conn = SpamdConnection(socket='/var/run/spamassassin/spamd.sock') usage = """ usage: %prog [options] options: -c --config "configuration file" -i --inputdir "mbox input directory" -o --outputdir "mbox output directory" """ parser = OptionParser(usage) parser.add_option('-c', '--config', dest="settingsfile", help="Baruwa configuration file", default='/etc/baruwa/production.ini') parser.add_option('-i', '--inputdir', dest="mboxdir", help="Mbox directory") parser.add_option('-o', '--outputdir', dest="outputdir", help="Output directory") options, _ = parser.parse_args(argv) if not options.mboxdir: print usage print "Please specify the directory with the mbox files" sys.exit(2) if not options.outputdir: print usage print "Please specify the directory to store processed files" sys.exit(2) basepath = os.path.dirname(os.path.dirname(__file__)) configfile = os.path.join(basepath, options.settingsfile) if not os.path.exists(configfile): print parser.print_help() print "The config file %s does not exist" % configfile sys.exit(2) config = load_config(configfile) sqlalchemyurl = config.get('app:main', 'sqlalchemy.url', vars=dict(here=basepath)) sphinxurl = config.get('app:main', 'sphinx.url', vars=dict(here=basepath)) engine = create_engine(sqlalchemyurl) # sphinxengine = create_engine(sphinxurl) Session = sessionmaker(bind=engine) # SphinxSession = sessionmaker(bind=sphinxengine) session = Session() # sphinxsession = SphinxSession() mboxdir = options.mboxdir # count = 0 messages = [] archived = [] servername = hostname() msquarantine = get_config_option('QuarantineDir') cutoff = parse('01-01-05') qdirs = ["spam", "nonspam"] for (dirname, _, files) in os.walk(mboxdir): # print_ '*' * 100 print 'Processing: %(d)s' % dict(d=dirname) for mail in files: if mail.startswith('.'): continue count = 0 filename = os.path.join(dirname, mail) print "Processing mailbox: %s" % mail for message in mailbox.mbox(filename): try: msgdatetime = parse(message['date'], ignoretz=True) msgdate = msgdatetime.strftime("%Y-%m-%d") # msgtime = msgdatetime.strftime("%H:%M:%S") dirdate = msgdate.replace('-', '') # quarantinedir = os.path.join(basepath, 'data', # 'quarantine', dirdate) quarantinedir = os.path.join(msquarantine, dirdate) if not os.path.exists(quarantinedir): os.mkdir(quarantinedir) os.mkdir(os.path.join(quarantinedir, 'spam')) os.mkdir(os.path.join(quarantinedir, 'nonspam')) messageid = parseaddr(message['message-id'])[1] messageid = messageid.replace('/', '.') # messagepath = os.path.join(quarantinedir, messageid) exists = False for message_kind in qdirs: messagepath = os.path.join(quarantinedir, message_kind, messageid) if os.path.exists(messagepath): exists = True break if exists: # print_ "Skipping message with id: %s" % messageid continue count += 1 message_kind = 'nonspam' received = message.get_all('Received') or [] fromip = get_hostname(received) fromaddr = parseaddr(message['from'])[1] toaddr = parseaddr(message['to'])[1] fromdomain = fromaddr.split('@')[1] todomain = toaddr.split('@')[1] msgheaders = convert_headers(message.items()) if msgdatetime.date() < cutoff.date(): msg = Archive(messageid=messageid) else: msg = Message(messageid=messageid) msg.actions = "deliver" msg.clientip = fromip msg.from_address = fromaddr.lower() msg.from_domain = fromdomain.lower() msg.to_address = toaddr.lower() msg.to_domain = todomain.lower() msg.hostname = servername msg.timestamp = msgdatetime msg.date = msgdatetime.date() msg.time = msgdatetime.time() msg.subject = get_header(message['subject']) msg.headers = msgheaders conn.addheader('User', 'andrew') conn.check(SYMBOLS, message.as_string()) isspam, msg.sascore = conn.getspamstatus() msg.spam = int(isspam) msg.spamreport = generate_sareport( msg.spam, msg.sascore, conn.response_message, session) msg.size = len(message.as_string()) msg.isquarantined = 1 if msg.spam: msg.actions = "store" message_kind = 'spam' msg.scaned = 1 if msg.sascore > 10: msg.highspam = 1 if msgdatetime.date() < cutoff.date(): messages.append(msg) else: archived.append(msg) messagepath = os.path.join(quarantinedir, message_kind, messageid) msghandle = open(messagepath, 'w') msghandle.write(message.as_string()) msghandle.close() # print_ "Processed: %s" % messageid if (count % 100) == 0: flush2db(session, messages, archived, count, sphinxurl) else: print_(" Processed: %(c)d" % dict(c=count)) except (IndexError, ValueError, AttributeError, IOError): pass outfile = os.path.join(options.outputdir, mail) shutil.move(filename, outfile) flush2db(session, messages, archived, count, sphinxurl) except KeyboardInterrupt: if 'session' in locals() and (messages or archived): flush2db(session, messages, archived, count, sphinxurl) print "\nExiting..."
def command(self): "command" self.init() if asbool(self.conf.get('ms.quarantine.shared')): lock_name = 'cleanquarantine' else: lock_name = 'cleanquarantine-%s' % system_hostname() if acquire_lock(lock_name, self.conf): try: days_to_retain = int( self.conf.get('ms.quarantine.days_to_keep', 0)) quarantine_dir = get_config_option('QuarantineDir') if (quarantine_dir.startswith(('/etc', '/lib', '/home', '/bin', '/sbin', '..'))): return False if ((not os.path.exists(quarantine_dir)) or (days_to_retain == 0)): return False ignore_dirs = ['spam', 'mcp', 'nonspam'] def process_dir(dirs, process_path, direc): "process dirs" if os.path.exists(os.path.join(process_path, direc)): dirs.extend([f for f in os.listdir( os.path.join(process_path, direc))]) dirs = [f for f in os.listdir(quarantine_dir) if os.path.isdir(os.path.join(quarantine_dir, f)) and QDIR.match(f) and should_be_pruned(f, days_to_retain)] dirs.sort() for direc in dirs: process_path = os.path.join(quarantine_dir, direc) ids = [f for f in os.listdir(process_path) if f not in ignore_dirs] for ignore_dir in ignore_dirs: process_dir(ids, process_path, ignore_dir) year, month, day = (int(direc[:4]), int(direc[4:-2]), int(direc[6:])) startdate = datetime.datetime(year, month, day, 00, 00, 00) enddate = datetime.datetime(year, month, day, 23, 59, 59) localzone = make_tz(self.conf['baruwa.timezone']) startdate = localzone.localize(startdate) enddate = localzone.localize(enddate) startdate = pytz.utc.normalize( startdate.astimezone(pytz.utc)) enddate = pytz.utc.normalize(enddate.astimezone(pytz.utc)) sql = Message.__table__.update().where(and_( Message.messageid.in_(ids), Message.timestamp.between(startdate, enddate) )).values(isquarantined=0) Session.bind.execute(sql) if (os.path.isabs(process_path) and (not os.path.islink(process_path))): try: shutil.rmtree(process_path) except shutil.Error: print >> sys.stderr, ("Failed to remove %(path)s" % dict(path=process_path)) else: print >> sys.stderr, ("%(path)s is a symlink skipping" % dict(path=process_path)) finally: Session.close() release_lock(lock_name, self.conf)
parser.add_option('-c', '--config', dest="settingsfile", help="Baruwa configuration file", default='/etc/baruwa/production.ini') options, args = parser.parse_args() basepath = os.path.dirname(os.path.dirname(__file__)) configfile = os.path.join(basepath, options.settingsfile) config = load_config(configfile) sqlalchemyurl = config.get('app:main', 'sqlalchemy.url', vars=dict(here=basepath)) engine = create_engine(sqlalchemyurl) Session = sessionmaker(bind=engine) session = Session() quarantine = get_config_option('QuarantineDir') count = 0 for (dirname, dirs, files) in os.walk(quarantine): for mail in files: if mail.startswith('.'): continue count += 1 msg = session.query(Message.id)\ .filter(Message.messageid==mail)\ .all() if not msg: msg = session.query(Archive.id)\ .filter(Archive.messageid==mail)\ .all() if not msg: print "%(m)s not found" % dict(m=mail)
if __name__ == "__main__": # run the thing parser = OptionParser() parser.add_option( "-c", "--config", dest="settingsfile", help="Baruwa configuration file", default="/etc/baruwa/production.ini" ) options, args = parser.parse_args() basepath = os.path.dirname(os.path.dirname(__file__)) configfile = os.path.join(basepath, options.settingsfile) config = load_config(configfile) sqlalchemyurl = config.get("app:main", "sqlalchemy.url", vars=dict(here=basepath)) engine = create_engine(sqlalchemyurl) Session = sessionmaker(bind=engine) session = Session() quarantine = get_config_option("QuarantineDir") count = 0 for (dirname, dirs, files) in os.walk(quarantine): for mail in files: if mail.startswith("."): continue count += 1 msg = session.query(Message.id).filter(Message.messageid == mail).all() if not msg: msg = session.query(Archive.id).filter(Archive.messageid == mail).all() if not msg: print "%(m)s not found" % dict(m=mail) filename = os.path.join(dirname, mail) os.unlink(filename) print "-" * 100 print "%(c)d messages found" % dict(c=count)