def scan_mail_log(logger, env): collector = { "other-services": set(), "imap-logins": {}, "postgrey": {}, "rejected-mail": {}, } collector["real_mail_addresses"] = set( mailconfig.get_mail_users(env)) | set( alias[0] for alias in mailconfig.get_mail_aliases(env)) for fn in ('/var/log/mail.log.1', '/var/log/mail.log'): if not os.path.exists(fn): continue with open(fn, 'rb') as log: for line in log: line = line.decode("utf8", errors='replace') scan_mail_log_line(line.strip(), collector) if collector["imap-logins"]: logger.add_heading("Recent IMAP Logins") logger.print_block( "The most recent login from each remote IP adddress is show.") for k in utils.sort_email_addresses(collector["imap-logins"], env): for ip, date in sorted(collector["imap-logins"][k].items(), key=lambda kv: kv[1]): logger.print_line(k + "\t" + str(date) + "\t" + ip) if collector["postgrey"]: logger.add_heading("Greylisted Mail") logger.print_block( "The following mail was greylisted, meaning the emails were temporarily rejected. Legitimate senders will try again within ten minutes." ) logger.print_line("recipient" + "\t" + "received" + "\t" + "sender" + "\t" + "delivered") for recipient in utils.sort_email_addresses(collector["postgrey"], env): for (client_address, sender), (first_date, delivered_date) in sorted( collector["postgrey"][recipient].items(), key=lambda kv: kv[1][0]): logger.print_line(recipient + "\t" + str(first_date) + "\t" + sender + "\t" + (("delivered " + str(delivered_date) ) if delivered_date else "no retry yet")) if collector["rejected-mail"]: logger.add_heading("Rejected Mail") logger.print_block("The following incoming mail was rejected.") for k in utils.sort_email_addresses(collector["rejected-mail"], env): for date, sender, message in collector["rejected-mail"][k]: logger.print_line(k + "\t" + str(date) + "\t" + sender + "\t" + message) if len(collector["other-services"]) > 0: logger.add_heading("Other") logger.print_block("Unrecognized services in the log: " + ", ".join(collector["other-services"]))
def get_mail_aliases(env): # Returns a sorted list of tuples of (alias, forward-to string). c = open_database(env) c.execute('SELECT source, destination FROM aliases') aliases = dict((row[0],row[1]) for row in c.fetchall()) # make dict # put in a canonical order: sort by domain, then by email address lexicographically aliases = [ (source, aliases[source]) for source in utils.sort_email_addresses(aliases.keys(), env) ] return aliases
def get_mail_aliases(env): # Returns a sorted list of tuples of (address, forward-tos, permitted-senders). c = open_database(env) c.execute('SELECT source, destination, permitted_senders FROM aliases') aliases = { row[0]: row for row in c.fetchall() } # make dict # put in a canonical order: sort by domain, then by email address lexicographically aliases = [ aliases[address] for address in utils.sort_email_addresses(aliases.keys(), env) ] return aliases
def get_mail_aliases(env): # Returns a sorted list of tuples of (alias, forward-to string). c = open_database(env) c.execute('SELECT source, destination FROM aliases') aliases = { row[0]: row[1] for row in c.fetchall() } # make dict # put in a canonical order: sort by domain, then by email address lexicographically aliases = [ (source, aliases[source]) for source in utils.sort_email_addresses(aliases.keys(), env) ] return aliases
def get_mail_aliases(env): # Returns a sorted list of tuples of (address, forward-tos, permitted-senders, auto). c = open_database(env) c.execute('SELECT source, destination, permitted_senders, 0 as auto FROM aliases UNION SELECT source, destination, permitted_senders, 1 as auto FROM auto_aliases') aliases = { row[0]: row for row in c.fetchall() } # make dict # put in a canonical order: sort by domain, then by email address lexicographically aliases = [ aliases[address] for address in utils.sort_email_addresses(aliases.keys(), env) ] return aliases
def scan_mail_log(logger, env): collector = { "other-services": set(), "imap-logins": { }, "postgrey": { }, "rejected-mail": { }, } collector["real_mail_addresses"] = set(mailconfig.get_mail_users(env)) | set(alias[0] for alias in mailconfig.get_mail_aliases(env)) for fn in ('/var/log/mail.log.1', '/var/log/mail.log'): if not os.path.exists(fn): continue with open(fn) as log: for line in log: scan_mail_log_line(line.strip(), collector) if collector["imap-logins"]: logger.add_heading("Recent IMAP Logins") logger.print_block("The most recent login from each remote IP adddress is show.") for k in utils.sort_email_addresses(collector["imap-logins"], env): for ip, date in sorted(collector["imap-logins"][k].items(), key = lambda kv : kv[1]): logger.print_line(k + "\t" + str(date) + "\t" + ip) if collector["postgrey"]: logger.add_heading("Greylisted Mail") logger.print_block("The following mail was greylisted, meaning the emails were temporarily rejected. Legitimate senders will try again within ten minutes.") logger.print_line("recipient" + "\t" + "received" + "\t" + "sender" + "\t" + "delivered") for recipient in utils.sort_email_addresses(collector["postgrey"], env): for (client_address, sender), (first_date, delivered_date) in sorted(collector["postgrey"][recipient].items(), key = lambda kv : kv[1][0]): logger.print_line(recipient + "\t" + str(first_date) + "\t" + sender + "\t" + (("delivered " + str(delivered_date)) if delivered_date else "no retry yet")) if collector["rejected-mail"]: logger.add_heading("Rejected Mail") logger.print_block("The following incoming mail was rejected.") for k in utils.sort_email_addresses(collector["rejected-mail"], env): for date, sender, message in collector["rejected-mail"][k]: logger.print_line(k + "\t" + str(date) + "\t" + sender + "\t" + message) if len(collector["other-services"]) > 0: logger.add_heading("Other") logger.print_block("Unrecognized services in the log: " + ", ".join(collector["other-services"]))
def get_mail_aliases(env, as_json=False): c = open_database(env) c.execute("SELECT source, destination FROM aliases") aliases = {row[0]: row[1] for row in c.fetchall()} # make dict # put in a canonical order: sort by domain, then by email address lexicographically aliases = [(source, aliases[source]) for source in utils.sort_email_addresses(aliases.keys(), env)] # sort # but put automatic aliases to administrator@ last aliases.sort(key=lambda x: x[1] == get_system_administrator(env)) if as_json: required_aliases = get_required_aliases(env) aliases = [ { "source": alias[0], "destination": [d.strip() for d in alias[1].split(",")], "required": alias[0] in required_aliases or alias[0] == get_system_administrator(env), } for alias in aliases ] return aliases
def get_mail_users(env, as_map=False): # When `as_map` is False, this function returns a flat, sorted # array of all user accounts. If True, it returns a dict where key # is the user and value is a dict having, dn, maildrop and # mail addresses c = open_database(env) pager = c.paged_search(env.LDAP_USERS_BASE, "(objectClass=mailUser)", attributes=['maildrop', 'mail', 'cn']) if as_map: users = {} for rec in pager: users[rec['maildrop'][0]] = { "dn": rec['dn'], "mail": rec['mail'], "maildrop": rec['maildrop'][0], "display_name": rec['cn'][0] } return users else: users = [rec['maildrop'][0] for rec in pager] return utils.sort_email_addresses(users, env)
def get_mail_aliases(env, as_json=False): c = open_database(env) c.execute('SELECT source, destination FROM aliases') aliases = { row[0]: row[1] for row in c.fetchall() } # make dict # put in a canonical order: sort by domain, then by email address lexicographically aliases = [ (source, aliases[source]) for source in utils.sort_email_addresses(aliases.keys(), env) ] # sort # but put automatic aliases to administrator@ last aliases.sort(key = lambda x : x[1] == get_system_administrator(env)) if as_json: required_aliases = get_required_aliases(env) aliases = [ { "source": alias[0], "destination": [d.strip() for d in alias[1].split(",")], "required": alias[0] in required_aliases or alias[0] == get_system_administrator(env), } for alias in aliases ] return aliases
def get_mail_users(env, as_json=False): c = open_database(env) c.execute("SELECT email, privileges FROM users") # turn into a list of tuples, but sorted by domain & email address users = {row[0]: row[1] for row in c.fetchall()} # make dict users = [(email, users[email]) for email in utils.sort_email_addresses(users.keys(), env)] if not as_json: return [email for email, privileges in users] else: aliases = get_mail_alias_map(env) return [ { "email": email, "privileges": parse_privs(privileges), "status": "active", "aliases": [ (alias, sorted(evaluate_mail_alias_map(alias, aliases, env))) for alias in aliases.get(email.lower(), []) ], } for email, privileges in users ]
def get_mail_users(env, as_json=False): c = open_database(env) c.execute('SELECT email, privileges FROM users') # turn into a list of tuples, but sorted by domain & email address users = {row[0]: row[1] for row in c.fetchall()} # make dict users = [(email, users[email]) for email in utils.sort_email_addresses(users.keys(), env)] if not as_json: return [email for email, privileges in users] else: aliases = get_mail_alias_map(env) return [{ "email": email, "privileges": parse_privs(privileges), "status": "active", "aliases": [(alias, sorted(evaluate_mail_alias_map(alias, aliases, env))) for alias in aliases.get(email.lower(), [])] } for email, privileges in users]
def get_mail_users(env): # Returns a flat, sorted list of all user accounts. c = open_database(env) c.execute('SELECT email FROM users') users = [ row[0] for row in c.fetchall() ] return utils.sort_email_addresses(users, env)
def get_mail_users(env): # Returns a flat, sorted list of all user accounts. c = open_database(env) c.execute('SELECT email FROM users') users = [ row[0] for row in c.fetchall() ] return utils.sort_email_addresses(users, env)
def scan_mail_log(logger, env): """ Scan the system's mail log files and collect interesting data This function scans the 2 most recent mail log files in /var/log/. Args: logger (ConsoleOutput): Object used for writing messages to the console env (dict): Dictionary containing MiaB settings """ collector = { "other-services": set(), "imap-logins": {}, "pop3-logins": {}, "postgrey": {}, "rejected-mail": {}, "activity-by-hour": { "imap-logins": defaultdict(int), "pop3-logins": defaultdict(int), "smtp-sends": defaultdict(int), "smtp-receives": defaultdict(int), }, "real_mail_addresses": ( set(mailconfig.get_mail_users(env)) | set(alias[0] for alias in mailconfig.get_mail_aliases(env)) ) } for fn in ('/var/log/mail.log.1', '/var/log/mail.log'): if not os.path.exists(fn): continue with open(fn, 'rb') as log: for line in log: line = line.decode("utf8", errors='replace') scan_mail_log_line(line.strip(), collector) if collector["imap-logins"]: logger.add_heading("Recent IMAP Logins") logger.print_block("The most recent login from each remote IP adddress is shown.") for k in utils.sort_email_addresses(collector["imap-logins"], env): for ip, date in sorted(collector["imap-logins"][k].items(), key=lambda kv: kv[1]): logger.print_line(k + "\t" + str(date) + "\t" + ip) if collector["pop3-logins"]: logger.add_heading("Recent POP3 Logins") logger.print_block("The most recent login from each remote IP adddress is shown.") for k in utils.sort_email_addresses(collector["pop3-logins"], env): for ip, date in sorted(collector["pop3-logins"][k].items(), key=lambda kv: kv[1]): logger.print_line(k + "\t" + str(date) + "\t" + ip) if collector["postgrey"]: logger.add_heading("Greylisted Mail") logger.print_block("The following mail was greylisted, meaning the emails were temporarily rejected. " "Legitimate senders will try again within ten minutes.") logger.print_line("recipient" + "\t" + "received" + 3 * "\t" + "sender" + 6 * "\t" + "delivered") for recipient in utils.sort_email_addresses(collector["postgrey"], env): sorted_recipients = sorted(collector["postgrey"][recipient].items(), key=lambda kv: kv[1][0]) for (client_address, sender), (first_date, delivered_date) in sorted_recipients: logger.print_line( recipient + "\t" + str(first_date) + "\t" + sender + "\t" + (("delivered " + str(delivered_date)) if delivered_date else "no retry yet") ) if collector["rejected-mail"]: logger.add_heading("Rejected Mail") logger.print_block("The following incoming mail was rejected.") for k in utils.sort_email_addresses(collector["rejected-mail"], env): for date, sender, message in collector["rejected-mail"][k]: logger.print_line(k + "\t" + str(date) + "\t" + sender + "\t" + message) logger.add_heading("Activity by Hour") logger.print_block("Dovecot logins and Postfix mail traffic per hour.") logger.print_block("Hour\tIMAP\tPOP3\tSent\tReceived") for h in range(24): logger.print_line( "%d\t%d\t\t%d\t\t%d\t\t%d" % ( h, collector["activity-by-hour"]["imap-logins"][h], collector["activity-by-hour"]["pop3-logins"][h], collector["activity-by-hour"]["smtp-sends"][h], collector["activity-by-hour"]["smtp-receives"][h], ) ) if len(collector["other-services"]) > 0: logger.add_heading("Other") logger.print_block("Unrecognized services in the log: " + ", ".join(collector["other-services"]))
def get_mail_aliases(env, as_map=False): # Retrieve all mail aliases. # # If as_map is False, the function returns a sorted array of tuples: # # (address(lowercase), forward-tos{string,csv}, permitted-senders{string,csv}) # # If as-map is True, it returns a dict whose keys are # address(lowercase) and whose values are: # # { dn: {string}, # mail: {string} # forward_tos: {array of string}, # permited_senders: {array of string}, # description: {string} # } # c = open_database(env) # get all permitted senders pager = c.paged_search(env.LDAP_PERMITTED_SENDERS_BASE, "(objectClass=mailGroup)", attributes=["mail", "member"]) # make a dict of permitted senders, key=mail(lowercase) value=members permitted_senders = { rec["mail"][0].lower(): rec["member"] for rec in pager } # get all alias groups pager = c.paged_search( env.LDAP_ALIASES_BASE, "(objectClass=mailGroup)", attributes=['mail', 'member', 'rfc822MailMember', 'description']) # make a dict of aliases # key=email(lowercase), value=(email, forward-tos, permitted-senders). aliases = {} for alias in pager: alias_email = alias['mail'][0] alias_email_lc = alias_email.lower() # chase down each member's email address, because a member is a dn forward_tos = [] for fwd_to in c.chase_members(alias['member'], 'mail', env): forward_tos.append(fwd_to[0]) for fwd_to in alias['rfc822MailMember']: forward_tos.append(fwd_to) # chase down permitted senders' email addresses allowed_senders = [] if alias_email_lc in permitted_senders: members = permitted_senders[alias_email_lc] for mail_list in c.chase_members(members, 'mail', env): for mail in mail_list: allowed_senders.append(mail) aliases[alias_email_lc] = { "dn": alias["dn"], "mail": alias_email, "forward_tos": forward_tos, "permitted_senders": allowed_senders, "description": alias["description"][0] } if not as_map: # put in a canonical order: sort by domain, then by email address lexicographically list = [] for address in utils.sort_email_addresses(aliases.keys(), env): alias = aliases[address] xft = ",".join(alias["forward_tos"]) xas = ",".join(alias["permitted_senders"]) list.append((address, xft, None if xas == "" else xas)) return list else: return aliases
def scan_mail_log(logger, env): """ Scan the system's mail log files and collect interesting data This function scans the 2 most recent mail log files in /var/log/. Args: logger (ConsoleOutput): Object used for writing messages to the console env (dict): Dictionary containing MiaB settings """ collector = { "other-services": set(), "imap-logins": {}, "pop3-logins": {}, "postgrey": {}, "rejected-mail": {}, "activity-by-hour": { "imap-logins": defaultdict(int), "pop3-logins": defaultdict(int), "smtp-sends": defaultdict(int), "smtp-receives": defaultdict(int), }, "real_mail_addresses": (set(mailconfig.get_mail_users(env)) | set(alias[0] for alias in mailconfig.get_mail_aliases(env))) } for fn in ('/var/log/mail.log.1', '/var/log/mail.log'): if not os.path.exists(fn): continue with open(fn, 'rb') as log: for line in log: line = line.decode("utf8", errors='replace') scan_mail_log_line(line.strip(), collector) if collector["imap-logins"]: logger.add_heading("Recent IMAP Logins") logger.print_block( "The most recent login from each remote IP adddress is shown.") for k in utils.sort_email_addresses(collector["imap-logins"], env): for ip, date in sorted(collector["imap-logins"][k].items(), key=lambda kv: kv[1]): logger.print_line(k + "\t" + str(date) + "\t" + ip) if collector["pop3-logins"]: logger.add_heading("Recent POP3 Logins") logger.print_block( "The most recent login from each remote IP adddress is shown.") for k in utils.sort_email_addresses(collector["pop3-logins"], env): for ip, date in sorted(collector["pop3-logins"][k].items(), key=lambda kv: kv[1]): logger.print_line(k + "\t" + str(date) + "\t" + ip) if collector["postgrey"]: logger.add_heading("Greylisted Mail") logger.print_block( "The following mail was greylisted, meaning the emails were temporarily rejected. " "Legitimate senders will try again within ten minutes.") logger.print_line("recipient" + "\t" + "received" + 3 * "\t" + "sender" + 6 * "\t" + "delivered") for recipient in utils.sort_email_addresses(collector["postgrey"], env): sorted_recipients = sorted( collector["postgrey"][recipient].items(), key=lambda kv: kv[1][0]) for (client_address, sender), (first_date, delivered_date) in sorted_recipients: logger.print_line(recipient + "\t" + str(first_date) + "\t" + sender + "\t" + (("delivered " + str(delivered_date) ) if delivered_date else "no retry yet")) if collector["rejected-mail"]: logger.add_heading("Rejected Mail") logger.print_block("The following incoming mail was rejected.") for k in utils.sort_email_addresses(collector["rejected-mail"], env): for date, sender, message in collector["rejected-mail"][k]: logger.print_line(k + "\t" + str(date) + "\t" + sender + "\t" + message) logger.add_heading("Activity by Hour") logger.print_block("Dovecot logins and Postfix mail traffic per hour.") logger.print_block("Hour\tIMAP\tPOP3\tSent\tReceived") for h in range(24): logger.print_line("%d\t%d\t\t%d\t\t%d\t\t%d" % ( h, collector["activity-by-hour"]["imap-logins"][h], collector["activity-by-hour"]["pop3-logins"][h], collector["activity-by-hour"]["smtp-sends"][h], collector["activity-by-hour"]["smtp-receives"][h], )) if len(collector["other-services"]) > 0: logger.add_heading("Other") logger.print_block("Unrecognized services in the log: " + ", ".join(collector["other-services"]))