def scan_mail_log(logger, env):
collector = {
"other-services": set(),
"imap-logins": {},
"postgrey": {},
"rejected-mail": {},
}
collector["real_mail_addresses"] = set(
mailconfig.get_mail_users(env)) | set(
alias[0] for alias in mailconfig.get_mail_aliases(env))
for fn in ('/var/log/mail.log.1', '/var/log/mail.log'):
if not os.path.exists(fn): continue
with open(fn, 'rb') as log:
for line in log:
line = line.decode("utf8", errors='replace')
scan_mail_log_line(line.strip(), collector)
if collector["imap-logins"]:
logger.add_heading("Recent IMAP Logins")
logger.print_block(
"The most recent login from each remote IP adddress is show.")
for k in utils.sort_email_addresses(collector["imap-logins"], env):
for ip, date in sorted(collector["imap-logins"][k].items(),
key=lambda kv: kv[1]):
logger.print_line(k + "\t" + str(date) + "\t" + ip)
if collector["postgrey"]:
logger.add_heading("Greylisted Mail")
logger.print_block(
"The following mail was greylisted, meaning the emails were temporarily rejected. Legitimate senders will try again within ten minutes."
)
logger.print_line("recipient" + "\t" + "received" + "\t" + "sender" +
"\t" + "delivered")
for recipient in utils.sort_email_addresses(collector["postgrey"],
env):
for (client_address,
sender), (first_date, delivered_date) in sorted(
collector["postgrey"][recipient].items(),
key=lambda kv: kv[1][0]):
logger.print_line(recipient + "\t" + str(first_date) + "\t" +
sender + "\t" +
(("delivered " + str(delivered_date)
) if delivered_date else "no retry yet"))
if collector["rejected-mail"]:
logger.add_heading("Rejected Mail")
logger.print_block("The following incoming mail was rejected.")
for k in utils.sort_email_addresses(collector["rejected-mail"], env):
for date, sender, message in collector["rejected-mail"][k]:
logger.print_line(k + "\t" + str(date) + "\t" + sender + "\t" +
message)
if len(collector["other-services"]) > 0:
logger.add_heading("Other")
logger.print_block("Unrecognized services in the log: " +
", ".join(collector["other-services"]))
def get_mail_aliases(env):
# Returns a sorted list of tuples of (alias, forward-to string).
c = open_database(env)
c.execute('SELECT source, destination FROM aliases')
aliases = dict((row[0],row[1]) for row in c.fetchall()) # make dict
# put in a canonical order: sort by domain, then by email address lexicographically
aliases = [ (source, aliases[source]) for source in utils.sort_email_addresses(aliases.keys(), env) ]
return aliases
def get_mail_aliases(env):
# Returns a sorted list of tuples of (address, forward-tos, permitted-senders).
c = open_database(env)
c.execute('SELECT source, destination, permitted_senders FROM aliases')
aliases = { row[0]: row for row in c.fetchall() } # make dict
# put in a canonical order: sort by domain, then by email address lexicographically
aliases = [ aliases[address] for address in utils.sort_email_addresses(aliases.keys(), env) ]
return aliases
def get_mail_aliases(env):
# Returns a sorted list of tuples of (alias, forward-to string).
c = open_database(env)
c.execute('SELECT source, destination FROM aliases')
aliases = { row[0]: row[1] for row in c.fetchall() } # make dict
# put in a canonical order: sort by domain, then by email address lexicographically
aliases = [ (source, aliases[source]) for source in utils.sort_email_addresses(aliases.keys(), env) ]
return aliases
def get_mail_aliases(env):
# Returns a sorted list of tuples of (address, forward-tos, permitted-senders, auto).
c = open_database(env)
c.execute('SELECT source, destination, permitted_senders, 0 as auto FROM aliases UNION SELECT source, destination, permitted_senders, 1 as auto FROM auto_aliases')
aliases = { row[0]: row for row in c.fetchall() } # make dict
# put in a canonical order: sort by domain, then by email address lexicographically
aliases = [ aliases[address] for address in utils.sort_email_addresses(aliases.keys(), env) ]
return aliases
def scan_mail_log(logger, env):
collector = {
"other-services": set(),
"imap-logins": { },
"postgrey": { },
"rejected-mail": { },
}
collector["real_mail_addresses"] = set(mailconfig.get_mail_users(env)) | set(alias[0] for alias in mailconfig.get_mail_aliases(env))
for fn in ('/var/log/mail.log.1', '/var/log/mail.log'):
if not os.path.exists(fn): continue
with open(fn) as log:
for line in log:
scan_mail_log_line(line.strip(), collector)
if collector["imap-logins"]:
logger.add_heading("Recent IMAP Logins")
logger.print_block("The most recent login from each remote IP adddress is show.")
for k in utils.sort_email_addresses(collector["imap-logins"], env):
for ip, date in sorted(collector["imap-logins"][k].items(), key = lambda kv : kv[1]):
logger.print_line(k + "\t" + str(date) + "\t" + ip)
if collector["postgrey"]:
logger.add_heading("Greylisted Mail")
logger.print_block("The following mail was greylisted, meaning the emails were temporarily rejected. Legitimate senders will try again within ten minutes.")
logger.print_line("recipient" + "\t" + "received" + "\t" + "sender" + "\t" + "delivered")
for recipient in utils.sort_email_addresses(collector["postgrey"], env):
for (client_address, sender), (first_date, delivered_date) in sorted(collector["postgrey"][recipient].items(), key = lambda kv : kv[1][0]):
logger.print_line(recipient + "\t" + str(first_date) + "\t" + sender + "\t" + (("delivered " + str(delivered_date)) if delivered_date else "no retry yet"))
if collector["rejected-mail"]:
logger.add_heading("Rejected Mail")
logger.print_block("The following incoming mail was rejected.")
for k in utils.sort_email_addresses(collector["rejected-mail"], env):
for date, sender, message in collector["rejected-mail"][k]:
logger.print_line(k + "\t" + str(date) + "\t" + sender + "\t" + message)
if len(collector["other-services"]) > 0:
logger.add_heading("Other")
logger.print_block("Unrecognized services in the log: " + ", ".join(collector["other-services"]))
def get_mail_aliases(env, as_json=False):
c = open_database(env)
c.execute("SELECT source, destination FROM aliases")
aliases = {row[0]: row[1] for row in c.fetchall()} # make dict
# put in a canonical order: sort by domain, then by email address lexicographically
aliases = [(source, aliases[source]) for source in utils.sort_email_addresses(aliases.keys(), env)] # sort
# but put automatic aliases to administrator@ last
aliases.sort(key=lambda x: x[1] == get_system_administrator(env))
if as_json:
required_aliases = get_required_aliases(env)
aliases = [
{
"source": alias[0],
"destination": [d.strip() for d in alias[1].split(",")],
"required": alias[0] in required_aliases or alias[0] == get_system_administrator(env),
}
for alias in aliases
]
return aliases
def get_mail_users(env, as_map=False):
# When `as_map` is False, this function returns a flat, sorted
# array of all user accounts. If True, it returns a dict where key
# is the user and value is a dict having, dn, maildrop and
# mail addresses
c = open_database(env)
pager = c.paged_search(env.LDAP_USERS_BASE,
"(objectClass=mailUser)",
attributes=['maildrop', 'mail', 'cn'])
if as_map:
users = {}
for rec in pager:
users[rec['maildrop'][0]] = {
"dn": rec['dn'],
"mail": rec['mail'],
"maildrop": rec['maildrop'][0],
"display_name": rec['cn'][0]
}
return users
else:
users = [rec['maildrop'][0] for rec in pager]
return utils.sort_email_addresses(users, env)
def get_mail_aliases(env, as_json=False):
c = open_database(env)
c.execute('SELECT source, destination FROM aliases')
aliases = { row[0]: row[1] for row in c.fetchall() } # make dict
# put in a canonical order: sort by domain, then by email address lexicographically
aliases = [ (source, aliases[source]) for source in utils.sort_email_addresses(aliases.keys(), env) ] # sort
# but put automatic aliases to administrator@ last
aliases.sort(key = lambda x : x[1] == get_system_administrator(env))
if as_json:
required_aliases = get_required_aliases(env)
aliases = [
{
"source": alias[0],
"destination": [d.strip() for d in alias[1].split(",")],
"required": alias[0] in required_aliases or alias[0] == get_system_administrator(env),
}
for alias in aliases
]
return aliases
def get_mail_users(env, as_json=False):
c = open_database(env)
c.execute("SELECT email, privileges FROM users")
# turn into a list of tuples, but sorted by domain & email address
users = {row[0]: row[1] for row in c.fetchall()} # make dict
users = [(email, users[email]) for email in utils.sort_email_addresses(users.keys(), env)]
if not as_json:
return [email for email, privileges in users]
else:
aliases = get_mail_alias_map(env)
return [
{
"email": email,
"privileges": parse_privs(privileges),
"status": "active",
"aliases": [
(alias, sorted(evaluate_mail_alias_map(alias, aliases, env)))
for alias in aliases.get(email.lower(), [])
],
}
for email, privileges in users
]
def get_mail_users(env, as_json=False):
c = open_database(env)
c.execute('SELECT email, privileges FROM users')
# turn into a list of tuples, but sorted by domain & email address
users = {row[0]: row[1] for row in c.fetchall()} # make dict
users = [(email, users[email])
for email in utils.sort_email_addresses(users.keys(), env)]
if not as_json:
return [email for email, privileges in users]
else:
aliases = get_mail_alias_map(env)
return [{
"email":
email,
"privileges":
parse_privs(privileges),
"status":
"active",
"aliases":
[(alias, sorted(evaluate_mail_alias_map(alias, aliases, env)))
for alias in aliases.get(email.lower(), [])]
} for email, privileges in users]
def get_mail_users(env):
# Returns a flat, sorted list of all user accounts.
c = open_database(env)
c.execute('SELECT email FROM users')
users = [ row[0] for row in c.fetchall() ]
return utils.sort_email_addresses(users, env)
def get_mail_users(env):
# Returns a flat, sorted list of all user accounts.
c = open_database(env)
c.execute('SELECT email FROM users')
users = [ row[0] for row in c.fetchall() ]
return utils.sort_email_addresses(users, env)
def scan_mail_log(logger, env):
""" Scan the system's mail log files and collect interesting data
This function scans the 2 most recent mail log files in /var/log/.
Args:
logger (ConsoleOutput): Object used for writing messages to the console
env (dict): Dictionary containing MiaB settings
"""
collector = {
"other-services": set(),
"imap-logins": {},
"pop3-logins": {},
"postgrey": {},
"rejected-mail": {},
"activity-by-hour": {
"imap-logins": defaultdict(int),
"pop3-logins": defaultdict(int),
"smtp-sends": defaultdict(int),
"smtp-receives": defaultdict(int),
},
"real_mail_addresses": (
set(mailconfig.get_mail_users(env)) | set(alias[0] for alias in mailconfig.get_mail_aliases(env))
)
}
for fn in ('/var/log/mail.log.1', '/var/log/mail.log'):
if not os.path.exists(fn):
continue
with open(fn, 'rb') as log:
for line in log:
line = line.decode("utf8", errors='replace')
scan_mail_log_line(line.strip(), collector)
if collector["imap-logins"]:
logger.add_heading("Recent IMAP Logins")
logger.print_block("The most recent login from each remote IP adddress is shown.")
for k in utils.sort_email_addresses(collector["imap-logins"], env):
for ip, date in sorted(collector["imap-logins"][k].items(), key=lambda kv: kv[1]):
logger.print_line(k + "\t" + str(date) + "\t" + ip)
if collector["pop3-logins"]:
logger.add_heading("Recent POP3 Logins")
logger.print_block("The most recent login from each remote IP adddress is shown.")
for k in utils.sort_email_addresses(collector["pop3-logins"], env):
for ip, date in sorted(collector["pop3-logins"][k].items(), key=lambda kv: kv[1]):
logger.print_line(k + "\t" + str(date) + "\t" + ip)
if collector["postgrey"]:
logger.add_heading("Greylisted Mail")
logger.print_block("The following mail was greylisted, meaning the emails were temporarily rejected. "
"Legitimate senders will try again within ten minutes.")
logger.print_line("recipient" + "\t" + "received" + 3 * "\t" + "sender" + 6 * "\t" + "delivered")
for recipient in utils.sort_email_addresses(collector["postgrey"], env):
sorted_recipients = sorted(collector["postgrey"][recipient].items(), key=lambda kv: kv[1][0])
for (client_address, sender), (first_date, delivered_date) in sorted_recipients:
logger.print_line(
recipient + "\t" + str(first_date) + "\t" + sender + "\t" +
(("delivered " + str(delivered_date)) if delivered_date else "no retry yet")
)
if collector["rejected-mail"]:
logger.add_heading("Rejected Mail")
logger.print_block("The following incoming mail was rejected.")
for k in utils.sort_email_addresses(collector["rejected-mail"], env):
for date, sender, message in collector["rejected-mail"][k]:
logger.print_line(k + "\t" + str(date) + "\t" + sender + "\t" + message)
logger.add_heading("Activity by Hour")
logger.print_block("Dovecot logins and Postfix mail traffic per hour.")
logger.print_block("Hour\tIMAP\tPOP3\tSent\tReceived")
for h in range(24):
logger.print_line(
"%d\t%d\t\t%d\t\t%d\t\t%d" % (
h,
collector["activity-by-hour"]["imap-logins"][h],
collector["activity-by-hour"]["pop3-logins"][h],
collector["activity-by-hour"]["smtp-sends"][h],
collector["activity-by-hour"]["smtp-receives"][h],
)
)
if len(collector["other-services"]) > 0:
logger.add_heading("Other")
logger.print_block("Unrecognized services in the log: " + ", ".join(collector["other-services"]))
def get_mail_aliases(env, as_map=False):
# Retrieve all mail aliases.
#
# If as_map is False, the function returns a sorted array of tuples:
#
# (address(lowercase), forward-tos{string,csv}, permitted-senders{string,csv})
#
# If as-map is True, it returns a dict whose keys are
# address(lowercase) and whose values are:
#
# { dn: {string},
# mail: {string}
# forward_tos: {array of string},
# permited_senders: {array of string},
# description: {string}
# }
#
c = open_database(env)
# get all permitted senders
pager = c.paged_search(env.LDAP_PERMITTED_SENDERS_BASE,
"(objectClass=mailGroup)",
attributes=["mail", "member"])
# make a dict of permitted senders, key=mail(lowercase) value=members
permitted_senders = {
rec["mail"][0].lower(): rec["member"]
for rec in pager
}
# get all alias groups
pager = c.paged_search(
env.LDAP_ALIASES_BASE,
"(objectClass=mailGroup)",
attributes=['mail', 'member', 'rfc822MailMember', 'description'])
# make a dict of aliases
# key=email(lowercase), value=(email, forward-tos, permitted-senders).
aliases = {}
for alias in pager:
alias_email = alias['mail'][0]
alias_email_lc = alias_email.lower()
# chase down each member's email address, because a member is a dn
forward_tos = []
for fwd_to in c.chase_members(alias['member'], 'mail', env):
forward_tos.append(fwd_to[0])
for fwd_to in alias['rfc822MailMember']:
forward_tos.append(fwd_to)
# chase down permitted senders' email addresses
allowed_senders = []
if alias_email_lc in permitted_senders:
members = permitted_senders[alias_email_lc]
for mail_list in c.chase_members(members, 'mail', env):
for mail in mail_list:
allowed_senders.append(mail)
aliases[alias_email_lc] = {
"dn": alias["dn"],
"mail": alias_email,
"forward_tos": forward_tos,
"permitted_senders": allowed_senders,
"description": alias["description"][0]
}
if not as_map:
# put in a canonical order: sort by domain, then by email address lexicographically
list = []
for address in utils.sort_email_addresses(aliases.keys(), env):
alias = aliases[address]
xft = ",".join(alias["forward_tos"])
xas = ",".join(alias["permitted_senders"])
list.append((address, xft, None if xas == "" else xas))
return list
else:
return aliases
def scan_mail_log(logger, env):
""" Scan the system's mail log files and collect interesting data
This function scans the 2 most recent mail log files in /var/log/.
Args:
logger (ConsoleOutput): Object used for writing messages to the console
env (dict): Dictionary containing MiaB settings
"""
collector = {
"other-services":
set(),
"imap-logins": {},
"pop3-logins": {},
"postgrey": {},
"rejected-mail": {},
"activity-by-hour": {
"imap-logins": defaultdict(int),
"pop3-logins": defaultdict(int),
"smtp-sends": defaultdict(int),
"smtp-receives": defaultdict(int),
},
"real_mail_addresses":
(set(mailconfig.get_mail_users(env))
| set(alias[0] for alias in mailconfig.get_mail_aliases(env)))
}
for fn in ('/var/log/mail.log.1', '/var/log/mail.log'):
if not os.path.exists(fn):
continue
with open(fn, 'rb') as log:
for line in log:
line = line.decode("utf8", errors='replace')
scan_mail_log_line(line.strip(), collector)
if collector["imap-logins"]:
logger.add_heading("Recent IMAP Logins")
logger.print_block(
"The most recent login from each remote IP adddress is shown.")
for k in utils.sort_email_addresses(collector["imap-logins"], env):
for ip, date in sorted(collector["imap-logins"][k].items(),
key=lambda kv: kv[1]):
logger.print_line(k + "\t" + str(date) + "\t" + ip)
if collector["pop3-logins"]:
logger.add_heading("Recent POP3 Logins")
logger.print_block(
"The most recent login from each remote IP adddress is shown.")
for k in utils.sort_email_addresses(collector["pop3-logins"], env):
for ip, date in sorted(collector["pop3-logins"][k].items(),
key=lambda kv: kv[1]):
logger.print_line(k + "\t" + str(date) + "\t" + ip)
if collector["postgrey"]:
logger.add_heading("Greylisted Mail")
logger.print_block(
"The following mail was greylisted, meaning the emails were temporarily rejected. "
"Legitimate senders will try again within ten minutes.")
logger.print_line("recipient" + "\t" + "received" + 3 * "\t" +
"sender" + 6 * "\t" + "delivered")
for recipient in utils.sort_email_addresses(collector["postgrey"],
env):
sorted_recipients = sorted(
collector["postgrey"][recipient].items(),
key=lambda kv: kv[1][0])
for (client_address,
sender), (first_date, delivered_date) in sorted_recipients:
logger.print_line(recipient + "\t" + str(first_date) + "\t" +
sender + "\t" +
(("delivered " + str(delivered_date)
) if delivered_date else "no retry yet"))
if collector["rejected-mail"]:
logger.add_heading("Rejected Mail")
logger.print_block("The following incoming mail was rejected.")
for k in utils.sort_email_addresses(collector["rejected-mail"], env):
for date, sender, message in collector["rejected-mail"][k]:
logger.print_line(k + "\t" + str(date) + "\t" + sender + "\t" +
message)
logger.add_heading("Activity by Hour")
logger.print_block("Dovecot logins and Postfix mail traffic per hour.")
logger.print_block("Hour\tIMAP\tPOP3\tSent\tReceived")
for h in range(24):
logger.print_line("%d\t%d\t\t%d\t\t%d\t\t%d" % (
h,
collector["activity-by-hour"]["imap-logins"][h],
collector["activity-by-hour"]["pop3-logins"][h],
collector["activity-by-hour"]["smtp-sends"][h],
collector["activity-by-hour"]["smtp-receives"][h],
))
if len(collector["other-services"]) > 0:
logger.add_heading("Other")
logger.print_block("Unrecognized services in the log: " +
", ".join(collector["other-services"]))
