def mail_users():
if request.args.get("format", "") == "json":
users = get_mail_users(env, as_json=True)
return Response(json.dumps(users),
status=200,
mimetype='application/json')
else:
return "".join(x + "\n" for x in get_mail_users(env))
def index():
# Render the control panel. This route does not require user authentication
# so it must be safe!
no_admins_exist = (len([user for user in get_mail_users(env, as_json=True) if "admin" in user['privileges']]) == 0)
return render_template('index.html',
hostname=env['PRIMARY_HOSTNAME'],
no_admins_exist=no_admins_exist,
)
def index():
# Render the control panel. This route does not require user authentication
# so it must be safe!
no_admins_exist = (len([user for user in get_mail_users(env, as_json=True) if "admin" in user['privileges']]) == 0)
return render_template('index.html',
hostname=env['PRIMARY_HOSTNAME'],
no_admins_exist=no_admins_exist,
)
def scan_mail_log(logger, env):
collector = {
"other-services": set(),
"imap-logins": {},
"postgrey": {},
"rejected-mail": {},
}
collector["real_mail_addresses"] = set(
mailconfig.get_mail_users(env)) | set(
alias[0] for alias in mailconfig.get_mail_aliases(env))
for fn in ('/var/log/mail.log.1', '/var/log/mail.log'):
if not os.path.exists(fn): continue
with open(fn, 'rb') as log:
for line in log:
line = line.decode("utf8", errors='replace')
scan_mail_log_line(line.strip(), collector)
if collector["imap-logins"]:
logger.add_heading("Recent IMAP Logins")
logger.print_block(
"The most recent login from each remote IP adddress is show.")
for k in utils.sort_email_addresses(collector["imap-logins"], env):
for ip, date in sorted(collector["imap-logins"][k].items(),
key=lambda kv: kv[1]):
logger.print_line(k + "\t" + str(date) + "\t" + ip)
if collector["postgrey"]:
logger.add_heading("Greylisted Mail")
logger.print_block(
"The following mail was greylisted, meaning the emails were temporarily rejected. Legitimate senders will try again within ten minutes."
)
logger.print_line("recipient" + "\t" + "received" + "\t" + "sender" +
"\t" + "delivered")
for recipient in utils.sort_email_addresses(collector["postgrey"],
env):
for (client_address,
sender), (first_date, delivered_date) in sorted(
collector["postgrey"][recipient].items(),
key=lambda kv: kv[1][0]):
logger.print_line(recipient + "\t" + str(first_date) + "\t" +
sender + "\t" +
(("delivered " + str(delivered_date)
) if delivered_date else "no retry yet"))
if collector["rejected-mail"]:
logger.add_heading("Rejected Mail")
logger.print_block("The following incoming mail was rejected.")
for k in utils.sort_email_addresses(collector["rejected-mail"], env):
for date, sender, message in collector["rejected-mail"][k]:
logger.print_line(k + "\t" + str(date) + "\t" + sender + "\t" +
message)
if len(collector["other-services"]) > 0:
logger.add_heading("Other")
logger.print_block("Unrecognized services in the log: " +
", ".join(collector["other-services"]))
def index():
# Render the control panel. This route does not require user authentication
# so it must be safe!
no_users_exist = (len(get_mail_users(env)) == 0)
no_admins_exist = (len(get_admins(env)) == 0)
return render_template('index.html',
hostname=env['PRIMARY_HOSTNAME'],
storage_root=env['STORAGE_ROOT'],
no_users_exist=no_users_exist,
no_admins_exist=no_admins_exist,
)
def index():
# Render the control panel. This route does not require user authentication
# so it must be safe!
no_users_exist = (len(get_mail_users(env)) == 0)
no_admins_exist = (len(get_admins(env)) == 0)
import boto.s3
backup_s3_hosts = [(r.name, r.endpoint) for r in boto.s3.regions()]
return render_template('index.html',
hostname=env['PRIMARY_HOSTNAME'],
storage_root=env['STORAGE_ROOT'],
no_users_exist=no_users_exist,
no_admins_exist=no_admins_exist,
backup_s3_hosts=backup_s3_hosts,
)
def index():
# Render the control panel. This route does not require user authentication
# so it must be safe!
no_users_exist = (len(get_mail_users(env)) == 0)
no_admins_exist = (len(get_admins(env)) == 0)
import boto.s3
backup_s3_hosts = [(r.name, r.endpoint) for r in boto.s3.regions()]
return render_template(
'index.html',
hostname=env['PRIMARY_HOSTNAME'],
storage_root=env['STORAGE_ROOT'],
no_users_exist=no_users_exist,
no_admins_exist=no_admins_exist,
backup_s3_hosts=backup_s3_hosts,
)
def scan_mail_log(logger, env):
collector = {
"other-services": set(),
"imap-logins": { },
"postgrey": { },
"rejected-mail": { },
}
collector["real_mail_addresses"] = set(mailconfig.get_mail_users(env)) | set(alias[0] for alias in mailconfig.get_mail_aliases(env))
for fn in ('/var/log/mail.log.1', '/var/log/mail.log'):
if not os.path.exists(fn): continue
with open(fn) as log:
for line in log:
scan_mail_log_line(line.strip(), collector)
if collector["imap-logins"]:
logger.add_heading("Recent IMAP Logins")
logger.print_block("The most recent login from each remote IP adddress is show.")
for k in utils.sort_email_addresses(collector["imap-logins"], env):
for ip, date in sorted(collector["imap-logins"][k].items(), key = lambda kv : kv[1]):
logger.print_line(k + "\t" + str(date) + "\t" + ip)
if collector["postgrey"]:
logger.add_heading("Greylisted Mail")
logger.print_block("The following mail was greylisted, meaning the emails were temporarily rejected. Legitimate senders will try again within ten minutes.")
logger.print_line("recipient" + "\t" + "received" + "\t" + "sender" + "\t" + "delivered")
for recipient in utils.sort_email_addresses(collector["postgrey"], env):
for (client_address, sender), (first_date, delivered_date) in sorted(collector["postgrey"][recipient].items(), key = lambda kv : kv[1][0]):
logger.print_line(recipient + "\t" + str(first_date) + "\t" + sender + "\t" + (("delivered " + str(delivered_date)) if delivered_date else "no retry yet"))
if collector["rejected-mail"]:
logger.add_heading("Rejected Mail")
logger.print_block("The following incoming mail was rejected.")
for k in utils.sort_email_addresses(collector["rejected-mail"], env):
for date, sender, message in collector["rejected-mail"][k]:
logger.print_line(k + "\t" + str(date) + "\t" + sender + "\t" + message)
if len(collector["other-services"]) > 0:
logger.add_heading("Other")
logger.print_block("Unrecognized services in the log: " + ", ".join(collector["other-services"]))
def index():
# Render the control panel. This route does not require user authentication
# so it must be safe!
no_users_exist = (len(get_mail_users(env)) == 0)
no_admins_exist = (len(get_admins(env)) == 0)
utils.fix_boto() # must call prior to importing boto
import boto.s3
backup_s3_hosts = [(r.name, r.endpoint) for r in boto.s3.regions()]
lsb = utils.shell("check_output", ["/usr/bin/lsb_release", "-d"])
return render_template(
'index.html',
hostname=env['PRIMARY_HOSTNAME'],
storage_root=env['STORAGE_ROOT'],
no_users_exist=no_users_exist,
no_admins_exist=no_admins_exist,
backup_s3_hosts=backup_s3_hosts,
csr_country_codes=csr_country_codes,
)
def mail_users():
return "".join(x+"\n" for x in get_mail_users(env))
def get_data_user_list(): return jsonify(get_mail_users(env, as_map=False))
def scan_mail_log(logger, env):
""" Scan the system's mail log files and collect interesting data
This function scans the 2 most recent mail log files in /var/log/.
Args:
logger (ConsoleOutput): Object used for writing messages to the console
env (dict): Dictionary containing MiaB settings
"""
collector = {
"other-services": set(),
"imap-logins": {},
"pop3-logins": {},
"postgrey": {},
"rejected-mail": {},
"activity-by-hour": {
"imap-logins": defaultdict(int),
"pop3-logins": defaultdict(int),
"smtp-sends": defaultdict(int),
"smtp-receives": defaultdict(int),
},
"real_mail_addresses": (
set(mailconfig.get_mail_users(env)) | set(alias[0] for alias in mailconfig.get_mail_aliases(env))
)
}
for fn in ('/var/log/mail.log.1', '/var/log/mail.log'):
if not os.path.exists(fn):
continue
with open(fn, 'rb') as log:
for line in log:
line = line.decode("utf8", errors='replace')
scan_mail_log_line(line.strip(), collector)
if collector["imap-logins"]:
logger.add_heading("Recent IMAP Logins")
logger.print_block("The most recent login from each remote IP adddress is shown.")
for k in utils.sort_email_addresses(collector["imap-logins"], env):
for ip, date in sorted(collector["imap-logins"][k].items(), key=lambda kv: kv[1]):
logger.print_line(k + "\t" + str(date) + "\t" + ip)
if collector["pop3-logins"]:
logger.add_heading("Recent POP3 Logins")
logger.print_block("The most recent login from each remote IP adddress is shown.")
for k in utils.sort_email_addresses(collector["pop3-logins"], env):
for ip, date in sorted(collector["pop3-logins"][k].items(), key=lambda kv: kv[1]):
logger.print_line(k + "\t" + str(date) + "\t" + ip)
if collector["postgrey"]:
logger.add_heading("Greylisted Mail")
logger.print_block("The following mail was greylisted, meaning the emails were temporarily rejected. "
"Legitimate senders will try again within ten minutes.")
logger.print_line("recipient" + "\t" + "received" + 3 * "\t" + "sender" + 6 * "\t" + "delivered")
for recipient in utils.sort_email_addresses(collector["postgrey"], env):
sorted_recipients = sorted(collector["postgrey"][recipient].items(), key=lambda kv: kv[1][0])
for (client_address, sender), (first_date, delivered_date) in sorted_recipients:
logger.print_line(
recipient + "\t" + str(first_date) + "\t" + sender + "\t" +
(("delivered " + str(delivered_date)) if delivered_date else "no retry yet")
)
if collector["rejected-mail"]:
logger.add_heading("Rejected Mail")
logger.print_block("The following incoming mail was rejected.")
for k in utils.sort_email_addresses(collector["rejected-mail"], env):
for date, sender, message in collector["rejected-mail"][k]:
logger.print_line(k + "\t" + str(date) + "\t" + sender + "\t" + message)
logger.add_heading("Activity by Hour")
logger.print_block("Dovecot logins and Postfix mail traffic per hour.")
logger.print_block("Hour\tIMAP\tPOP3\tSent\tReceived")
for h in range(24):
logger.print_line(
"%d\t%d\t\t%d\t\t%d\t\t%d" % (
h,
collector["activity-by-hour"]["imap-logins"][h],
collector["activity-by-hour"]["pop3-logins"][h],
collector["activity-by-hour"]["smtp-sends"][h],
collector["activity-by-hour"]["smtp-receives"][h],
)
)
if len(collector["other-services"]) > 0:
logger.add_heading("Other")
logger.print_block("Unrecognized services in the log: " + ", ".join(collector["other-services"]))
def mail_users():
if request.args.get("format", "") == "json":
users = get_mail_users(env, as_json=True)
return Response(json.dumps(users), status=200, mimetype='application/json')
else:
return "".join(x+"\n" for x in get_mail_users(env))
def scan_mail_log(logger, env):
""" Scan the system's mail log files and collect interesting data
This function scans the 2 most recent mail log files in /var/log/.
Args:
logger (ConsoleOutput): Object used for writing messages to the console
env (dict): Dictionary containing MiaB settings
"""
collector = {
"other-services":
set(),
"imap-logins": {},
"pop3-logins": {},
"postgrey": {},
"rejected-mail": {},
"activity-by-hour": {
"imap-logins": defaultdict(int),
"pop3-logins": defaultdict(int),
"smtp-sends": defaultdict(int),
"smtp-receives": defaultdict(int),
},
"real_mail_addresses":
(set(mailconfig.get_mail_users(env))
| set(alias[0] for alias in mailconfig.get_mail_aliases(env)))
}
for fn in ('/var/log/mail.log.1', '/var/log/mail.log'):
if not os.path.exists(fn):
continue
with open(fn, 'rb') as log:
for line in log:
line = line.decode("utf8", errors='replace')
scan_mail_log_line(line.strip(), collector)
if collector["imap-logins"]:
logger.add_heading("Recent IMAP Logins")
logger.print_block(
"The most recent login from each remote IP adddress is shown.")
for k in utils.sort_email_addresses(collector["imap-logins"], env):
for ip, date in sorted(collector["imap-logins"][k].items(),
key=lambda kv: kv[1]):
logger.print_line(k + "\t" + str(date) + "\t" + ip)
if collector["pop3-logins"]:
logger.add_heading("Recent POP3 Logins")
logger.print_block(
"The most recent login from each remote IP adddress is shown.")
for k in utils.sort_email_addresses(collector["pop3-logins"], env):
for ip, date in sorted(collector["pop3-logins"][k].items(),
key=lambda kv: kv[1]):
logger.print_line(k + "\t" + str(date) + "\t" + ip)
if collector["postgrey"]:
logger.add_heading("Greylisted Mail")
logger.print_block(
"The following mail was greylisted, meaning the emails were temporarily rejected. "
"Legitimate senders will try again within ten minutes.")
logger.print_line("recipient" + "\t" + "received" + 3 * "\t" +
"sender" + 6 * "\t" + "delivered")
for recipient in utils.sort_email_addresses(collector["postgrey"],
env):
sorted_recipients = sorted(
collector["postgrey"][recipient].items(),
key=lambda kv: kv[1][0])
for (client_address,
sender), (first_date, delivered_date) in sorted_recipients:
logger.print_line(recipient + "\t" + str(first_date) + "\t" +
sender + "\t" +
(("delivered " + str(delivered_date)
) if delivered_date else "no retry yet"))
if collector["rejected-mail"]:
logger.add_heading("Rejected Mail")
logger.print_block("The following incoming mail was rejected.")
for k in utils.sort_email_addresses(collector["rejected-mail"], env):
for date, sender, message in collector["rejected-mail"][k]:
logger.print_line(k + "\t" + str(date) + "\t" + sender + "\t" +
message)
logger.add_heading("Activity by Hour")
logger.print_block("Dovecot logins and Postfix mail traffic per hour.")
logger.print_block("Hour\tIMAP\tPOP3\tSent\tReceived")
for h in range(24):
logger.print_line("%d\t%d\t\t%d\t\t%d\t\t%d" % (
h,
collector["activity-by-hour"]["imap-logins"][h],
collector["activity-by-hour"]["pop3-logins"][h],
collector["activity-by-hour"]["smtp-sends"][h],
collector["activity-by-hour"]["smtp-receives"][h],
))
if len(collector["other-services"]) > 0:
logger.add_heading("Other")
logger.print_block("Unrecognized services in the log: " +
", ".join(collector["other-services"]))
def scan_mail_log(env):
""" Scan the system's mail log files and collect interesting data
This function scans the 2 most recent mail log files in /var/log/.
Args:
env (dict): Dictionary containing MiaB settings
"""
collector = {
"scan_count": 0, # Number of lines scanned
"parse_count":
0, # Number of lines parsed (i.e. that had their contents examined)
"scan_time": time.time(), # The time in seconds the scan took
"sent_mail": OrderedDict(), # Data about email sent by users
"received_mail": OrderedDict(), # Data about email received by users
"logins": OrderedDict(), # Data about login activity
"postgrey": {}, # Data about greylisting of email addresses
"rejected": OrderedDict(), # Emails that were blocked
"known_addresses": None, # Addresses handled by the Miab installation
"other-services": set(),
}
try:
import mailconfig
collector["known_addresses"] = (
set(mailconfig.get_mail_users(env))
| set(alias[0] for alias in mailconfig.get_mail_aliases(env)))
except ImportError:
pass
print("Scanning logs from {:%Y-%m-%d %H:%M:%S} to {:%Y-%m-%d %H:%M:%S}".
format(END_DATE, START_DATE))
# Scan the lines in the log files until the date goes out of range
scan_files(collector)
if not collector["scan_count"]:
print("No log lines scanned...")
return
collector["scan_time"] = time.time() - collector["scan_time"]
print(
"{scan_count} Log lines scanned, {parse_count} lines parsed in {scan_time:.2f} "
"seconds\n".format(**collector))
# Print Sent Mail report
if collector["sent_mail"]:
msg = "Sent email"
print_header(msg)
data = OrderedDict(
sorted(collector["sent_mail"].items(), key=email_sort))
print_user_table(
data.keys(),
data=[
("sent", [u["sent_count"] for u in data.values()]),
("hosts", [len(u["hosts"]) for u in data.values()]),
],
sub_data=[
("sending hosts", [u["hosts"] for u in data.values()]),
],
activity=[
("sent", [u["activity-by-hour"] for u in data.values()]),
],
earliest=[u["earliest"] for u in data.values()],
latest=[u["latest"] for u in data.values()],
)
accum = defaultdict(int)
data = collector["sent_mail"].values()
for h in range(24):
accum[h] = sum(d["activity-by-hour"][h] for d in data)
print_time_table(["sent"], [accum])
# Print Received Mail report
if collector["received_mail"]:
msg = "Received email"
print_header(msg)
data = OrderedDict(
sorted(collector["received_mail"].items(), key=email_sort))
print_user_table(
data.keys(),
data=[
("received", [u["received_count"] for u in data.values()]),
],
activity=[
("sent", [u["activity-by-hour"] for u in data.values()]),
],
earliest=[u["earliest"] for u in data.values()],
latest=[u["latest"] for u in data.values()],
)
accum = defaultdict(int)
for h in range(24):
accum[h] = sum(d["activity-by-hour"][h] for d in data.values())
print_time_table(["received"], [accum])
# Print login report
if collector["logins"]:
msg = "User logins per hour"
print_header(msg)
data = OrderedDict(sorted(collector["logins"].items(), key=email_sort))
# Get a list of all of the protocols seen in the logs in reverse count order.
all_protocols = defaultdict(int)
for u in data.values():
for protocol_name, count in u["totals_by_protocol"].items():
all_protocols[protocol_name] += count
all_protocols = [
k for k, v in sorted(all_protocols.items(), key=lambda kv: -kv[1])
]
print_user_table(
data.keys(),
data=[
(
protocol_name,
[
round(
u["totals_by_protocol"][protocol_name] /
(u["latest"] - u["earliest"]).total_seconds() *
60 * 60, 1) if
(u["latest"] - u["earliest"]).total_seconds() > 0 else
0 # prevent division by zero
for u in data.values()
]) for protocol_name in all_protocols
],
sub_data=[("Protocol and Source", [[
"{} {}: {} times".format(protocol_name, host, count)
for (protocol_name, host), count in sorted(
u["totals_by_protocol_and_host"].items(),
key=lambda kv: -kv[1])
] for u in data.values()])],
activity=[
(protocol_name,
[u["activity-by-hour"][protocol_name] for u in data.values()])
for protocol_name in all_protocols
],
earliest=[u["earliest"] for u in data.values()],
latest=[u["latest"] for u in data.values()],
numstr=lambda n: str(round(n, 1)),
)
accum = {
protocol_name: defaultdict(int)
for protocol_name in all_protocols
}
for h in range(24):
for protocol_name in all_protocols:
accum[protocol_name][h] = sum(
d["activity-by-hour"][protocol_name][h]
for d in data.values())
print_time_table(
all_protocols,
[accum[protocol_name] for protocol_name in all_protocols])
if collector["postgrey"]:
msg = "Greylisted Email {:%Y-%m-%d %H:%M:%S} and {:%Y-%m-%d %H:%M:%S}"
print_header(msg.format(END_DATE, START_DATE))
print(textwrap.fill(
"The following mail was greylisted, meaning the emails were temporarily rejected. "
"Legitimate senders must try again after three minutes.",
width=80,
initial_indent=" ",
subsequent_indent=" "),
end='\n\n')
data = OrderedDict(
sorted(collector["postgrey"].items(), key=email_sort))
users = []
received = []
senders = []
sender_clients = []
delivered_dates = []
for recipient in data:
sorted_recipients = sorted(data[recipient].items(),
key=lambda kv: kv[1][0] or kv[1][1])
for (client_address,
sender), (first_date, delivered_date) in sorted_recipients:
if first_date:
users.append(recipient)
received.append(first_date)
senders.append(sender)
delivered_dates.append(delivered_date)
sender_clients.append(client_address)
print_user_table(
users,
data=[("received", received), ("sender", senders),
("delivered",
[str(d) or "no retry yet" for d in delivered_dates]),
("sending host", sender_clients)],
delimit=True,
)
if collector["rejected"]:
msg = "Blocked Email {:%Y-%m-%d %H:%M:%S} and {:%Y-%m-%d %H:%M:%S}"
print_header(msg.format(END_DATE, START_DATE))
data = OrderedDict(
sorted(collector["rejected"].items(), key=email_sort))
rejects = []
if VERBOSE:
for user_data in data.values():
user_rejects = []
for date, sender, message in user_data["blocked"]:
if len(sender) > 64:
sender = sender[:32] + "…" + sender[-32:]
user_rejects.append("%s - %s " % (date, sender))
user_rejects.append(" %s" % message)
rejects.append(user_rejects)
print_user_table(
data.keys(),
data=[
("blocked", [len(u["blocked"]) for u in data.values()]),
],
sub_data=[
("blocked emails", rejects),
],
earliest=[u["earliest"] for u in data.values()],
latest=[u["latest"] for u in data.values()],
)
if collector["other-services"] and VERBOSE and False:
print_header("Other services")
print("The following unkown services were found in the log file.")
print(" ", *sorted(list(collector["other-services"])), sep='\n│ ')
def mail_users():
if request.args.get("format", "") == "json":
return json_response(get_mail_users(env, as_json=True) + get_archived_mail_users(env))
else:
return "".join(x+"\n" for x in get_mail_users(env))
def __init__(self, start_date=None, end_date=None, filters=None, no_filter=False,
sent=True, received=True, imap=False, pop3=False, grey=False, rejected=False):
super().__init__()
# Try and get all the email addresses known to this box
known_addresses = []
if not no_filter:
try:
env_vars = utils.load_environment()
import mailconfig
known_addresses = sorted(
set(mailconfig.get_mail_users(env_vars)) |
set(alias[0] for alias in mailconfig.get_mail_aliases(env_vars)),
key=email_sort
)
except (FileNotFoundError, ImportError):
pass
start_date = start_date or datetime.now()
end_date = end_date or start_date - timedelta(weeks=52)
self.update({
'end_of_file': False, # Indicates whether the end of the log files was reached
'start_date': start_date,
'end_date': end_date,
'line_count': 0, # Number of lines scanned
'parse_count': 0, # Number of lines parsed (i.e. that had their contents examined)
'scan_time': time.time(), # The time in seconds the scan took
'unknown services': set(), # Services encountered that were not recognized
'known_addresses': known_addresses, # Addresses handled by MiaB
'services': {}, # What services to scan for
'data': OrderedDict(), # Scan data, per service
})
# Caching is only useful with longer filter lists, but doesn't seem to hurt performance in shorter ones
user_match = lru_cache(maxsize=None)(partial(filter_match, [f.lower() for f in filters] if filters else None))
if sent:
data = {}
self['data']['sent mail'] = {
'scan': partial(scan_postfix_submission, data, user_match),
'data': data,
}
self['services']['postfix/submission/smtpd'] = self['data']['sent mail']
if received:
data = {}
self['data']['received mail'] = {
'scan': partial(scan_postfix_lmtp, data, user_match),
'data': data,
}
self['services']['postfix/lmtp'] = self['data']['received mail']
if imap:
data = {}
self['data']['imap login'] = {
'scan': partial(scan_login, data, user_match),
'data': data,
}
self['services']['imap-login'] = self['data']['imap login']
if pop3:
data = {}
self['data']['pop3 login'] = {
'scan': partial(scan_login, data, user_match),
'data': data,
}
self['services']['pop3-login'] = self['data']['pop3 login']
if grey:
data = {}
self['data']['grey-listed mail'] = {
'scan': partial(scan_greylist, data, user_match),
'data': data,
}
self['services']['postgrey'] = self['data']['grey-listed mail']
if rejected:
data = {}
self['data']['blocked mail'] = {
'scan': partial(scan_rejects, data, self['known_addresses'], user_match),
'data': data,
}
self['services']['postfix/smtpd'] = self['data']['blocked mail']
def mail_users():
return "".join(x + "\n" for x in get_mail_users(env))
def mail_users():
if request.args.get("format", "") == "json":
return json_response(get_mail_users(env, as_json=True) + get_archived_mail_users(env))
else:
return "".join(x+"\n" for x in get_mail_users(env))
def mail_users():
if request.args.get("format", "") == "json":
return json_response(
get_mail_users_ex(env, with_archived=True, with_slow_info=True))
else:
return "".join(x + "\n" for x in get_mail_users(env))
def scan_mail_log(env):
""" Scan the system's mail log files and collect interesting data
This function scans the 2 most recent mail log files in /var/log/.
Args:
env (dict): Dictionary containing MiaB settings
"""
collector = {
"scan_count": 0, # Number of lines scanned
"parse_count": 0, # Number of lines parsed (i.e. that had their contents examined)
"scan_time": time.time(), # The time in seconds the scan took
"sent_mail": OrderedDict(), # Data about email sent by users
"received_mail": OrderedDict(), # Data about email received by users
"dovecot": OrderedDict(), # Data about Dovecot activity
"postgrey": {}, # Data about greylisting of email addresses
"rejected": OrderedDict(), # Emails that were blocked
"known_addresses": None, # Addresses handled by the Miab installation
"other-services": set(),
}
try:
import mailconfig
collector["known_addresses"] = (set(mailconfig.get_mail_users(env)) |
set(alias[0] for alias in mailconfig.get_mail_aliases(env)))
except ImportError:
pass
print("Scanning from {:%Y-%m-%d %H:%M:%S} back to {:%Y-%m-%d %H:%M:%S}".format(
START_DATE, END_DATE)
)
# Scan the lines in the log files until the date goes out of range
scan_files(collector)
if not collector["scan_count"]:
print("No log lines scanned...")
return
collector["scan_time"] = time.time() - collector["scan_time"]
print("{scan_count} Log lines scanned, {parse_count} lines parsed in {scan_time:.2f} "
"seconds\n".format(**collector))
# Print Sent Mail report
if collector["sent_mail"]:
msg = "Sent email between {:%Y-%m-%d %H:%M:%S} and {:%Y-%m-%d %H:%M:%S}"
print_header(msg.format(END_DATE, START_DATE))
data = OrderedDict(sorted(collector["sent_mail"].items(), key=email_sort))
print_user_table(
data.keys(),
data=[
("sent", [u["sent_count"] for u in data.values()]),
("hosts", [len(u["hosts"]) for u in data.values()]),
],
sub_data=[
("sending hosts", [u["hosts"] for u in data.values()]),
],
activity=[
("sent", [u["activity-by-hour"] for u in data.values()]),
],
earliest=[u["earliest"] for u in data.values()],
latest=[u["latest"] for u in data.values()],
)
accum = defaultdict(int)
data = collector["sent_mail"].values()
for h in range(24):
accum[h] = sum(d["activity-by-hour"][h] for d in data)
print_time_table(
["sent"],
[accum]
)
# Print Received Mail report
if collector["received_mail"]:
msg = "Received email between {:%Y-%m-%d %H:%M:%S} and {:%Y-%m-%d %H:%M:%S}"
print_header(msg.format(END_DATE, START_DATE))
data = OrderedDict(sorted(collector["received_mail"].items(), key=email_sort))
print_user_table(
data.keys(),
data=[
("received", [u["received_count"] for u in data.values()]),
],
activity=[
("sent", [u["activity-by-hour"] for u in data.values()]),
],
earliest=[u["earliest"] for u in data.values()],
latest=[u["latest"] for u in data.values()],
)
accum = defaultdict(int)
for h in range(24):
accum[h] = sum(d["activity-by-hour"][h] for d in data.values())
print_time_table(
["received"],
[accum]
)
# Print Dovecot report
if collector["dovecot"]:
msg = "Email client logins between {:%Y-%m-%d %H:%M:%S} and {:%Y-%m-%d %H:%M:%S}"
print_header(msg.format(END_DATE, START_DATE))
data = OrderedDict(sorted(collector["dovecot"].items(), key=email_sort))
print_user_table(
data.keys(),
data=[
("imap", [u["imap"] for u in data.values()]),
("pop3", [u["pop3"] for u in data.values()]),
],
sub_data=[
("IMAP IP addresses", [[k + " (%d)" % v for k, v in u["imap-logins"].items()]
for u in data.values()]),
("POP3 IP addresses", [[k + " (%d)" % v for k, v in u["pop3-logins"].items()]
for u in data.values()]),
],
activity=[
("imap", [u["activity-by-hour"]["imap"] for u in data.values()]),
("pop3", [u["activity-by-hour"]["pop3"] for u in data.values()]),
],
earliest=[u["earliest"] for u in data.values()],
latest=[u["latest"] for u in data.values()],
)
accum = {"imap": defaultdict(int), "pop3": defaultdict(int), "both": defaultdict(int)}
for h in range(24):
accum["imap"][h] = sum(d["activity-by-hour"]["imap"][h] for d in data.values())
accum["pop3"][h] = sum(d["activity-by-hour"]["pop3"][h] for d in data.values())
accum["both"][h] = accum["imap"][h] + accum["pop3"][h]
print_time_table(
["imap", "pop3", " +"],
[accum["imap"], accum["pop3"], accum["both"]]
)
if collector["postgrey"]:
msg = "Greylisted Email {:%Y-%m-%d %H:%M:%S} and {:%Y-%m-%d %H:%M:%S}"
print_header(msg.format(END_DATE, START_DATE))
print(textwrap.fill(
"The following mail was greylisted, meaning the emails were temporarily rejected. "
"Legitimate senders will try again within ten minutes.",
width=80, initial_indent=" ", subsequent_indent=" "
), end='\n\n')
data = OrderedDict(sorted(collector["postgrey"].items(), key=email_sort))
users = []
received = []
senders = []
sender_clients = []
delivered_dates = []
for recipient in data:
sorted_recipients = sorted(data[recipient].items(), key=lambda kv: kv[1][0] or kv[1][1])
for (client_address, sender), (first_date, delivered_date) in sorted_recipients:
if first_date:
users.append(recipient)
received.append(first_date)
senders.append(sender)
delivered_dates.append(delivered_date)
sender_clients.append(client_address)
print_user_table(
users,
data=[
("received", received),
("sender", senders),
("delivered", [str(d) or "no retry yet" for d in delivered_dates]),
("sending host", sender_clients)
],
delimit=True,
)
if collector["rejected"]:
msg = "Blocked Email {:%Y-%m-%d %H:%M:%S} and {:%Y-%m-%d %H:%M:%S}"
print_header(msg.format(END_DATE, START_DATE))
data = OrderedDict(sorted(collector["rejected"].items(), key=email_sort))
rejects = []
if VERBOSE:
for user_data in data.values():
user_rejects = []
for date, sender, message in user_data["blocked"]:
if len(sender) > 64:
sender = sender[:32] + "…" + sender[-32:]
user_rejects.append("%s - %s " % (date, sender))
user_rejects.append(" %s" % message)
rejects.append(user_rejects)
print_user_table(
data.keys(),
data=[
("blocked", [len(u["blocked"]) for u in data.values()]),
],
sub_data=[
("blocked emails", rejects),
],
earliest=[u["earliest"] for u in data.values()],
latest=[u["latest"] for u in data.values()],
)
if collector["other-services"] and VERBOSE and False:
print_header("Other services")
print("The following unkown services were found in the log file.")
print(" ", *sorted(list(collector["other-services"])), sep='\n│ ')
def mail_users():
if request.args.get("format", "") == "json":
return json_response(get_mail_users_ex(env, with_archived=True, with_slow_info=True))
else:
return "".join(x+"\n" for x in get_mail_users(env))
def scan_mail_log(env):
""" Scan the system's mail log files and collect interesting data
This function scans the 2 most recent mail log files in /var/log/.
Args:
env (dict): Dictionary containing MiaB settings
"""
collector = {
"scan_count": 0, # Number of lines scanned
"parse_count": 0, # Number of lines parsed (i.e. that had their contents examined)
"scan_time": time.time(), # The time in seconds the scan took
"sent_mail": OrderedDict(), # Data about email sent by users
"received_mail": OrderedDict(), # Data about email received by users
"logins": OrderedDict(), # Data about login activity
"postgrey": {}, # Data about greylisting of email addresses
"rejected": OrderedDict(), # Emails that were blocked
"known_addresses": None, # Addresses handled by the Miab installation
"other-services": set(),
}
try:
import mailconfig
collector["known_addresses"] = (set(mailconfig.get_mail_users(env)) |
set(alias[0] for alias in mailconfig.get_mail_aliases(env)))
except ImportError:
pass
print("Scanning logs from {:%Y-%m-%d %H:%M:%S} to {:%Y-%m-%d %H:%M:%S}".format(
END_DATE, START_DATE)
)
# Scan the lines in the log files until the date goes out of range
scan_files(collector)
if not collector["scan_count"]:
print("No log lines scanned...")
return
collector["scan_time"] = time.time() - collector["scan_time"]
print("{scan_count} Log lines scanned, {parse_count} lines parsed in {scan_time:.2f} "
"seconds\n".format(**collector))
# Print Sent Mail report
if collector["sent_mail"]:
msg = "Sent email"
print_header(msg)
data = OrderedDict(sorted(collector["sent_mail"].items(), key=email_sort))
print_user_table(
data.keys(),
data=[
("sent", [u["sent_count"] for u in data.values()]),
("hosts", [len(u["hosts"]) for u in data.values()]),
],
sub_data=[
("sending hosts", [u["hosts"] for u in data.values()]),
],
activity=[
("sent", [u["activity-by-hour"] for u in data.values()]),
],
earliest=[u["earliest"] for u in data.values()],
latest=[u["latest"] for u in data.values()],
)
accum = defaultdict(int)
data = collector["sent_mail"].values()
for h in range(24):
accum[h] = sum(d["activity-by-hour"][h] for d in data)
print_time_table(
["sent"],
[accum]
)
# Print Received Mail report
if collector["received_mail"]:
msg = "Received email"
print_header(msg)
data = OrderedDict(sorted(collector["received_mail"].items(), key=email_sort))
print_user_table(
data.keys(),
data=[
("received", [u["received_count"] for u in data.values()]),
],
activity=[
("sent", [u["activity-by-hour"] for u in data.values()]),
],
earliest=[u["earliest"] for u in data.values()],
latest=[u["latest"] for u in data.values()],
)
accum = defaultdict(int)
for h in range(24):
accum[h] = sum(d["activity-by-hour"][h] for d in data.values())
print_time_table(
["received"],
[accum]
)
# Print login report
if collector["logins"]:
msg = "User logins per hour"
print_header(msg)
data = OrderedDict(sorted(collector["logins"].items(), key=email_sort))
# Get a list of all of the protocols seen in the logs in reverse count order.
all_protocols = defaultdict(int)
for u in data.values():
for protocol_name, count in u["totals_by_protocol"].items():
all_protocols[protocol_name] += count
all_protocols = [k for k, v in sorted(all_protocols.items(), key=lambda kv : -kv[1])]
print_user_table(
data.keys(),
data=[
(protocol_name, [
round(u["totals_by_protocol"][protocol_name] / (u["latest"]-u["earliest"]).total_seconds() * 60*60, 1)
if (u["latest"]-u["earliest"]).total_seconds() > 0
else 0 # prevent division by zero
for u in data.values()])
for protocol_name in all_protocols
],
sub_data=[
("Protocol and Source", [[
"{} {}: {} times".format(protocol_name, host, count)
for (protocol_name, host), count
in sorted(u["totals_by_protocol_and_host"].items(), key=lambda kv:-kv[1])
] for u in data.values()])
],
activity=[
(protocol_name, [u["activity-by-hour"][protocol_name] for u in data.values()])
for protocol_name in all_protocols
],
earliest=[u["earliest"] for u in data.values()],
latest=[u["latest"] for u in data.values()],
numstr=lambda n : str(round(n, 1)),
)
accum = { protocol_name: defaultdict(int) for protocol_name in all_protocols }
for h in range(24):
for protocol_name in all_protocols:
accum[protocol_name][h] = sum(d["activity-by-hour"][protocol_name][h] for d in data.values())
print_time_table(
all_protocols,
[accum[protocol_name] for protocol_name in all_protocols]
)
if collector["postgrey"]:
msg = "Greylisted Email {:%Y-%m-%d %H:%M:%S} and {:%Y-%m-%d %H:%M:%S}"
print_header(msg.format(END_DATE, START_DATE))
print(textwrap.fill(
"The following mail was greylisted, meaning the emails were temporarily rejected. "
"Legitimate senders must try again after three minutes.",
width=80, initial_indent=" ", subsequent_indent=" "
), end='\n\n')
data = OrderedDict(sorted(collector["postgrey"].items(), key=email_sort))
users = []
received = []
senders = []
sender_clients = []
delivered_dates = []
for recipient in data:
sorted_recipients = sorted(data[recipient].items(), key=lambda kv: kv[1][0] or kv[1][1])
for (client_address, sender), (first_date, delivered_date) in sorted_recipients:
if first_date:
users.append(recipient)
received.append(first_date)
senders.append(sender)
delivered_dates.append(delivered_date)
sender_clients.append(client_address)
print_user_table(
users,
data=[
("received", received),
("sender", senders),
("delivered", [str(d) or "no retry yet" for d in delivered_dates]),
("sending host", sender_clients)
],
delimit=True,
)
if collector["rejected"]:
msg = "Blocked Email {:%Y-%m-%d %H:%M:%S} and {:%Y-%m-%d %H:%M:%S}"
print_header(msg.format(END_DATE, START_DATE))
data = OrderedDict(sorted(collector["rejected"].items(), key=email_sort))
rejects = []
if VERBOSE:
for user_data in data.values():
user_rejects = []
for date, sender, message in user_data["blocked"]:
if len(sender) > 64:
sender = sender[:32] + "…" + sender[-32:]
user_rejects.append("%s - %s " % (date, sender))
user_rejects.append(" %s" % message)
rejects.append(user_rejects)
print_user_table(
data.keys(),
data=[
("blocked", [len(u["blocked"]) for u in data.values()]),
],
sub_data=[
("blocked emails", rejects),
],
earliest=[u["earliest"] for u in data.values()],
latest=[u["latest"] for u in data.values()],
)
if collector["other-services"] and VERBOSE and False:
print_header("Other services")
print("The following unkown services were found in the log file.")
print(" ", *sorted(list(collector["other-services"])), sep='\n│ ')
