def get_outbound_stats(): #smtp
logfile = mailman_logs + 'smtp'
stats={}
stats['outbound_mails.total'] = 0
time_now = int(time.time())
five_mins_ago = time_now-300
for line in reverse_tail.run(logfile):
if not line:
continue
s = re.match(r"^(?P<ts>\w+ \d+ \d+:\d+:\d+ \d+).* smtp to (?P<list_name>.*) for (?P<rcpt_count>\d+) recips", line)
#print line
ts = s.group('ts')
list_name = s.group('list_name')
rcpt_count = s.group('rcpt_count')
ts = int(time.mktime(time.strptime(ts,'%b %d %H:%M:%S %Y')))
if ts < five_mins_ago:
break
else:
stats['outbound_mails.total'] += int(rcpt_count)
list_name = re.sub('\.', '-', list_name)
# collect stats on outbound mails per list
list_ns = 'outbound_mails.' + list_name + '.total'
if not list_ns in stats:
stats[list_ns] = 0
stats[list_ns] += int(rcpt_count)
push_stats(stats)
def get_outbound_stats(): #smtp
logfile = mailman_logs + 'smtp'
stats = {}
stats['outbound_mails.total'] = 0
time_now = int(time.time())
five_mins_ago = time_now - 300
for line in reverse_tail.run(logfile):
if not line:
continue
s = re.match(
r"^(?P<ts>\w+ \d+ \d+:\d+:\d+ \d+).* smtp to (?P<list_name>.*) for (?P<rcpt_count>\d+) recips",
line)
#print line
ts = s.group('ts')
list_name = s.group('list_name')
rcpt_count = s.group('rcpt_count')
ts = int(time.mktime(time.strptime(ts, '%b %d %H:%M:%S %Y')))
if ts < five_mins_ago:
break
else:
stats['outbound_mails.total'] += int(rcpt_count)
list_name = re.sub('\.', '-', list_name)
# collect stats on outbound mails per list
list_ns = 'outbound_mails.' + list_name + '.total'
if not list_ns in stats:
stats[list_ns] = 0
stats[list_ns] += int(rcpt_count)
push_stats(stats)
def get_spam_stats():
logfile = postfix_log
stats = {}
# stats['total'] = 0
# stats['inbound'] = 0
# stats['outbound'] = 0
# stats['queue_size'] = 0
stats['spamhaus_blocks'] = 0
stats['spamasassin_rejects'] = 0
stats['greylist_rejects'] = 0
stats['greylist_allow'] = 0
time_now = int(time.time())
five_mins_ago = time_now - 300
year = time.strftime('%Y', time.localtime())
for line in reverse_tail.run(logfile):
if not line:
continue
m = re.match(r"(?P<ts>\w+\s+\d+ \d+:\d+:\d+).*$", line)
if m:
ts = m.group('ts')
ts = ts + ' ' + year
ts = int(time.mktime(time.strptime(ts, '%b %d %H:%M:%S %Y')))
if ts < five_mins_ago:
break
m = re.match(
r"^\w+\s+\d+ \d+:\d+:\d+.*: (?:554.*(?P<spamhaus>blocked)|(?P<spamd>spamd): result: Y)",
line)
if m:
if m.group('spamhaus'):
stats['spamhaus_blocks'] += 1
if m.group('spamd'):
stats['spamasassin_rejects'] += 1
m = re.match(
r"\w+\s+\d+ \d+:\d+:\d+.* postgrey\[\d+\]: action=(:?(?P<greylist>greylist)|(?P<pass>pass)),.*",
line)
if m:
if m.group('greylist'):
stats['greylist_rejects'] += 1
if m.group('pass'):
stats['greylist_allow'] += 1
push_stats(stats)
def get_post_stats():
logfile = mailman_logs + 'post'
stats = {}
stats['posts.total'] = 0
time_now = int(time.time())
five_mins_ago = time_now - 300
for line in reverse_tail.run(logfile):
if not line:
continue
s = re.match(
r"^(?P<ts>\w+ \d+ \d+:\d+:\d+ \d+) .*post to (?P<list_name>.*) from (?P<poster>.*@*.), size=(?P<msg_size>\d+), message-id.*, (?P<result>.*)$",
line)
if s:
ts = s.group('ts')
list_name = s.group('list_name')
poster = s.group('poster')
msg_size = s.group('msg_size')
result = s.group('result')
ts = int(time.mktime(time.strptime(ts, '%b %d %H:%M:%S %Y')))
if ts < five_mins_ago:
break
else:
#print line
stats['posts.total'] += 1
list_name = re.sub('\.', '-', list_name)
# collect stats on posts - success/failures, posts per list
if result == 'success':
list_ns = list_name + '.posts' #set the graphite namespace to ${list-name}.posts
if not list_ns in stats:
stats[list_ns] = 0
stats[list_ns] += 1
else:
list_ns = list_name + '.post_failures'
if not list_ns in stats:
stats[list_ns] = 0
stats[list_ns] += int(result.split()[0])
if stats['posts.total'] > 0:
push_stats(stats)
def get_spam_stats():
logfile = postfix_log
stats={}
# stats['total'] = 0
# stats['inbound'] = 0
# stats['outbound'] = 0
# stats['queue_size'] = 0
stats['spamhaus_blocks'] = 0
stats['spamasassin_rejects'] = 0
stats['greylist_rejects'] = 0
stats['greylist_allow'] = 0
time_now = int(time.time())
five_mins_ago = time_now-300
year = time.strftime('%Y', time.localtime())
for line in reverse_tail.run(logfile):
if not line:
continue
m = re.match(r"(?P<ts>\w+\s+\d+ \d+:\d+:\d+).*$", line)
if m:
ts = m.group('ts')
ts = ts + ' ' + year
ts = int(time.mktime(time.strptime(ts, '%b %d %H:%M:%S %Y')))
if ts < five_mins_ago:
break
m = re.match(r"^\w+\s+\d+ \d+:\d+:\d+.*: (?:554.*(?P<spamhaus>blocked)|(?P<spamd>spamd): result: Y)", line)
if m:
if m.group('spamhaus'):
stats['spamhaus_blocks'] += 1
if m.group('spamd'):
stats['spamasassin_rejects'] += 1
m = re.match(r"\w+\s+\d+ \d+:\d+:\d+.* postgrey\[\d+\]: action=(:?(?P<greylist>greylist)|(?P<pass>pass)),.*", line)
if m:
if m.group('greylist'):
stats['greylist_rejects'] += 1
if m.group('pass'):
stats['greylist_allow'] += 1
push_stats(stats)
def get_post_stats():
logfile = mailman_logs + 'post'
stats={}
stats['posts.total'] = 0
time_now = int(time.time())
five_mins_ago = time_now-300
for line in reverse_tail.run(logfile):
if not line:
continue
s = re.match(r"^(?P<ts>\w+ \d+ \d+:\d+:\d+ \d+) .*post to (?P<list_name>.*) from (?P<poster>.*@*.), size=(?P<msg_size>\d+), message-id.*, (?P<result>.*)$", line)
if s:
ts = s.group('ts')
list_name = s.group('list_name')
poster = s.group('poster')
msg_size = s.group('msg_size')
result = s.group('result')
ts = int(time.mktime(time.strptime(ts,'%b %d %H:%M:%S %Y')))
if ts < five_mins_ago:
break
else:
#print line
stats['posts.total'] += 1
list_name = re.sub('\.', '-', list_name)
# collect stats on posts - success/failures, posts per list
if result == 'success':
list_ns = list_name + '.posts' #set the graphite namespace to ${list-name}.posts
if not list_ns in stats:
stats[list_ns] = 0
stats[list_ns] += 1
else:
list_ns = list_name + '.post_failures'
if not list_ns in stats:
stats[list_ns] = 0
stats[list_ns] += int(result.split()[0])
if stats['posts.total'] > 0:
push_stats(stats)
def get_accesslog_stats(domain):
logfile = logpath + domain + '-access.log'
stats = {}
stats['requests.total'] = 0
stats['bytes_sent'] = 0
stats['requests.api'] = 0
stats['api_bytes_sent'] = 0
time_now = int(time.time())
five_mins_ago = time_now - 300
for line in reverse_tail.run(logfile):
if not line:
continue
s = re.match(
r"^(?P<ip>\d+\.\d+\.\d+\.\d+|^.*).*\[(?P<ts>.*) .*\] \"(?P<request>.*)\" (?P<http_status>\d+) (?P<bytes>\d+)",
line)
ip = s.group('ip')
ts = s.group('ts')
request = s.group('request') # get + uri
http_status = s.group('http_status')
content_size = s.group('bytes')
#11/Oct/2013:02:26:37
ts = int(time.mktime(time.strptime(ts, '%d/%b/%Y:%H:%M:%S')))
if ts < five_mins_ago:
# print line
break
else:
# print line
stats['requests.total'] += 1
stats['bytes_sent'] += int(content_size)
if request == '-': #request is empty
continue
else:
uri = request.split(' ')[1]
if re.match(r"\/api\/", uri):
stats['requests.api'] += 1
stats['api_bytes_sent'] += int(content_size)
push_stats(stats, domain)
def get_subscribe_stats():
logfile = mailman_logs + 'subscribe'
stats = {}
time_now = int(time.time())
five_mins_ago = time_now - 300
for line in reverse_tail.run(logfile):
if not line:
continue
s = re.match(
r"^(?P<ts>\w+ \d+ \d+:\d+:\d+ \d+) \(.*\) (?P<list_name>.*?): (?:(?P<action>new|deleted|pending))",
line)
if s:
ts = s.group('ts')
list_name = s.group('list_name')
action = s.group('action')
ts = int(time.mktime(time.strptime(ts, '%b %d %H:%M:%S %Y')))
if ts < five_mins_ago:
break
else:
list_name = re.sub('\.', '-', list_name)
if action == 'new':
list_ns = 'subscribe.' + list_name + '.new' #setup graphite ns
if action == 'deleted':
list_ns = 'subscribe.' + list_name + '.deleted'
if action == 'pending':
list_ns = 'subscribe.' + list_name + '.pending'
# ip = re.match(r"^.*: pending .* (\d+\.\d+\.\d+\.\d+)", line)
if not list_ns in stats:
stats[list_ns] = 0
stats[list_ns] += 1
push_stats(stats)
def get_accesslog_stats(domain):
logfile = logpath + domain + '-access.log'
stats={}
stats['requests.total'] = 0
stats['bytes_sent'] = 0
stats['requests.api'] = 0
stats['api_bytes_sent'] = 0
time_now = int(time.time())
five_mins_ago = time_now-300
for line in reverse_tail.run(logfile):
if not line:
continue
s = re.match(r"^(?P<ip>\d+\.\d+\.\d+\.\d+|^.*).*\[(?P<ts>.*) .*\] \"(?P<request>.*)\" (?P<http_status>\d+) (?P<bytes>\d+)", line)
ip = s.group('ip')
ts = s.group('ts')
request = s.group('request') # get + uri
http_status = s.group('http_status')
content_size = s.group('bytes')
#11/Oct/2013:02:26:37
ts = int(time.mktime(time.strptime(ts,'%d/%b/%Y:%H:%M:%S')))
if ts < five_mins_ago:
# print line
break
else:
# print line
stats['requests.total'] += 1
stats['bytes_sent'] += int(content_size)
if request == '-': #request is empty
continue
else:
uri = request.split(' ')[1]
if re.match(r"\/api\/", uri):
stats['requests.api'] += 1
stats['api_bytes_sent'] += int(content_size)
push_stats(stats, domain)
def get_subscribe_stats():
logfile = mailman_logs + 'subscribe'
stats={}
time_now = int(time.time())
five_mins_ago = time_now-300
for line in reverse_tail.run(logfile):
if not line:
continue
s = re.match(r"^(?P<ts>\w+ \d+ \d+:\d+:\d+ \d+) \(.*\) (?P<list_name>.*?): (?:(?P<action>new|deleted|pending))", line)
if s:
ts = s.group('ts')
list_name = s.group('list_name')
action = s.group('action')
ts = int(time.mktime(time.strptime(ts,'%b %d %H:%M:%S %Y')))
if ts < five_mins_ago:
break
else:
list_name = re.sub('\.', '-', list_name)
if action == 'new':
list_ns = 'subscribe.' + list_name + '.new' #setup graphite ns
if action == 'deleted':
list_ns = 'subscribe.' + list_name + '.deleted'
if action == 'pending':
list_ns = 'subscribe.' + list_name + '.pending'
# ip = re.match(r"^.*: pending .* (\d+\.\d+\.\d+\.\d+)", line)
if not list_ns in stats:
stats[list_ns] = 0
stats[list_ns] += 1
push_stats(stats)
def get_errorlog_stats(domain):
logfile = logpath + domain + '-error.log'
stats = {}
stats['errors'] = 0
stats['warnings'] = 0
time_now = int(time.time())
five_mins_ago = time_now - 300
for line in reverse_tail.run(logfile):
if not line:
continue
#2013/10/11 06:03:28
s = re.match(r"^(?P<date>.*) (?P<time>\d+:\d+:\d+)", line)
ts = s.group('date') + s.group('time')
ts = int(time.mktime(time.strptime(ts, '%Y/%m/%d%H:%M:%S')))
if ts < five_mins_ago:
break
else:
if '[warn]' in line:
stats['warnings'] += 1
if '[error]' in line:
stats['errors'] += 1
push_stats(stats, domain)
def get_errorlog_stats(domain):
logfile = logpath + domain + '-error.log'
stats={}
stats['errors'] = 0
stats['warnings'] = 0
time_now = int(time.time())
five_mins_ago = time_now-300
for line in reverse_tail.run(logfile):
if not line:
continue
#2013/10/11 06:03:28
s = re.match(r"^(?P<date>.*) (?P<time>\d+:\d+:\d+)", line)
ts = s.group('date') + s.group('time')
ts = int(time.mktime(time.strptime(ts, '%Y/%m/%d%H:%M:%S')))
if ts < five_mins_ago:
break
else:
if '[warn]' in line:
stats['warnings'] += 1
if '[error]' in line:
stats['errors'] += 1
push_stats(stats, domain)