def get_outbound_stats(): #smtp logfile = mailman_logs + 'smtp' stats={} stats['outbound_mails.total'] = 0 time_now = int(time.time()) five_mins_ago = time_now-300 for line in reverse_tail.run(logfile): if not line: continue s = re.match(r"^(?P<ts>\w+ \d+ \d+:\d+:\d+ \d+).* smtp to (?P<list_name>.*) for (?P<rcpt_count>\d+) recips", line) #print line ts = s.group('ts') list_name = s.group('list_name') rcpt_count = s.group('rcpt_count') ts = int(time.mktime(time.strptime(ts,'%b %d %H:%M:%S %Y'))) if ts < five_mins_ago: break else: stats['outbound_mails.total'] += int(rcpt_count) list_name = re.sub('\.', '-', list_name) # collect stats on outbound mails per list list_ns = 'outbound_mails.' + list_name + '.total' if not list_ns in stats: stats[list_ns] = 0 stats[list_ns] += int(rcpt_count) push_stats(stats)
def get_outbound_stats(): #smtp logfile = mailman_logs + 'smtp' stats = {} stats['outbound_mails.total'] = 0 time_now = int(time.time()) five_mins_ago = time_now - 300 for line in reverse_tail.run(logfile): if not line: continue s = re.match( r"^(?P<ts>\w+ \d+ \d+:\d+:\d+ \d+).* smtp to (?P<list_name>.*) for (?P<rcpt_count>\d+) recips", line) #print line ts = s.group('ts') list_name = s.group('list_name') rcpt_count = s.group('rcpt_count') ts = int(time.mktime(time.strptime(ts, '%b %d %H:%M:%S %Y'))) if ts < five_mins_ago: break else: stats['outbound_mails.total'] += int(rcpt_count) list_name = re.sub('\.', '-', list_name) # collect stats on outbound mails per list list_ns = 'outbound_mails.' + list_name + '.total' if not list_ns in stats: stats[list_ns] = 0 stats[list_ns] += int(rcpt_count) push_stats(stats)
def get_spam_stats(): logfile = postfix_log stats = {} # stats['total'] = 0 # stats['inbound'] = 0 # stats['outbound'] = 0 # stats['queue_size'] = 0 stats['spamhaus_blocks'] = 0 stats['spamasassin_rejects'] = 0 stats['greylist_rejects'] = 0 stats['greylist_allow'] = 0 time_now = int(time.time()) five_mins_ago = time_now - 300 year = time.strftime('%Y', time.localtime()) for line in reverse_tail.run(logfile): if not line: continue m = re.match(r"(?P<ts>\w+\s+\d+ \d+:\d+:\d+).*$", line) if m: ts = m.group('ts') ts = ts + ' ' + year ts = int(time.mktime(time.strptime(ts, '%b %d %H:%M:%S %Y'))) if ts < five_mins_ago: break m = re.match( r"^\w+\s+\d+ \d+:\d+:\d+.*: (?:554.*(?P<spamhaus>blocked)|(?P<spamd>spamd): result: Y)", line) if m: if m.group('spamhaus'): stats['spamhaus_blocks'] += 1 if m.group('spamd'): stats['spamasassin_rejects'] += 1 m = re.match( r"\w+\s+\d+ \d+:\d+:\d+.* postgrey\[\d+\]: action=(:?(?P<greylist>greylist)|(?P<pass>pass)),.*", line) if m: if m.group('greylist'): stats['greylist_rejects'] += 1 if m.group('pass'): stats['greylist_allow'] += 1 push_stats(stats)
def get_post_stats(): logfile = mailman_logs + 'post' stats = {} stats['posts.total'] = 0 time_now = int(time.time()) five_mins_ago = time_now - 300 for line in reverse_tail.run(logfile): if not line: continue s = re.match( r"^(?P<ts>\w+ \d+ \d+:\d+:\d+ \d+) .*post to (?P<list_name>.*) from (?P<poster>.*@*.), size=(?P<msg_size>\d+), message-id.*, (?P<result>.*)$", line) if s: ts = s.group('ts') list_name = s.group('list_name') poster = s.group('poster') msg_size = s.group('msg_size') result = s.group('result') ts = int(time.mktime(time.strptime(ts, '%b %d %H:%M:%S %Y'))) if ts < five_mins_ago: break else: #print line stats['posts.total'] += 1 list_name = re.sub('\.', '-', list_name) # collect stats on posts - success/failures, posts per list if result == 'success': list_ns = list_name + '.posts' #set the graphite namespace to ${list-name}.posts if not list_ns in stats: stats[list_ns] = 0 stats[list_ns] += 1 else: list_ns = list_name + '.post_failures' if not list_ns in stats: stats[list_ns] = 0 stats[list_ns] += int(result.split()[0]) if stats['posts.total'] > 0: push_stats(stats)
def get_spam_stats(): logfile = postfix_log stats={} # stats['total'] = 0 # stats['inbound'] = 0 # stats['outbound'] = 0 # stats['queue_size'] = 0 stats['spamhaus_blocks'] = 0 stats['spamasassin_rejects'] = 0 stats['greylist_rejects'] = 0 stats['greylist_allow'] = 0 time_now = int(time.time()) five_mins_ago = time_now-300 year = time.strftime('%Y', time.localtime()) for line in reverse_tail.run(logfile): if not line: continue m = re.match(r"(?P<ts>\w+\s+\d+ \d+:\d+:\d+).*$", line) if m: ts = m.group('ts') ts = ts + ' ' + year ts = int(time.mktime(time.strptime(ts, '%b %d %H:%M:%S %Y'))) if ts < five_mins_ago: break m = re.match(r"^\w+\s+\d+ \d+:\d+:\d+.*: (?:554.*(?P<spamhaus>blocked)|(?P<spamd>spamd): result: Y)", line) if m: if m.group('spamhaus'): stats['spamhaus_blocks'] += 1 if m.group('spamd'): stats['spamasassin_rejects'] += 1 m = re.match(r"\w+\s+\d+ \d+:\d+:\d+.* postgrey\[\d+\]: action=(:?(?P<greylist>greylist)|(?P<pass>pass)),.*", line) if m: if m.group('greylist'): stats['greylist_rejects'] += 1 if m.group('pass'): stats['greylist_allow'] += 1 push_stats(stats)
def get_post_stats(): logfile = mailman_logs + 'post' stats={} stats['posts.total'] = 0 time_now = int(time.time()) five_mins_ago = time_now-300 for line in reverse_tail.run(logfile): if not line: continue s = re.match(r"^(?P<ts>\w+ \d+ \d+:\d+:\d+ \d+) .*post to (?P<list_name>.*) from (?P<poster>.*@*.), size=(?P<msg_size>\d+), message-id.*, (?P<result>.*)$", line) if s: ts = s.group('ts') list_name = s.group('list_name') poster = s.group('poster') msg_size = s.group('msg_size') result = s.group('result') ts = int(time.mktime(time.strptime(ts,'%b %d %H:%M:%S %Y'))) if ts < five_mins_ago: break else: #print line stats['posts.total'] += 1 list_name = re.sub('\.', '-', list_name) # collect stats on posts - success/failures, posts per list if result == 'success': list_ns = list_name + '.posts' #set the graphite namespace to ${list-name}.posts if not list_ns in stats: stats[list_ns] = 0 stats[list_ns] += 1 else: list_ns = list_name + '.post_failures' if not list_ns in stats: stats[list_ns] = 0 stats[list_ns] += int(result.split()[0]) if stats['posts.total'] > 0: push_stats(stats)
def get_accesslog_stats(domain): logfile = logpath + domain + '-access.log' stats = {} stats['requests.total'] = 0 stats['bytes_sent'] = 0 stats['requests.api'] = 0 stats['api_bytes_sent'] = 0 time_now = int(time.time()) five_mins_ago = time_now - 300 for line in reverse_tail.run(logfile): if not line: continue s = re.match( r"^(?P<ip>\d+\.\d+\.\d+\.\d+|^.*).*\[(?P<ts>.*) .*\] \"(?P<request>.*)\" (?P<http_status>\d+) (?P<bytes>\d+)", line) ip = s.group('ip') ts = s.group('ts') request = s.group('request') # get + uri http_status = s.group('http_status') content_size = s.group('bytes') #11/Oct/2013:02:26:37 ts = int(time.mktime(time.strptime(ts, '%d/%b/%Y:%H:%M:%S'))) if ts < five_mins_ago: # print line break else: # print line stats['requests.total'] += 1 stats['bytes_sent'] += int(content_size) if request == '-': #request is empty continue else: uri = request.split(' ')[1] if re.match(r"\/api\/", uri): stats['requests.api'] += 1 stats['api_bytes_sent'] += int(content_size) push_stats(stats, domain)
def get_subscribe_stats(): logfile = mailman_logs + 'subscribe' stats = {} time_now = int(time.time()) five_mins_ago = time_now - 300 for line in reverse_tail.run(logfile): if not line: continue s = re.match( r"^(?P<ts>\w+ \d+ \d+:\d+:\d+ \d+) \(.*\) (?P<list_name>.*?): (?:(?P<action>new|deleted|pending))", line) if s: ts = s.group('ts') list_name = s.group('list_name') action = s.group('action') ts = int(time.mktime(time.strptime(ts, '%b %d %H:%M:%S %Y'))) if ts < five_mins_ago: break else: list_name = re.sub('\.', '-', list_name) if action == 'new': list_ns = 'subscribe.' + list_name + '.new' #setup graphite ns if action == 'deleted': list_ns = 'subscribe.' + list_name + '.deleted' if action == 'pending': list_ns = 'subscribe.' + list_name + '.pending' # ip = re.match(r"^.*: pending .* (\d+\.\d+\.\d+\.\d+)", line) if not list_ns in stats: stats[list_ns] = 0 stats[list_ns] += 1 push_stats(stats)
def get_accesslog_stats(domain): logfile = logpath + domain + '-access.log' stats={} stats['requests.total'] = 0 stats['bytes_sent'] = 0 stats['requests.api'] = 0 stats['api_bytes_sent'] = 0 time_now = int(time.time()) five_mins_ago = time_now-300 for line in reverse_tail.run(logfile): if not line: continue s = re.match(r"^(?P<ip>\d+\.\d+\.\d+\.\d+|^.*).*\[(?P<ts>.*) .*\] \"(?P<request>.*)\" (?P<http_status>\d+) (?P<bytes>\d+)", line) ip = s.group('ip') ts = s.group('ts') request = s.group('request') # get + uri http_status = s.group('http_status') content_size = s.group('bytes') #11/Oct/2013:02:26:37 ts = int(time.mktime(time.strptime(ts,'%d/%b/%Y:%H:%M:%S'))) if ts < five_mins_ago: # print line break else: # print line stats['requests.total'] += 1 stats['bytes_sent'] += int(content_size) if request == '-': #request is empty continue else: uri = request.split(' ')[1] if re.match(r"\/api\/", uri): stats['requests.api'] += 1 stats['api_bytes_sent'] += int(content_size) push_stats(stats, domain)
def get_subscribe_stats(): logfile = mailman_logs + 'subscribe' stats={} time_now = int(time.time()) five_mins_ago = time_now-300 for line in reverse_tail.run(logfile): if not line: continue s = re.match(r"^(?P<ts>\w+ \d+ \d+:\d+:\d+ \d+) \(.*\) (?P<list_name>.*?): (?:(?P<action>new|deleted|pending))", line) if s: ts = s.group('ts') list_name = s.group('list_name') action = s.group('action') ts = int(time.mktime(time.strptime(ts,'%b %d %H:%M:%S %Y'))) if ts < five_mins_ago: break else: list_name = re.sub('\.', '-', list_name) if action == 'new': list_ns = 'subscribe.' + list_name + '.new' #setup graphite ns if action == 'deleted': list_ns = 'subscribe.' + list_name + '.deleted' if action == 'pending': list_ns = 'subscribe.' + list_name + '.pending' # ip = re.match(r"^.*: pending .* (\d+\.\d+\.\d+\.\d+)", line) if not list_ns in stats: stats[list_ns] = 0 stats[list_ns] += 1 push_stats(stats)
def get_errorlog_stats(domain): logfile = logpath + domain + '-error.log' stats = {} stats['errors'] = 0 stats['warnings'] = 0 time_now = int(time.time()) five_mins_ago = time_now - 300 for line in reverse_tail.run(logfile): if not line: continue #2013/10/11 06:03:28 s = re.match(r"^(?P<date>.*) (?P<time>\d+:\d+:\d+)", line) ts = s.group('date') + s.group('time') ts = int(time.mktime(time.strptime(ts, '%Y/%m/%d%H:%M:%S'))) if ts < five_mins_ago: break else: if '[warn]' in line: stats['warnings'] += 1 if '[error]' in line: stats['errors'] += 1 push_stats(stats, domain)
def get_errorlog_stats(domain): logfile = logpath + domain + '-error.log' stats={} stats['errors'] = 0 stats['warnings'] = 0 time_now = int(time.time()) five_mins_ago = time_now-300 for line in reverse_tail.run(logfile): if not line: continue #2013/10/11 06:03:28 s = re.match(r"^(?P<date>.*) (?P<time>\d+:\d+:\d+)", line) ts = s.group('date') + s.group('time') ts = int(time.mktime(time.strptime(ts, '%Y/%m/%d%H:%M:%S'))) if ts < five_mins_ago: break else: if '[warn]' in line: stats['warnings'] += 1 if '[error]' in line: stats['errors'] += 1 push_stats(stats, domain)