def main():
init_logging() # Before anything else, initialise logging.
logger.info("Beginning process cycle at %s (UTC)", timefunc.utcnow())
socket.setdefaulttimeout(config.timeout)
global stat_re, addy_re, chain_re
stat_re = re.compile("(\w{1,12})\s+([0-9A-H?]{12}\s.*)")
addy_re = re.compile('\$remailer\{"([0-9a-z]{1,12})"\}\s\=\s"\<(.*)\>\s')
chain_re = re.compile("\((\S{1,12})\s(\S{1,12})\)")
# Are we running in testmode? Testmode implies the script was executed
# without a --live argument.
testmode = live_or_test(sys.argv)
# If not in testmode, fetch url's and process them
if not testmode:
pingers = db.pinger_names()
for row in pingers:
url = url_fetch(row[1])
if url:
url_process(row[0], url)
# Fetch pubring.mix files and write them to the DB
getkeystats()
else:
logger.debug("Running in testmode, url's will not be retreived")
# We need to do some periodic housekeeping. It's not very process
# intensive so might as well do it every time we run.
db.housekeeping(timefunc.hours_ago(config.dead_after_hours))
# For a pinger to be considered active, it must appear in tables mlist2
# and pingers. This basically means, don't create empty pinger columns
# in the index file.
active_pingers = db.active_pinger_names()
# A boolean value to rotate row colours within the index
rotate_color = 0
# The main loop. This creates individual remailer text files and
# indexing data based on database values.
for name, addy in db.distinct_rem_names():
logger.debug("Generating statsistics for remailer %s", name)
# remailer_vitals is a dictionary of standard deviation and average
# values for a specific remailer.
remailer_vitals = gen_remailer_vitals(name, addy)
# remailer_active_pings: Based on the vitals generated above, we now
# extract stats lines for pingers considered active. The up_hist
# part is used by the fail_recover routine.
remailer_active_pings = db.remailer_active_pings(remailer_vitals)
# If a remailers is perceived to be dead, timestamp it in the
# genealogy table. Likewise, if it's not dead, unstamp it.
fail_recover(name, addy, remailer_active_pings)
# Write the remailer text file that contains pinger stats and averages
logger.debug("Writing stats file for %s %s", name, addy)
write_remailer_stats(name, addy, remailer_vitals)
# Rotate the colour used in index generation.
rotate_color = not rotate_color
db.gene_find_new()
index()
genealogy()
uptimes()
chainstats()
writekeystats()
logger.info("Processing cycle completed at %s (UTC)", timefunc.utcnow())
def index():
"""Generate an HTML index table referencing remailer name against pinger
name. The content of the table is remailer uptimes."""
# Generate an index filename and open it.
filename = "%s/index.html" % config.reportdir
index = open(filename, 'w')
# Write the standard HTML headers and the initial BODY parts.
index.write('''<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" "http://www.w3.org/TR/html4/loose.dtd">
<html>
<head>
<meta http-equiv="Content-Type" content="text/html; charset=ISO-8859-1">
<meta http-equiv="Content-Style-Type" content="text/css2" />
<meta name="keywords" content="Mixmaster,Echolot,Remailer,Banana,Bananasplit">
<title>Bananasplit Website - Meta Statistics</title>
<link rel="StyleSheet" href="stats.css" type="text/css">
</head>
<body>
<table border="0" bgcolor="#000000">\n''')
# For a pinger to be considered active, it must appear in tables mlist2
# and pingers. This basically means, don't create empty pinger columns
# in the index file.
active_pingers = active_pinger_names()
index.write('<tr bgcolor="#F08080"><th></th><th>Chain From</th><th>Chain To</th>\n')
# Write the pinger names as a header row.
for pinger in active_pingers:
index.write('<th><a href="%s">%s</a></th>\n' % (pinger[1], pinger[0]))
# Add the final header row titles and close the row.
index.write('<th>Average</th><th>StdDev</th><th>Count</th></tr>\n')
# Set a flag for rotating backgroup colours
color_flag = False
# Define some time related variables
now = utcnow()
# Make a dictionary of how many chain from's for each remailer
chfr_dict = chain_from_count2()
chto_dict = chain_to_count2()
for name, addy in distinct_rem_names():
# We should have a key for every remailer, this is just a safeguard
fr_count = keycheck(chfr_dict, name)
to_count = keycheck(chto_dict, name)
# First lets establish the bacground color for the row being processed
color = bg_color(color_flag)
color_flag = not color_flag
file, chfr, chto, url = gen_filename(name, addy)
# Generate the HTML for Table Row headers
index.write('<tr bgcolor="%s"><th class="tableleft">\n' % (color,))
# Generate the HTML for the Remailer Name column
index.write('<a href="%s" title="%s">' % (url, addy))
index.write('%s</a></th>\n' % (name,))
# Generate the HTML for the Chain From column
index.write('<td align="center">')
index.write('<a href="chfr.%s" title="Broken Chains from %s">' % (url, addy))
index.write('%s</a></td>\n' % (fr_count,))
# Generate the HTML for the Chain To column
index.write('<td align="center">')
index.write('<a href="chto.%s" title="Broken Chains to %s">' % (url, addy))
index.write('%s</a></td>\n' % (to_count,))
uptimes = remailer_index_pings(name, addy)
# Now we need a loop to parse each pinger column within the remailer
# row
for pinger in active_pingers:
ping_name = pinger[0]
title = "Remailer: %s Pinger: %s" % (name, ping_name)
index.write('<td align="center" title="%s">' % title)
if ping_name in uptimes:
index.write('%3.1f</td>\n' % (uptimes[ping_name],))
else:
index.write('</td>\n')
avg,stddev,count = remailer_index_stats(name, addy)
# Average and StdDev can return 'None' if remailers have no current
# data. We have to catch this in order to present floats to the string
# formatting line.
if avg == None: avg = 0
if stddev == None: stddev = 0
# Write the average, stddev and count to the end of each index row.
index.write('<td>%3.2f</td><td>%3.2f</td><td>%d</td>' % (avg, stddev, count))
index.write('</tr>\n')
# Add a row to the bottom of the table showing the count of remailer known
# to each pinger. We have to write a few empty cells to make things line
# up because of the chain counts and totals columns.
pinger_totals = remailer_index_count()
index.write('<tr bgcolor="#F08080"><th class="tableleft">Count</th><td></td><td></td>\n')
for pinger in active_pingers:
ping_name = pinger[0]
if pinger_totals.has_key(ping_name):
index.write('<td title="%s">%s</td>\n' % (ping_name, pinger_totals[ping_name]))
else:
index.write('<td title="%s">0</td>\n' % (ping_name,))
index.write('<td></td><td></td><td></td></tr>\n</table>\n')
index.write('<br>Last update: %s (UTC)<br>\n' % utcnow())
index.write('<br><a href="%s">Remailer Genealogy</a>' % config.gene_report_name)
index.write('<br><a href="%s">Failing Remailers</a>' % config.failed_report_name)
index.write('<br><a href="%s">Uptime Averages</a>' % config.uptime_report_name)
index.write('<br><a href="%s">Keyring Stats</a>' % config.keyindex)
# Write closing tags and close file
index.write('</body></html>')
index.close()
def failed(report_name):
filename = "%s/%s" % (config.reportdir, config.failed_report_name)
htmlfile = open(filename, 'w')
# Write standard html header section
htmlfile.write("""<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" "http://www.w3.org/TR/html4/loose.dtd">
<html>
<head>
<meta http-equiv="Content-Type" content="text/html; charset=ISO-8859-1">
<meta http-equiv="Content-Style-Type" content="text/css2" />
<meta name="keywords" content="Mixmaster,Remailer,Banana,Bananasplit">
<title>Bananasplit Website - Failing Remailers</title>
<link rel="StyleSheet" href="stats.css" type="text/css">
</head>
<body>
<h1>Failing Remailers</h1>
Remailer stats are normally biased away from the current day. This is because
different latencies would skew results due to outstanding pings. However the
current day does provide a rapid insight into remailers that are currently
experiencing problems.<br>
<p>The following table lists remailers that are <u>currently</u> averaging less
than %d%% return on pings. Please consider that this could be due to very high
latency rather than actual remailer failure.</p>
<p><i>Note: Bunker is not actually failing. It's a Middleman remailer with no
current stats which means it cannot random hop pings back to the pinger.</i></P>
<table border="0" bgcolor="#000000">
<tr bgcolor="#F08080">
<th>Remailer Name</th><th>Ping Responses</th>
</tr>""" % (config.failpoint * 10))
rotate_color = 0
for name, addy in db.distinct_rem_names():
remailer_vitals = gen_remailer_vitals(name, addy)
logger.debug("Checking remailer %s %s", name, addy)
addy_noat = addy.replace('@',".")
full_name = "%s.%s" % (name, addy_noat)
active_pings = db.remailer_active_pings(remailer_vitals)
if len(active_pings) == 0:
logger.info("We have no active pings for %s %s", name, addy)
continue
remailer_vitals['uptime'] = up_today(active_pings)
if remailer_vitals['uptime'] < config.failpoint:
logger.info("Remailer %s %s is failed.", name, addy)
# Rotate background colours for rows
if rotate_color:
bgcolor = "#ADD8E6"
else:
bgcolor = "#E0FFFF"
rotate_color = not rotate_color
htmlfile.write('<tr bgcolor="%s"><th class="tableleft"><a href="%s.txt" title="%s">%s</a></th>' % (bgcolor, full_name, addy, name))
htmlfile.write('<td>%d%%</td></tr>\n' % (remailer_vitals['uptime'] * 10))
# if remailer_vitals('uptime'] > config.goodpoint:
# # Stats are greater than 50%, so delete any entries for this
# # remailer from the failed table.
# logger.debug("%s is healthy, deleting any DB entries it might have", name)
htmlfile.write('</table>\n')
htmlfile.write('<br>Last update: %s (UTC)<br>\n' % utcnow())
htmlfile.write('<br><a href="index.html">Return to Index</a>')
htmlfile.write('</body></html>')