def main():
# initialize the gmail module
success,explanation_str = send_gmail.init_gmail()
if not success:
integrationtestlib.log(explanation_str)
sys.exit(0)
#add Eric Kimbrel to the email notify list
integrationtestlib.notify_list.append("*****@*****.**")
try:
integrationtestlib.log("Looking up time_servers")
# verify that there are at least 8 time servers running
servers = advertise_lookup("time_server")
if len(servers) < 8:
integrationtestlib.log('WARNING: only '+str(len(servers))+' timeservers are running!')
integrationtestlib.notify('WARNING: test_time_servers_running.py FAILED, only '+str(len(servers))+' timeservers are running!', "test_time_servers_running test failed")
integrationtestlib.log("Finished looking up test servers... Test Passed")
print "........................................................\n"
except:
integrationtestlib.notify("Test failed for an unknown reason:\n" +
traceback.format_exc(), "test_time_servers_running test failed")
def monitor_processes(monitor_process_list, command_list, machine_name):
"""
<Purpose>
Checks to make sure that the critical processes on the machine 'seattle' are still running
<Exceptions>
None
<Arguments>
monitor_process_list - a list of all the critical processes that should be checked to
see if they are up and running.
command_list - a list of all the commands required to find all the relevant processes
<Return>
None
"""
#string that holds the name of all the processes that are found to be running using the
#ps commands that was passed in as argument
processes_string=""
integrationtestlib.log("Starting monitoring process on "+machine_name)
#run a command on the linux machine to find all the relevant processes
for command in command_list:
try:
relevant_processes, command_error = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE).communicate()
except:
integrationtestlib.handle_exception("Failed to run command: "+command)
sys.exit(1)
#make a string of all the processes
processes_string = processes_string+relevant_processes
print processes_string
#keeps track to see if any processes are down
critical_process_down=False
error_message="WARNING: Critical processes down! Seattle developers please start the processes up as soon as possible\n"
error_message=error_message+"Listing processes that are down:\n"
#goes through the list of monitor_process_list to ensure that all processes are running
for critical_process in monitor_process_list:
integrationtestlib.log("Checking process: "+critical_process+".......")
if not critical_process in processes_string:
critical_process_down=True
error_message = error_message+critical_process+" is down on "+machine_name+".poly.edu\n"
print "FAIL"
else:
print "PASS"
error_message=error_message+"end of list of processes that are down.\n................................"
if critical_process_down:
integrationtestlib.notify(error_message, "Critical process down!")
irc_seattlebot.send_msg(error_message)
else:
integrationtestlib.log("All critical processes on "+machine_name+" are up and running")
print(".........................................................")
def lookup_timedout():
"""
<Purpose>
Waits for lookup_done_event and notifies the folks on the
notify_list (global var) of the lookup timeout.
<Arguments>
None.
<Exceptions>
None.
<Side Effects>
Sends an email to the notify_list folks
<Returns>
None.
"""
integrationtestlib.log("in lookup_timedout()")
notify_msg = "Centralized lookup failed -- lookup_timedout() fired after 30 sec."
# wait for the event to be set, timeout after 30 minutes
wait_time = 1800
tstamp_before_wait = nonportable.getruntime()
lookup_done_event.wait(wait_time)
tstamp_after_wait = nonportable.getruntime()
t_waited = tstamp_after_wait - tstamp_before_wait
if abs(wait_time - t_waited) < 5:
notify_msg += " And lookup stalled for over 30 minutes (max timeout value)."
else:
notify_msg += " And lookup stalled for " + str(t_waited) + " seconds"
integrationtestlib.notify(notify_msg)
return
def monitor_processes(monitor_process_list, command_list, machine_name):
"""
<Purpose>
Checks to make sure that the critical processes on the machine 'seattle' are still running
<Exceptions>
None
<Arguments>
monitor_process_list - a list of all the critical processes that should be checked to
see if they are up and running.
command_list - a list of all the commands required to find all the relevant processes
<Return>
None
"""
#string that holds the name of all the processes that are found to be running using the
#ps commands that was passed in as argument
processes_string=""
integrationtestlib.log("Starting monitoring process on "+machine_name)
#run a command on the linux machine to find all the relevant processes
for command in command_list:
try:
relevant_processes, command_error = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE).communicate()
except:
integrationtestlib.handle_exception("Failed to run command: "+command)
sys.exit(1)
#make a string of all the processes
processes_string = processes_string+relevant_processes
print processes_string
#keeps track to see if any processes are down
critical_process_down=False
error_message="WARNING: Critical processes down! Seattle developers please start the processes up as soon as possible\n"
error_message=error_message+"Listing processes that are down:\n"
#goes through the list of monitor_process_list to ensure that all processes are running
for critical_process in monitor_process_list:
integrationtestlib.log("Checking process: "+critical_process+".......")
if not critical_process in processes_string:
critical_process_down=True
error_message = error_message+critical_process+" is down on "+machine_name+".cs.washington.edu\n"
print "FAIL"
else:
print "PASS"
error_message=error_message+"end of list of processes that are down.\n................................"
if critical_process_down:
integrationtestlib.notify(error_message)
irc_seattlebot.send_msg(error_message)
else:
integrationtestlib.log("All critical processes on "+machine_name+" are up and running")
print(".........................................................")
def main():
success,explanation_str = send_gmail.init_gmail()
if not success:
integrationtestlib.log(explanation_str)
sys.exit(0)
command = "df -h | grep '/dev/sda' | awk '{print $5}'"
command_output_fd = os.popen(command)
# Get the output and get rid of the extra lines and % sign.
disk_use_percent = int(command_output_fd.read().split()[0][:-1])
hostname = socket.gethostname() + ".cs.washington.edu"
subject = "High disk usage"
if disk_use_percent > 95:
message = "CRITICAL: Very High Disk Usage on %s: %s percent used" % ( hostname, disk_use_percent)
integrationtestlib.notify(message, subject)
irc_seattlebot.send_msg(message)
elif disk_use_percent > 90:
message = "WARNING: High disk usage on %s: %s percent used" % ( hostname, disk_use_percent)
integrationtestlib.notify(message, subject)
def lookup_timedout():
"""
<Purpose>
Waits for lookup_done_event and notifies the folks on the
notify_list (global var) of the lookup timeout.
<Arguments>
None.
<Exceptions>
None.
<Side Effects>
Sends an email to the notify_list folks
<Returns>
None.
"""
integrationtestlib.log("in lookup_timedout()")
notify_msg = "Centralized lookup failed -- lookup_timedout() fired after 30 sec."
# wait for the event to be set, timeout after 30 minutes
wait_time = 1800
tstamp_before_wait = nonportable.getruntime()
lookup_done_event.wait(wait_time)
tstamp_after_wait = nonportable.getruntime()
t_waited = tstamp_after_wait - tstamp_before_wait
if abs(wait_time - t_waited) < 5:
notify_msg += " And lookup stalled for over 30 minutes (max timeout value)."
else:
notify_msg += " And lookup stalled for " + str(t_waited) + " seconds"
integrationtestlib.notify(notify_msg)
return
def main():
# initialize the gmail module
success, explanation_str = send_gmail.init_gmail()
if not success:
integrationtestlib.log(explanation_str)
sys.exit(0)
#add Eric Kimbrel to the email notify list
integrationtestlib.notify_list.append("*****@*****.**")
try:
integrationtestlib.log("Looking up time_servers")
# verify that there are at least 8 time servers running
servers = advertise_lookup("time_server")
if len(servers) < 8:
integrationtestlib.log('WARNING: only ' + str(len(servers)) +
' timeservers are running!')
integrationtestlib.notify(
'WARNING: test_time_servers_running.py FAILED, only ' +
str(len(servers)) + ' timeservers are running!',
"test_time_servers_running test failed")
integrationtestlib.log(
"Finished looking up test servers... Test Passed")
print "........................................................\n"
except:
integrationtestlib.notify(
"Test failed for an unknown reason:\n" + traceback.format_exc(),
"test_time_servers_running test failed")
def sendmail(start_time, err_msg, elapsed_time, err_interval, counter,IP): # Recipient email to the send the notification... integrationtestlib.notify_list = ['*****@*****.**'] subject = "Error caught while testing GeoIP." message = "Monitoring script started at: " + str(time.ctime(start_time)) message += "\n Error occurred " + str(counter) + " times, in the last " +\ str(elapsed_time/60.0) + " mins." message += "\n Time since the last error: " + str(err_interval/60.0) message += "\n Error: "+ str(err_msg) + "\n\n" integrationtestlib.notify(message, subject)
def sendmail(start_time, err_msg, elapsed_time, err_interval, counter, IP):
# Recipient email to the send the notification...
integrationtestlib.notify_list = ['*****@*****.**']
subject = "Error caught while testing GeoIP."
message = "Monitoring script started at: " + str(time.ctime(start_time))
message += "\n Error occurred " + str(counter) + " times, in the last " +\
str(elapsed_time/60.0) + " mins."
message += "\n Time since the last error: " + str(err_interval / 60.0)
message += "\n Error: " + str(err_msg) + "\n\n"
integrationtestlib.notify(message, subject)
def report_results():
# If there are no entries in the dictionary, then no errors occurred.
if len(ERRORS) == 0:
integrationtestlib.log('All tests successful!')
return
# Otherwise, errors occurred...
error_string = 'The following errors occurred:\n'
for function in ERRORS:
for error in ERRORS[function]:
error_string += '\n[' + function + '] ' + error
integrationtestlib.log(error_string)
integrationtestlib.notify(error_string, 'Custom Installer Builder test failure')
def check_opendht_servers():
"""
<Purpose>
Checks to see how many servers are up for the opendht advertisement.
If the number of server is less then 100 then an email notification
is sent to the seattle developers.
<Argument>
None
<Exception>
None
<Side_Effects>
None
<Returns>
None
"""
notify_message = "There aren't enough opendht servers up and running currently."
subject = "opendht_servercount test failed."
try:
# Retrieve the list of servers for opendht
opendht_server_list = openDHTadvertise_get_proxy_list(
maxnumberofattempts=150)
except:
# An exception is raised if there are no servers up and running for opendht by
# openDHTadvertise_get_proxy_list().
integrationtestlib.handle_exception(
"There are no servers up for opendht!", subject)
integrationtestlib.log(
"Retrieved the list of opendht servers up and running.")
integrationtestlib.log(
"There are " + str(len(opendht_server_list)) +
" servers up and running for opendht advertisement.")
# Check to see if there are less then 100 servrs up and running.
# If there are less then 100 servers running then notify the seattle
# developers about it.
if len(opendht_server_list) < min_num_servers:
subject += " There are only " + str(
len(opendht_server_list
)) + " servers up and running for opendht advertisement."
integrationtestlib.notify(notify_message, subject)
def report_results(test_results):
failures = ''
for os_name in test_results:
tests_failed = ''
for test_name in test_results[os_name]:
if not test_results[os_name][test_name]:
tests_failed += test_name + ' '
if tests_failed != '':
failures += ' ' + os_name + ': ' + tests_failed + '\n'
if failures == '':
integrationtestlib.log('All tests successful!')
else:
results = 'The following tests failed:\n\n' + failures
integrationtestlib.log(results)
integrationtestlib.notify(results, 'Custom Installer Builder test failure')
def report_results(test_results):
failures = ""
for os_name in test_results:
tests_failed = ""
for test_name in test_results[os_name]:
if not test_results[os_name][test_name]:
tests_failed += test_name + " "
if tests_failed != "":
failures += " " + os_name + ": " + tests_failed + "\n"
if failures == "":
integrationtestlib.log("All tests successful!")
else:
results = "The following tests failed:\n\n" + failures
integrationtestlib.log(results)
integrationtestlib.notify(results, "Custom Installer Builder test failure")
def report_results(test_results):
failures = ''
for os_name in test_results:
tests_failed = ''
for test_name in test_results[os_name]:
if not test_results[os_name][test_name]:
tests_failed += test_name + ' '
if tests_failed != '':
failures += ' ' + os_name + ': ' + tests_failed + '\n'
if failures == '':
integrationtestlib.log('All tests successful!')
else:
results = 'The following tests failed:\n\n' + failures
integrationtestlib.log(results)
integrationtestlib.notify(results, 'Custom Installer Builder test failure')
def main():
"""
<Purpose>
Call check_nodes and then notify developers if result is unusual.
<Exceptions>
None
<Side Effects>
May send out a notification email.
<Return>
None
"""
# Setup the gmail lib for sending notification
success, explanation_str = send_gmail.init_gmail()
if not success:
integrationtestlib.log(explanation_str)
sys.exit(0)
notification_subject = "test_lookup_node_states() failed"
results = check_nodes()
integrationtestlib.log("Lookup results: " + str(results))
# Check to see if any of the results is not normal, and
# send notifications accordingly.
message = "Too many nodes in state "
if results['acceptdonation'] > max_acceptdonation_nodes:
message += "acceptdonation: " + str(results['acceptdonation'])
elif results['canonical'] > max_canonical_nodes:
message += "canonical: " + str(results['canonical'])
if results['twopercent'] < min_twopercent_nodes:
message = "Too few nodes in state twopercent: " + str(
results['twopercent'])
message += "\nLookup results:\n" + str(results)
print message
integrationtestlib.notify(message, notification_subject)
def main():
success,explanation_str = send_gmail.init_gmail()
#integrationtestlib.notify_list=['*****@*****.**']
if not success:
integrationtestlib.log(explanation_str)
sys.exit(0)
command = "df -h | grep '/dev/sda' | awk '{print $5}'"
command_output_fd = os.popen(command)
# Get the output and get rid of the extra lines and % sign.
disk_use_percent = int(command_output_fd.read().split()[0][:-1])
command_output_fd.close()
disk_free_command = "df -h | grep '/dev/sda' | awk '{print $4}'"
disk_free_fd = os.popen(disk_free_command)
free_space = disk_free_fd.read()
disk_free_fd.close()
hostname = socket.gethostname() + ".poly.edu"
subject = "High disk usage"
if disk_use_percent >= 95:
message = "CRITICAL: Very High Disk Usage on %s: %s percent used.\n" % ( hostname, disk_use_percent)
message += "Disk space free: %s" % free_space
integrationtestlib.log(message)
integrationtestlib.notify(message, subject)
irc_seattlebot.send_msg(message)
elif disk_use_percent > 90:
message = "WARNING: High disk usage on %s: %s percent used.\n" % ( hostname, disk_use_percent)
message += "Disk space free: %s" % free_space
integrationtestlib.log(message)
integrationtestlib.notify(message, subject)
print "Current disk usage: %s percent" % disk_use_percent
print "Free disk space: %s" % free_space
def check_opendht_servers():
"""
<Purpose>
Checks to see how many servers are up for the opendht advertisement.
If the number of server is less then 100 then an email notification
is sent to the seattle developers.
<Argument>
None
<Exception>
None
<Side_Effects>
None
<Returns>
None
"""
notify_message = "There aren't enough opendht servers up and running currently."
subject = "opendht_servercount test failed."
try:
# Retrieve the list of servers for opendht
opendht_server_list = openDHTadvertise_get_proxy_list(maxnumberofattempts = 150)
except:
# An exception is raised if there are no servers up and running for opendht by
# openDHTadvertise_get_proxy_list().
integrationtestlib.handle_exception("There are no servers up for opendht!", subject)
integrationtestlib.log("Retrieved the list of opendht servers up and running.")
integrationtestlib.log("There are " + str(len(opendht_server_list)) + " servers up and running for opendht advertisement.")
# Check to see if there are less then 100 servrs up and running.
# If there are less then 100 servers running then notify the seattle
# developers about it.
if len(opendht_server_list) < min_num_servers:
subject += " There are only " + str(len(opendht_server_list)) + " servers up and running for opendht advertisement."
integrationtestlib.notify(notify_message, subject)
def main():
success, explanation_str = send_gmail.init_gmail()
#integrationtestlib.notify_list=['*****@*****.**']
if not success:
integrationtestlib.log(explanation_str)
sys.exit(0)
command = "df -h | grep '/dev/sda' | awk '{print $5}'"
command_output_fd = os.popen(command)
# Get the output and get rid of the extra lines and % sign.
disk_use_percent = int(command_output_fd.read().split()[0][:-1])
command_output_fd.close()
disk_free_command = "df -h | grep '/dev/sda' | awk '{print $4}'"
disk_free_fd = os.popen(disk_free_command)
free_space = disk_free_fd.read()
disk_free_fd.close()
hostname = socket.gethostname() + ".poly.edu"
subject = "High disk usage"
if disk_use_percent >= 95:
message = "CRITICAL: Very High Disk Usage on %s: %s percent used.\n" % (
hostname, disk_use_percent)
message += "Disk space free: %s" % free_space
integrationtestlib.log(message)
integrationtestlib.notify(message, subject)
irc_seattlebot.send_msg(message)
elif disk_use_percent > 90:
message = "WARNING: High disk usage on %s: %s percent used.\n" % (
hostname, disk_use_percent)
message += "Disk space free: %s" % free_space
integrationtestlib.log(message)
integrationtestlib.notify(message, subject)
print "Current disk usage: %s percent" % disk_use_percent
print "Free disk space: %s" % free_space
def main():
"""
<Purpose>
Call check_nodes with the two different servers: opendht and central. Retrieve the result
and then notify developers if result is unusual
<Exceptions>
None
<Side Effects>
May send out an email or notify on irc.
<Return>
None
"""
#setup the gmail for sending notification
success,explanation_str = send_gmail.init_gmail()
if not success:
integrationtestlib.log(explanation_str)
sys.exit(0)
notification_subject = "test_lookup_node_states() failed"
max_acceptdonation_nodes = 50
max_canonical_nodes = 50
max_movingtoonepercent = 20
min_onepercentmanyevents_nodes = 300
central_results = check_nodes('central')
integrationtestlib.log("Lookup results for central: "+ str(central_results))
#check to see if any of the results is not normal, and send notifications accordingly
#also send a message to irc.
if central_results['acceptdonation'] > max_acceptdonation_nodes:
message="Too many nodes in acceptdonation state: "+str(central_results['acceptdonation'])+"\nResults from 'central' server:\n"+str(central_results)
print message
integrationtestlib.notify(message, notification_subject)
elif central_results['canonical'] > max_canonical_nodes:
message="Too many nodes in canonical state: "+str(central_results['canonical'])+"\nResults from 'central' server:\n"+str(central_results)
print message
integrationtestlib.notify(message, notification_subject)
elif central_results['onepercent_manyevent'] < min_onepercentmanyevents_nodes:
message="Too few nodes in onepercentmanyevents state: "+str(central_results['onepercent_manyevent'])+"\nResults from 'central' server:\n"+str(central_results)
print message
integrationtestlib.notify(message, notification_subject)
opendht_results = check_nodes('opendht')
print opendht_results
def main():
# Initialize the gmail setup.
success, explanation_str = send_gmail.init_gmail()
if not success:
integrationtestlib.log(explanation_str)
sys.exit(0)
print "Beginning query."
success = True
try:
# Query zenodotus
if not _dns_mapping_exists(zenodotus_servername, zenodotus_ipaddr):
print "Zenodotus failed to respond properly!"
# Query is invalid!
success = False
integrationtestlib.notify(
"Error: Zenodotus has failed to correctly respond; the machine has likely been rebooted. Please restart the zenodotus server on zenodotus@blackbox. This report will be re-sent hourly while the problem persists.",
"Cron: Zenodotus failure")
# Check that advertised values work
# Map an entirely random IP to a random DNS name. The mapped IP does
# not have to actually exist (but should still be valid).
random_ip_address = _generate_random_ip_address()
random_publickey = rsa_gen_pubpriv_keys(1024)[0]
random_publickey_string = rsa_publickey_to_string(random_publickey)
random_subdomain = "test-" + sha_hexhash(random_publickey_string)
random_dns_entry = random_subdomain + '.' + zenodotus_servername
print "Announcing", random_dns_entry, random_ip_address
advertise_announce(random_dns_entry, random_ip_address, 60)
if not _dns_mapping_exists(random_dns_entry, random_ip_address):
print "Zenodotus failed to respond properly to advertised subdomain!"
# Query is invalid!
success = False
integrationtestlib.notify(
"Error: Zenodotus has failed to correctly respond to an advertised subdomain; there might be something wrong with the advertise server. This report will be re-sent hourly while the problem persists.",
"Cron: Zenodotus failure")
except Exception, e:
print "Unknown error!"
print str(e)
success = False
integrationtestlib.notify(
"Error: Zenodotus seems to be down! Error data: " + str(e),
"Cron: Zenodotus failure")
def main():
# Initialize the gmail setup.
success, explanation_str = send_gmail.init_gmail()
if not success:
integrationtestlib.log(explanation_str)
sys.exit(0)
print "Beginning query."
success = True
try:
# Query zenodotus
if not _dns_mapping_exists(zenodotus_servername, zenodotus_ipaddr):
print "Zenodotus failed to respond properly!"
# Query is invalid!
success = False
integrationtestlib.notify(
"Error: Zenodotus has failed to correctly respond; the machine has likely been rebooted. Please restart the zenodotus server on zenodotus@blackbox. This report will be re-sent hourly while the problem persists.",
"Cron: Zenodotus failure",
)
# Check that advertised values work
# Map an entirely random IP to a random DNS name. The mapped IP does
# not have to actually exist (but should still be valid).
random_ip_address = _generate_random_ip_address()
random_publickey = rsa_gen_pubpriv_keys(1024)[0]
random_publickey_string = rsa_publickey_to_string(random_publickey)
random_subdomain = "test-" + sha_hexhash(random_publickey_string)
random_dns_entry = random_subdomain + "." + zenodotus_servername
print "Announcing", random_dns_entry, random_ip_address
advertise_announce(random_dns_entry, random_ip_address, 60)
if not _dns_mapping_exists(random_dns_entry, random_ip_address):
print "Zenodotus failed to respond properly to advertised subdomain!"
# Query is invalid!
success = False
integrationtestlib.notify(
"Error: Zenodotus has failed to correctly respond to an advertised subdomain; there might be something wrong with the advertise server. This report will be re-sent hourly while the problem persists.",
"Cron: Zenodotus failure",
)
except Exception, e:
print "Unknown error!"
print str(e)
success = False
integrationtestlib.notify("Error: Zenodotus seems to be down! Error data: " + str(e), "Cron: Zenodotus failure")
def main():
"""
<Purpose>
Call check_nodes with the two different servers: opendht and central. Retrieve the result
and then notify developers if result is unusual
<Exceptions>
None
<Side Effects>
May send out an email or notify on irc.
<Return>
None
"""
#setup the gmail for sending notification
success,explanation_str = send_gmail.init_gmail()
if not success:
integrationtestlib.log(explanation_str)
sys.exit(0)
notification_subject = "test_lookup_node_states() failed"
max_acceptdonation_nodes = 50
max_canonical_nodes = 50
max_movingtotwopercent = 20
min_twopercent_nodes = 300
central_results = check_nodes('central')
integrationtestlib.log("Lookup results for central: "+ str(central_results))
#check to see if any of the results is not normal, and send notifications accordingly
#also send a message to irc.
if central_results['acceptdonation'] > max_acceptdonation_nodes:
message="Too many nodes in acceptdonation state: "+str(central_results['acceptdonation'])+"\nResults from 'central' server:\n"+str(central_results)
print message
integrationtestlib.notify(message, notification_subject)
elif central_results['canonical'] > max_canonical_nodes:
message="Too many nodes in canonical state: "+str(central_results['canonical'])+"\nResults from 'central' server:\n"+str(central_results)
print message
integrationtestlib.notify(message, notification_subject)
elif central_results['twopercent'] < min_twopercent_nodes:
message="Too few nodes in twopercent state: "+str(central_results['twopercent'])+"\nResults from 'central' server:\n"+str(central_results)
print message
integrationtestlib.notify(message, notification_subject)
def main():
"""
<Purpose>
This is the main launching point for the integration tests.
We check both our servers to ensure that both geoip servers
are up and running.
"""
# List of all the geoip servers.
geoip_server_list = ['http://geoipserver.poly.edu:12679', 'http://geoipserver2.poly.edu:12679']
# Keep track of stats.
servers_down = []
servers_running_properly = []
servers_with_bad_response = []
# Initialize the integration test lib.
success,explanation_str = send_gmail.init_gmail()
if not success:
integrationtestlib.log(explanation_str)
sys.exit(0)
# Contact each server to ensure they are running properly.
for server in geoip_server_list:
success, error_type, error_str = test_server(server)
if success:
servers_running_properly.append(server)
else:
if error_type == INCORRECT_ASSERTION:
# If we got the wrong answer, we note what the error was.
servers_with_bad_response.append((server, error_str))
elif error_type == SERVER_DOWN:
servers_down.append((server, error_str))
# Initialize the email text and subject.
email_subject = ""
email_text = ""
# If all the servers are not running properly we send out emails.
if len(servers_running_properly) != len(geoip_server_list):
# If all servers are down, then we are in trouble.
if len(servers_running_properly) == 0:
email_subject = "CRITICAL: All GeoIP servers are down!"
email_text = "CRITICAL: All GeoIP servers are down or are not running properly.\n\n"
# Only some servers are not functioning properly.
else:
email_subject = "WARNING: Some GeoIP servers are down or are not running properly."
email_text = "Not all the GeoIP servers are running properly!\n\n"
email_text += "Servers that are up and running properly:\n"
for server_name in servers_running_properly:
email_text += "\t%s\n" % server_name
# If servers are unresponseive.
if len(servers_down) > 0:
email_text += "Servers that are down:\n"
for (server_name, server_error) in servers_down:
email_text += "\t%s, Error: %s\n" % (server_name, server_error)
# If servers are giving bad response.
if len(servers_with_bad_response) > 0 :
email_text += "Servers with bad response:\n"
for (server_name, server_error) in servers_with_bad_response:
email_text += "\t%s, Error: %s\n" % (server_name, server_error)
# Send out the notification email.
integrationtestlib.notify(email_text, email_subject)
else:
email_text += "Servers that are up and running properly:\n"
for server_name in servers_running_properly:
email_text += "\t%s\n" % server_name
integrationtestlib.log(email_text)
if len(nodes) < 10:
integrationtestlib.log("WARNING: only " + str(len(nodes)) + " forwarders are avaiable")
integrationtestlib.log("WARNING: only " + str(len(total_nodes)) + " forwarders are running")
notify_str += (
"WARNING: test_nat_servers_running.py FAILED, only "
+ str(len(nodes))
+ " nat forwarders are avaiable!, "
+ str(len(total_nodes))
+ " are running."
)
# PART 2 check that nat forwarders are responsive
integrationtestlib.log("Checking that we can talk to a nat forwarder")
try:
response = nat_check_bi_directional(getmyip(), random.randint(20000, 62000))
except Exception, e:
notify_str += "WARNING: could not a get a response from nat forwarders: " + str(e)
integrationtestlib.log("WARNING: could not get a response from nat forwarders " + str(e))
if notify_str != "":
integrationtestlib.notify(notify_str, "nat test fail notice")
integrationtestlib.log("Finished running nat_tests")
print "------------------------------------------------------------"
if __name__ == "__main__":
main()
fail_test_str += "\n\nThe servers we connected to are: " + str(server_list)
notify_str += fail_test_str
integrationtestlib.log(fail_test_str)
integrationtestlib.log("Local time diff: " + str(local_end_time - local_start_time))
# Now do an ntp time test
try:
ntp_time_updatetime(12345)
ntp_t = time_gettime()
except Exception, e:
integrationtestlib.log("Failed to call ntp_time_updatetime(). Returned with exception: " + str(e))
notify_str += "\nFailed to call ntp_time_updatetime(). Returned with exception: " + str(e)
test_stop2 = getruntime()
if len(times) != 0:
diff = ntp_t - max(times)
if diff > (8 + test_stop2 - test_stop1):
exceedby = diff - (test_stop2 - test_stop1)
integrationtestlib.log("WARING large descrepancy between ntp and tcp times")
notify_str += " WARNING large descrepancy between ntp and tcp time: " + str(exceedby)
if notify_str != "":
integrationtestlib.notify(notify_str, "test_time_tcp test failed")
else:
integrationtestlib.log("Finished running test_time_tcp.py..... Test Passed")
print "......................................................\n"
if __name__ == "__main__":
main()
def main():
"""
<Purpose>
Program's main.
<Arguments>
None.
<Exceptions>
All exceptions are caught.
<Side Effects>
None.
<Returns>
None.
"""
# setup the gmail user/password to use when sending email
success, explanation_str = send_gmail.init_gmail()
if not success:
integrationtestlib.log(explanation_str)
sys.exit(0)
# download and install Seattle
download_and_install()
# sleep for a while, giving Clearinghouse time to process this new node
integrationtestlib.log("sleeping for 30 minutes...")
time.sleep(1800)
# retrieve the vesseldict from installed seattle
integrationtestlib.log("retrieving vesseldict from installed Seattle")
dict = {}
try:
f = open(prefix + "/seattle/seattle_repy/vesseldict", "r")
lines = f.readlines()
f.close()
dict = eval(lines[0])
except:
integrationtestlib.handle_exception(
"failed to open/read/eval vesseldict file",
"seattle downloadandinstall failed!")
# uninstall Seattle and remove its dir
uninstall_remove()
sys.exit(0)
# check if the vesseldict conforms to expectations
integrationtestlib.log("checking for twopercent pubkey in vessels..")
passed = False
try:
for vname, vdata in dict.items():
for k in vdata['userkeys']:
if k == twopercent_publickey:
integrationtestlib.log("passed")
passed = True
break
if passed:
break
except e:
integrationtestlib.handle_exception(
"failed in checking for twopercent key\n\nvesseldict is: " +
str(dict), "seattle downloadandinstall failed!")
# uninstall Seattle and remove its dir
uninstall_remove()
sys.exit(0)
# if vesseldict not as expected, notify some people
if not passed:
text = "check for twopercent key:\n" + str(
twopercent_publickey) + "..\n\nfailed\n\nvesseldict is: " + str(
dict)
integrationtestlib.log(text)
integrationtestlib.notify(text, "seattle downloadandinstall failed!")
# uninstall Seattle and remove its dir
uninstall_remove()
return
def main():
"""
<Purpose>
Program's main.
<Arguments>
None.
<Exceptions>
All exceptions are caught.
<Side Effects>
None.
<Returns>
None.
"""
# setup the gmail user/password to use when sending email
success, explanation_str = send_gmail.init_gmail()
if not success:
integrationtestlib.log(explanation_str)
sys.exit(0)
# download and install Seattle
download_and_install()
# sleep for a while, giving GENI time to process this new node
integrationtestlib.log("sleeping for 30 minutes...")
time.sleep(1800)
# retrieve the vesseldict from installed seattle
integrationtestlib.log("retrieving vesseldict from installed Seattle")
dict = {}
try:
f = open(prefix + "/seattle/seattle_repy/vesseldict", "r")
lines = f.readlines()
f.close()
dict = eval(lines[0])
except:
integrationtestlib.handle_exception(
"failed to open/read/eval vesseldict file", "seattle downloadandinstall failed!"
)
# uninstall Seattle and remove its dir
uninstall_remove()
sys.exit(0)
# check if the vesseldict conforms to expectations
integrationtestlib.log("checking for twopercent pubkey in vessels..")
passed = False
try:
for vname, vdata in dict.items():
for k in vdata["userkeys"]:
if k == twopercent_publickey:
integrationtestlib.log("passed")
passed = True
break
if passed:
break
except e:
integrationtestlib.handle_exception(
"failed in checking for twopercent key\n\nvesseldict is: " + str(dict), "seattle downloadandinstall failed!"
)
# uninstall Seattle and remove its dir
uninstall_remove()
sys.exit(0)
# if vesseldict not as expected, notify some people
if not passed:
text = "check for twopercent key:\n" + str(twopercent_publickey) + "..\n\nfailed\n\nvesseldict is: " + str(dict)
integrationtestlib.log(text)
integrationtestlib.notify(text, "seattle downloadandinstall failed!")
# uninstall Seattle and remove its dir
uninstall_remove()
return
try:
ntp_time_updatetime(12345)
ntp_t = time_gettime()
except Exception, e:
integrationtestlib.log(
"Failed to call ntp_time_updatetime(). Returned with exception: " +
str(e))
notify_str += "\nFailed to call ntp_time_updatetime(). Returned with exception: " + str(
e)
test_stop2 = getruntime()
if len(times) != 0:
diff = ntp_t - max(times)
if diff > (8 + test_stop2 - test_stop1):
exceedby = diff - (test_stop2 - test_stop1)
integrationtestlib.log(
'WARING large descrepancy between ntp and tcp times')
notify_str += ' WARNING large descrepancy between ntp and tcp time: ' + str(
exceedby)
if notify_str != '':
integrationtestlib.notify(notify_str, "test_time_tcp test failed")
else:
integrationtestlib.log(
"Finished running test_time_tcp.py..... Test Passed")
print "......................................................\n"
if __name__ == '__main__':
main()
nat_forwarders = []
try:
nat_forwarders = advertise.advertise_lookup(nat_forwarder_key)
except Exception, e:
integrationtestlib.handle_exception("Got exception when looking up nat forwarders", NAT_TEST_FAIL_NOTICE)
return
if len(nat_forwarders) < 10:
notify_str += ('WARNING: only '+ str(len(nat_forwarders))
+ ' nat forwarders are advertising under the key: '
+ repr(nat_forwarder_key) + '\n'
+ "Advertising forwarders: " + str(nat_forwarders) + '\n')
if notify_str:
integrationtestlib.log(notify_str)
integrationtestlib.notify(notify_str, NAT_TEST_FAIL_NOTICE)
# # PART 2 check that nat forwarders are responsive
# TODO: Change this to use Affixes on repyV2
# integrationtestlib.log("Checking that we can talk to a nat forwarder")
# try:
# response = nat_check_bi_directional(getmyip(),random.randint(20000,62000))
# except Exception, e:
# notify_str += 'WARNING: could not a get a response from nat forwarders: '+str(e)
# integrationtestlib.log('WARNING: could not get a response from nat forwarders '+str(e))
integrationtestlib.log("Finished running nat_tests")
print "------------------------------------------------------------"
def main():
"""
<Purpose>
Ping all the machines to see if they are up
<Exceptions>
none
<Side Effects>
Prints the ping result
<Returns>
None.
"""
# setup the gmail user/password to use when sending email
success, explanation_str = send_gmail.init_gmail()
if not success:
integrationtestlib.log(explanation_str)
sys.exit(0)
integrationtestlib.log("pinging critical machines")
#list of machines thats getting pinged
pinglist = []
#list that contains all the ping results
result_queue = []
#create a thread for each machine and ping them
for host in machine_list:
ping_current_machine = ping(str(host), result_queue)
pinglist.append(ping_current_machine)
ping_current_machine.start()
#join all the threads
for ping_host in pinglist:
ping_host.join()
#variable that keeps track if any machines are down
ALL_MACHINES_RUNNING = True
error_message = "WARNING: Seattle machines are down! Seattle developers please check on the machines.\n"
error_message += "Displaying ping result:\n"
#check to see if all the results were successful
#on failures notify the admins and send a message to the irc
for (success, ping_result) in result_queue:
if not success:
ALL_MACHINES_RUNNING = False
error_message += ping_result + "\n"
#if all machines were pinged successfully, notify on irc if option -m was used to run ping_machines.py
if ALL_MACHINES_RUNNING:
if len(sys.argv) >= 2 and sys.argv[1] == '-m':
irc_seattlebot.send_msg("The machines: " + str(machine_list) +
" were pinged successfully")
else:
integrationtestlib.notify(error_message, "Seattle machines down!")
irc_seattlebot.send_msg(error_message)
print time.ctime() + " : Done pinging all machiens."
print "--------------------------------------------"
def _notify_admins(msg): subject = 'SeattleGENI test failed.' integrationtestlib.notify(msg, subject)
def main():
"""
<Purpose>
This is the main launching point for the integration tests.
We check both our servers to ensure that both geoip servers
are up and running.
"""
# List of all the geoip servers.
geoip_server_list = [
'http://geoipserver.poly.edu:12679',
'http://geoipserver2.poly.edu:12679'
]
# Keep track of stats.
servers_down = []
servers_running_properly = []
servers_with_bad_response = []
# Initialize the integration test lib.
success, explanation_str = send_gmail.init_gmail()
if not success:
integrationtestlib.log(explanation_str)
sys.exit(0)
# Contact each server to ensure they are running properly.
for server in geoip_server_list:
success, error_type, error_str = test_server(server)
if success:
servers_running_properly.append(server)
else:
if error_type == INCORRECT_ASSERTION:
# If we got the wrong answer, we note what the error was.
servers_with_bad_response.append((server, error_str))
elif error_type == SERVER_DOWN:
servers_down.append((server, error_str))
# Initialize the email text and subject.
email_subject = ""
email_text = ""
# If all the servers are not running properly we send out emails.
if len(servers_running_properly) != len(geoip_server_list):
# If all servers are down, then we are in trouble.
if len(servers_running_properly) == 0:
email_subject = "CRITICAL: All GeoIP servers are down!"
email_text = "CRITICAL: All GeoIP servers are down or are not running properly.\n\n"
# Only some servers are not functioning properly.
else:
email_subject = "WARNING: Some GeoIP servers are down or are not running properly."
email_text = "Not all the GeoIP servers are running properly!\n\n"
email_text += "Servers that are up and running properly:\n"
for server_name in servers_running_properly:
email_text += "\t%s\n" % server_name
# If servers are unresponseive.
if len(servers_down) > 0:
email_text += "Servers that are down:\n"
for (server_name, server_error) in servers_down:
email_text += "\t%s, Error: %s\n" % (server_name, server_error)
# If servers are giving bad response.
if len(servers_with_bad_response) > 0:
email_text += "Servers with bad response:\n"
for (server_name, server_error) in servers_with_bad_response:
email_text += "\t%s, Error: %s\n" % (server_name, server_error)
# Send out the notification email.
integrationtestlib.notify(email_text, email_subject)
else:
email_text += "Servers that are up and running properly:\n"
for server_name in servers_running_properly:
email_text += "\t%s\n" % server_name
integrationtestlib.log(email_text)
def main():
success,explanation_str = send_gmail.init_gmail()
if not success:
integrationtestlib.log(explanation_str)
sys.exit(0)
updater_pid = None
repy_path = test_directory + os.sep + REPY_RELATIVE_PATH
logfile_name = "softwareupdater.old"
logfile_path = repy_path + logfile_name
# Wrap in a try...except so that the test stops running once we run
# into an error. We also want to unconditionally cleanup the test
# files, so that subsequent tests start fresh.
try:
download_and_unpack_seattle()
# We want to make sure that the installed versions can update, so we
# need to get the URL from the softwareupdater from within the
# installer. Because of this, we have no choice but to delay the
# import until now, otherwise the module would not yet exist.
sys.path.insert(0, REPY_RELATIVE_PATH)
import softwareupdater
metainfo_url = softwareupdater.softwareurl + 'metainfo'
# Retrieve the update metainfo file.
metainfo = urllib2.urlopen(metainfo_url).read()
# The metainfo file may be useful for debugging purposes if
# something goes wrong.
open(test_directory + os.sep + 'metainfo', 'w').write(metainfo)
hashes = get_metainfo_hashes(metainfo)
# I think its safe to assume that each release will always have this
# file...
file_to_modify_name = "nmmain.py"
file_to_modify_path = repy_path + file_to_modify_name
# We don't particularly care how this file is modified, as long as
# its hash changes. It should be okay if we append some data to the
# end of the file.
file_to_modify = open(file_to_modify_path, 'a')
file_to_modify.seek(0, 2)
file_to_modify.write("\n# An update should remove this line\n")
file_to_modify.close()
# The updater will take some time to run...
updater_pid = start_updater_process()
integrationtestlib.log("sleeping for 60 minutes...")
time.sleep(60 * 60)
# We have to assume that the metainfo file didn't change since we
# downloaded it, so this may give us some false negatives if a push
# was done after we retrieved the metainfo file.
if softwareupdater.get_file_hash(file_to_modify_path) != hashes[file_to_modify_name]:
# Maybe the signature expired?
updater_log = open(logfile_path, 'r')
last_line = updater_log.readlines()[-1].strip()
updater_log.close()
if "[do_rsync] Something is wrong with the metainfo: ['Expired signature', 'Expired signature']" in last_line:
# Not sure if we should notify the administrators about this one...
raise Exception("Metainfo signature expired!")
elif "Another software updater old process" in last_line:
raise Exception("Seattle failed to update because another software updater is currently running! Please investigate what is causing the extra updater to run.")
else:
# Something really bad happened...
raise Exception("Seattle failed to restore the modified file to the updated version. If there was a new release of Seattle done in the last hour, then it is possible that this is a false negative.")
# Make sure that we actually got a chance to run, since only one
# instance of the software updater is supposed to be running on each
# machine.
updater_log = open(logfile_path, 'r')
last_line = updater_log.readlines()[-1].strip()
updater_log.close()
except Exception, e:
backup_directory_name = "backup-" + time.strftime("%Y-%m-%d-%H:%M:%S")
text = "The updater integration test failed!\n"
text += "The seattle directory will be backed up to: '"+backup_directory_name+"'\n"
text += "In addition, the metainfo file and the downloaded seattle_linux.tgz will also be in the backup directory.\n"
text += "----------------------------\n"
text += traceback.format_exc()
integrationtestlib.log(text)
integrationtestlib.notify(text, "Seattle Update Test Failed!")
# Since something bad happened, we want to keep the directory
# intact, in case it helps with any debugging.
backup_seattle_directory_path = backup_directory_name + os.sep + "seattle"
os.makedirs(backup_seattle_directory_path)
os.rename("seattle", backup_seattle_directory_path)
os.rename("metainfo", backup_directory_name + os.sep + "metainfo")
os.rename("seattle_linux.tgz", backup_directory_name + os.sep + "seattle_linux.tgz")
# We do some cleaning up after, so update the log path
logfile_path = backup_seattle_directory_path + os.sep + "seattle_repy/" + logfile_name
def main():
"""
<Purpose>
Ping all the machines to see if they are up
<Exceptions>
none
<Side Effects>
Prints the ping result
<Returns>
None.
"""
# setup the gmail user/password to use when sending email
success,explanation_str = send_gmail.init_gmail()
if not success:
integrationtestlib.log(explanation_str)
sys.exit(0)
integrationtestlib.log("pinging critical machines")
#list of machines thats getting pinged
pinglist = []
#list that contains all the ping results
result_queue = []
#create a thread for each machine and ping them
for host in machine_list:
ping_current_machine = ping(str(host), result_queue)
pinglist.append(ping_current_machine)
ping_current_machine.start()
#join all the threads
for ping_host in pinglist:
ping_host.join()
#variable that keeps track if any machines are down
ALL_MACHINES_RUNNING = True
error_message="WARNING: Seattle machines are down! Seattle developers please check on the machines.\n"
error_message+="Displaying ping result:\n"
#check to see if all the results were successful
#on failures notify the admins and send a message to the irc
for (success, ping_result) in result_queue:
if not success:
ALL_MACHINES_RUNNING = False
error_message += ping_result+"\n"
#if all machines were pinged successfully, notify on irc if option -m was used to run ping_machines.py
if ALL_MACHINES_RUNNING:
if len(sys.argv) >= 2 and sys.argv[1] == '-m':
irc_seattlebot.send_msg("The machines: "+str(machine_list)+" were pinged successfully")
else:
integrationtestlib.notify(error_message, "Seattle machines down!")
irc_seattlebot.send_msg(error_message)
print time.ctime() + " : Done pinging all machiens."
print "--------------------------------------------"
except Exception, e:
integrationtestlib.handle_exception(
"Got exception when looking up nat forwarders",
NAT_TEST_FAIL_NOTICE)
return
if len(nat_forwarders) < 10:
notify_str += ('WARNING: only ' + str(len(nat_forwarders)) +
' nat forwarders are advertising under the key: ' +
repr(nat_forwarder_key) + '\n' +
"Advertising forwarders: " + str(nat_forwarders) +
'\n')
if notify_str:
integrationtestlib.log(notify_str)
integrationtestlib.notify(notify_str, NAT_TEST_FAIL_NOTICE)
# # PART 2 check that nat forwarders are responsive
# TODO: Change this to use Affixes on repyV2
# integrationtestlib.log("Checking that we can talk to a nat forwarder")
# try:
# response = nat_check_bi_directional(getmyip(),random.randint(20000,62000))
# except Exception, e:
# notify_str += 'WARNING: could not a get a response from nat forwarders: '+str(e)
# integrationtestlib.log('WARNING: could not get a response from nat forwarders '+str(e))
integrationtestlib.log("Finished running nat_tests")
print "------------------------------------------------------------"
def _notify_admins(msg):
subject = 'SeattleGENI test failed.'
integrationtestlib.notify(msg, subject)