def main():
# initialize the gmail module
success, explanation_str = send_gmail.init_gmail()
if not success:
integrationtestlib.log(explanation_str)
sys.exit(0)
# PART 1 verify that there are at least 10 nat forwarders running on
# each key
notify_str = ''
for nat_forwarder_key in nat_forwarder_keys:
integrationtestlib.log("Looking up nat forwarders for " +
repr(nat_forwarder_key))
nat_forwarders = []
try:
nat_forwarders = advertise.advertise_lookup(nat_forwarder_key)
except Exception, e:
integrationtestlib.handle_exception(
"Got exception when looking up nat forwarders",
NAT_TEST_FAIL_NOTICE)
return
if len(nat_forwarders) < 10:
notify_str += ('WARNING: only ' + str(len(nat_forwarders)) +
' nat forwarders are advertising under the key: ' +
repr(nat_forwarder_key) + '\n' +
"Advertising forwarders: " + str(nat_forwarders) +
'\n')
def main():
# initialize the gmail module
success,explanation_str = send_gmail.init_gmail()
if not success:
integrationtestlib.log(explanation_str)
sys.exit(0)
# PART 1 verify that there are at least 10 nat forwarders running on
# each key
notify_str = ''
for nat_forwarder_key in nat_forwarder_keys:
integrationtestlib.log("Looking up nat forwarders for " + repr(nat_forwarder_key))
nat_forwarders = []
try:
nat_forwarders = advertise.advertise_lookup(nat_forwarder_key)
except Exception, e:
integrationtestlib.handle_exception("Got exception when looking up nat forwarders", NAT_TEST_FAIL_NOTICE)
return
if len(nat_forwarders) < 10:
notify_str += ('WARNING: only '+ str(len(nat_forwarders))
+ ' nat forwarders are advertising under the key: '
+ repr(nat_forwarder_key) + '\n'
+ "Advertising forwarders: " + str(nat_forwarders) + '\n')
def main():
"""
<Purpose>
Program's main.
<Arguments>
None.
<Exceptions>
All exceptions are caught.
<Side Effects>
None.
<Returns>
None.
"""
# setup the gmail user/password to use when sending email
success,explanation_str = send_gmail.init_gmail()
if not success:
integrationtestlib.log(explanation_str)
sys.exit(0)
key = random.randint(4,2**30)
value = random.randint(4,2**30)
ttlval = 60
# put(key,value) with ttlval into the Centralized HT
integrationtestlib.log("calling centralizedadvertise_announce(key: " + str(key) + ", val: " + str(value) + ", ttl: " + str(ttlval) + ")")
try:
centralizedadvertise_announce(key,value,ttlval)
except:
integrationtestlib.handle_exception("centralizedadvertise_announce() failed")
sys.exit(0)
# a 30 second timer to email the notify_list on slow lookups
lookup_timedout_timer = threading.Timer(30, lookup_timedout)
# start the lookup timer
lookup_timedout_timer.start()
# get(key) from the centralized HT
integrationtestlib.log("calling centralizedadvertise_lookup(key: " + str(key) + ")")
try:
ret_value = centralizedadvertise_lookup(key)
print ret_value
#Check if the value being returned is the one we want
if value not in ret_value:
raise Exception("incorrect value returned")
#Check in case random.randint() produces same key again.
elif len(ret_value) > 1 :
raise Exception("Multiple copies of same key")
except:
integrationtestlib.handle_exception("centralizedadvertise_lookup() failed", "centralizedputget monitor script failure")
sys.exit(0)
lookup_timedout_timer.cancel()
lookup_done_event.set()
return
def monitor_processes(monitor_process_list, command_list, machine_name):
"""
<Purpose>
Checks to make sure that the critical processes on the machine 'seattle' are still running
<Exceptions>
None
<Arguments>
monitor_process_list - a list of all the critical processes that should be checked to
see if they are up and running.
command_list - a list of all the commands required to find all the relevant processes
<Return>
None
"""
#string that holds the name of all the processes that are found to be running using the
#ps commands that was passed in as argument
processes_string=""
integrationtestlib.log("Starting monitoring process on "+machine_name)
#run a command on the linux machine to find all the relevant processes
for command in command_list:
try:
relevant_processes, command_error = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE).communicate()
except:
integrationtestlib.handle_exception("Failed to run command: "+command)
sys.exit(1)
#make a string of all the processes
processes_string = processes_string+relevant_processes
print processes_string
#keeps track to see if any processes are down
critical_process_down=False
error_message="WARNING: Critical processes down! Seattle developers please start the processes up as soon as possible\n"
error_message=error_message+"Listing processes that are down:\n"
#goes through the list of monitor_process_list to ensure that all processes are running
for critical_process in monitor_process_list:
integrationtestlib.log("Checking process: "+critical_process+".......")
if not critical_process in processes_string:
critical_process_down=True
error_message = error_message+critical_process+" is down on "+machine_name+".cs.washington.edu\n"
print "FAIL"
else:
print "PASS"
error_message=error_message+"end of list of processes that are down.\n................................"
if critical_process_down:
integrationtestlib.notify(error_message)
irc_seattlebot.send_msg(error_message)
else:
integrationtestlib.log("All critical processes on "+machine_name+" are up and running")
print(".........................................................")
def monitor_processes(monitor_process_list, command_list, machine_name):
"""
<Purpose>
Checks to make sure that the critical processes on the machine 'seattle' are still running
<Exceptions>
None
<Arguments>
monitor_process_list - a list of all the critical processes that should be checked to
see if they are up and running.
command_list - a list of all the commands required to find all the relevant processes
<Return>
None
"""
#string that holds the name of all the processes that are found to be running using the
#ps commands that was passed in as argument
processes_string=""
integrationtestlib.log("Starting monitoring process on "+machine_name)
#run a command on the linux machine to find all the relevant processes
for command in command_list:
try:
relevant_processes, command_error = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE).communicate()
except:
integrationtestlib.handle_exception("Failed to run command: "+command)
sys.exit(1)
#make a string of all the processes
processes_string = processes_string+relevant_processes
print processes_string
#keeps track to see if any processes are down
critical_process_down=False
error_message="WARNING: Critical processes down! Seattle developers please start the processes up as soon as possible\n"
error_message=error_message+"Listing processes that are down:\n"
#goes through the list of monitor_process_list to ensure that all processes are running
for critical_process in monitor_process_list:
integrationtestlib.log("Checking process: "+critical_process+".......")
if not critical_process in processes_string:
critical_process_down=True
error_message = error_message+critical_process+" is down on "+machine_name+".poly.edu\n"
print "FAIL"
else:
print "PASS"
error_message=error_message+"end of list of processes that are down.\n................................"
if critical_process_down:
integrationtestlib.notify(error_message, "Critical process down!")
irc_seattlebot.send_msg(error_message)
else:
integrationtestlib.log("All critical processes on "+machine_name+" are up and running")
print(".........................................................")
def main():
"""
<Purpose>
Program's main.
<Arguments>
None.
<Exceptions>
All exceptions are caught.
<Side Effects>
None.
<Returns>
None.
"""
# setup the gmail user/password to use when sending email
success,explanation_str = send_gmail.init_gmail()
if not success:
integrationtestlib.log(explanation_str)
sys.exit(0)
key = random.randint(4,2**30)
value = random.randint(4,2**30)
ttlval = 60
# put(key,value) with ttlval into OpenDHT
integrationtestlib.log("calling openDHTadvertise.announce(key: " + str(key) + ", val: " + str(value) + ", ttl: " + str(ttlval) + ")")
try:
openDHTadvertise.announce(key,value,ttlval)
except:
integrationtestlib.handle_exception("openDHTadvertise.announce() failed")
sys.exit(0)
# a 30 second timer to email the notify_list on slow lookups
lookup_timedout_timer = threading.Timer(30, lookup_timedout)
# start the lookup timer
lookup_timedout_timer.start()
# get(key) from OpenDHT
integrationtestlib.log("calling openDHTadvertise.lookup(key: " + str(key) + ")")
try:
ret_value = openDHTadvertise.lookup(key)
# TODO: check the return value as well
# ret_value = int(ret_value[0])
except:
integrationtestlib.handle_exception("openDHTadvertise.lookup() failed")
sys.exit(0)
lookup_timedout_timer.cancel()
lookup_done_event.set()
return
def check_opendht_servers():
"""
<Purpose>
Checks to see how many servers are up for the opendht advertisement.
If the number of server is less then 100 then an email notification
is sent to the seattle developers.
<Argument>
None
<Exception>
None
<Side_Effects>
None
<Returns>
None
"""
notify_message = "There aren't enough opendht servers up and running currently."
subject = "opendht_servercount test failed."
try:
# Retrieve the list of servers for opendht
opendht_server_list = openDHTadvertise_get_proxy_list(
maxnumberofattempts=150)
except:
# An exception is raised if there are no servers up and running for opendht by
# openDHTadvertise_get_proxy_list().
integrationtestlib.handle_exception(
"There are no servers up for opendht!", subject)
integrationtestlib.log(
"Retrieved the list of opendht servers up and running.")
integrationtestlib.log(
"There are " + str(len(opendht_server_list)) +
" servers up and running for opendht advertisement.")
# Check to see if there are less then 100 servrs up and running.
# If there are less then 100 servers running then notify the seattle
# developers about it.
if len(opendht_server_list) < min_num_servers:
subject += " There are only " + str(
len(opendht_server_list
)) + " servers up and running for opendht advertisement."
integrationtestlib.notify(notify_message, subject)
def check_opendht_servers():
"""
<Purpose>
Checks to see how many servers are up for the opendht advertisement.
If the number of server is less then 100 then an email notification
is sent to the seattle developers.
<Argument>
None
<Exception>
None
<Side_Effects>
None
<Returns>
None
"""
notify_message = "There aren't enough opendht servers up and running currently."
subject = "opendht_servercount test failed."
try:
# Retrieve the list of servers for opendht
opendht_server_list = openDHTadvertise_get_proxy_list(maxnumberofattempts = 150)
except:
# An exception is raised if there are no servers up and running for opendht by
# openDHTadvertise_get_proxy_list().
integrationtestlib.handle_exception("There are no servers up for opendht!", subject)
integrationtestlib.log("Retrieved the list of opendht servers up and running.")
integrationtestlib.log("There are " + str(len(opendht_server_list)) + " servers up and running for opendht advertisement.")
# Check to see if there are less then 100 servrs up and running.
# If there are less then 100 servers running then notify the seattle
# developers about it.
if len(opendht_server_list) < min_num_servers:
subject += " There are only " + str(len(opendht_server_list)) + " servers up and running for opendht advertisement."
integrationtestlib.notify(notify_message, subject)
def main():
"""
<Purpose>
Program's main.
<Arguments>
None.
<Exceptions>
All exceptions are caught.
<Side Effects>
None.
<Returns>
None.
"""
# setup the gmail user/password to use when sending email
success, explanation_str = send_gmail.init_gmail()
if not success:
integrationtestlib.log(explanation_str)
sys.exit(0)
integrationtestlib.notify_list.append("*****@*****.**")
ttlval = 60
subject = "opendht with repy test failed"
fail_count = 0
message = "openDHTadvertise_lookup() or openDHTadvertise_announce failed 5 times."
for i in range(5):
key = random.randint(4, 2 ** 30)
value = random.randint(4, 2 ** 30)
# put(key,value) with ttlval into OpenDHT
integrationtestlib.log(
"calling openDHTadvertise_announce(key: "
+ str(key)
+ ", val: "
+ str(value)
+ ", ttl: "
+ str(ttlval)
+ ")"
)
try:
openDHTadvertise_announce(key, value, ttlval, proxiestocheck=10)
except:
fail_count += 1
message = (
message + "\nAnouncing with key: " + str(key) + ", value: " + str(value) + ", ttlval: " + str(ttlval)
)
if fail_count >= 5:
integrationtestlib.handle_exception(message, subject)
sys.exit(0)
continue
# a 60 second timer to email the notify_list on slow lookups
lookup_timedout_timer = threading.Timer(60, lookup_timedout)
# start the lookup timer
lookup_timedout_timer.start()
# get(key) from OpenDHT
integrationtestlib.log("calling openDHTadvertise_lookup(key: " + str(key) + ")")
try:
ret_value = openDHTadvertise_lookup(key, proxiestocheck=10)
# TODO: check the return value as well
# ret_value = int(ret_value[0])
except:
fail_count += 1
message = message + "Looking up with key: " + str(key)
if fail_count >= 5:
integrationtestlib.handle_exception(message, subject)
sys.exit(0)
lookup_timedout_timer.cancel()
lookup_done_event.set()
return
def main():
"""
<Purpose>
Program's main.
<Arguments>
None.
<Exceptions>
All exceptions are caught.
<Side Effects>
None.
<Returns>
None.
"""
# setup the gmail user/password to use when sending email
success, explanation_str = send_gmail.init_gmail()
if not success:
integrationtestlib.log(explanation_str)
sys.exit(0)
# download and install Seattle
download_and_install()
# sleep for a while, giving Clearinghouse time to process this new node
integrationtestlib.log("sleeping for 30 minutes...")
time.sleep(1800)
# retrieve the vesseldict from installed seattle
integrationtestlib.log("retrieving vesseldict from installed Seattle")
dict = {}
try:
f = open(prefix + "/seattle/seattle_repy/vesseldict", "r")
lines = f.readlines()
f.close()
dict = eval(lines[0])
except:
integrationtestlib.handle_exception(
"failed to open/read/eval vesseldict file",
"seattle downloadandinstall failed!")
# uninstall Seattle and remove its dir
uninstall_remove()
sys.exit(0)
# check if the vesseldict conforms to expectations
integrationtestlib.log("checking for twopercent pubkey in vessels..")
passed = False
try:
for vname, vdata in dict.items():
for k in vdata['userkeys']:
if k == twopercent_publickey:
integrationtestlib.log("passed")
passed = True
break
if passed:
break
except e:
integrationtestlib.handle_exception(
"failed in checking for twopercent key\n\nvesseldict is: " +
str(dict), "seattle downloadandinstall failed!")
# uninstall Seattle and remove its dir
uninstall_remove()
sys.exit(0)
# if vesseldict not as expected, notify some people
if not passed:
text = "check for twopercent key:\n" + str(
twopercent_publickey) + "..\n\nfailed\n\nvesseldict is: " + str(
dict)
integrationtestlib.log(text)
integrationtestlib.notify(text, "seattle downloadandinstall failed!")
# uninstall Seattle and remove its dir
uninstall_remove()
return
def main():
"""
<Purpose>
Program's main.
<Arguments>
None.
<Exceptions>
All exceptions are caught.
<Side Effects>
None.
<Returns>
None.
"""
# setup the gmail user/password to use when sending email
success, explanation_str = send_gmail.init_gmail()
if not success:
integrationtestlib.log(explanation_str)
sys.exit(0)
# download and install Seattle
download_and_install()
# sleep for a while, giving GENI time to process this new node
integrationtestlib.log("sleeping for 30 minutes...")
time.sleep(1800)
# retrieve the vesseldict from installed seattle
integrationtestlib.log("retrieving vesseldict from installed Seattle")
dict = {}
try:
f = open(prefix + "/seattle/seattle_repy/vesseldict", "r")
lines = f.readlines()
f.close()
dict = eval(lines[0])
except:
integrationtestlib.handle_exception(
"failed to open/read/eval vesseldict file", "seattle downloadandinstall failed!"
)
# uninstall Seattle and remove its dir
uninstall_remove()
sys.exit(0)
# check if the vesseldict conforms to expectations
integrationtestlib.log("checking for twopercent pubkey in vessels..")
passed = False
try:
for vname, vdata in dict.items():
for k in vdata["userkeys"]:
if k == twopercent_publickey:
integrationtestlib.log("passed")
passed = True
break
if passed:
break
except e:
integrationtestlib.handle_exception(
"failed in checking for twopercent key\n\nvesseldict is: " + str(dict), "seattle downloadandinstall failed!"
)
# uninstall Seattle and remove its dir
uninstall_remove()
sys.exit(0)
# if vesseldict not as expected, notify some people
if not passed:
text = "check for twopercent key:\n" + str(twopercent_publickey) + "..\n\nfailed\n\nvesseldict is: " + str(dict)
integrationtestlib.log(text)
integrationtestlib.notify(text, "seattle downloadandinstall failed!")
# uninstall Seattle and remove its dir
uninstall_remove()
return
def main():
"""
<Purpose>
Program's main.
<Arguments>
None.
<Exceptions>
All exceptions are caught.
<Side Effects>
None.
<Returns>
None.
"""
# setup the gmail user/password to use when sending email
success, explanation_str = send_gmail.init_gmail()
if not success:
integrationtestlib.log(explanation_str)
sys.exit(0)
integrationtestlib.notify_list.append("*****@*****.**")
ttlval = 60
subject = "opendht with repy test failed"
fail_count = 0
message = "openDHTadvertise_lookup() or openDHTadvertise_announce failed 5 times."
for i in range(5):
key = random.randint(4, 2**30)
value = random.randint(4, 2**30)
# put(key,value) with ttlval into OpenDHT
integrationtestlib.log("calling openDHTadvertise_announce(key: " +
str(key) + ", val: " + str(value) + ", ttl: " +
str(ttlval) + ")")
try:
openDHTadvertise_announce(key, value, ttlval, proxiestocheck=10)
except:
fail_count += 1
message = message + "\nAnouncing with key: " + str(
key) + ", value: " + str(value) + ", ttlval: " + str(ttlval)
if fail_count >= 5:
integrationtestlib.handle_exception(message, subject)
sys.exit(0)
continue
# a 60 second timer to email the notify_list on slow lookups
lookup_timedout_timer = threading.Timer(60, lookup_timedout)
# start the lookup timer
lookup_timedout_timer.start()
# get(key) from OpenDHT
integrationtestlib.log("calling openDHTadvertise_lookup(key: " +
str(key) + ")")
try:
ret_value = openDHTadvertise_lookup(key, proxiestocheck=10)
# TODO: check the return value as well
# ret_value = int(ret_value[0])
except:
fail_count += 1
message = message + "Looking up with key: " + str(key)
if fail_count >= 5:
integrationtestlib.handle_exception(message, subject)
sys.exit(0)
lookup_timedout_timer.cancel()
lookup_done_event.set()
return
def main():
"""
<Purpose>
Program's main.
<Arguments>
None.
<Exceptions>
All exceptions are caught.
<Side Effects>
None.
<Returns>
None.
"""
# setup the gmail user/password to use when sending email
success,explanation_str = send_gmail.init_gmail()
if not success:
integrationtestlib.log(explanation_str)
sys.exit(0)
integrationtestlib.notify_list.append("*****@*****.**")
key = str(random.randint(4,2**30))
value = str(random.randint(4,2**30))
ttlval = 60
subject = "DOR with repy test failed"
# put(key,value) with ttlval into DOR
integrationtestlib.log("calling DORadvertise_announce(key: " + str(key) + ", val: " + str(value) + ", ttl: " + str(ttlval) + ")")
try:
DORadvertise_announce(key, value, ttlval)
except:
message = "DORadvertise_lookup() failed.\nFailed while doing DORadvertise_announce(). "
message = message + "Anouncing with key: " + key + ", value: " + value + ", ttlval: " + str(ttlval)
integrationtestlib.handle_exception("DORadvertise_announce() failed", subject)
sys.exit(0)
# a 60 second timer to email the notify_list on slow lookups
lookup_timedout_timer = threading.Timer(60, lookup_timedout)
# start the lookup timer
lookup_timedout_timer.start()
# get(key) from DOR
integrationtestlib.log("calling DORadvertise_lookup(key: " + str(key) + ")")
try:
ret_value = DORadvertise_lookup(key)
# TODO: check the return value as well
# ret_value = int(ret_value[0])
except:
message = "DORadvertise_lookup() failed.\nFailed while doing DORadvertise_lookup(). "
message = message + "Looking up with key: " + key
integrationtestlib.handle_exception(message, subject)
sys.exit(0)
lookup_timedout_timer.cancel()
lookup_done_event.set()
return
def main():
"""
<Purpose>
Program's main.
<Arguments>
None.
<Exceptions>
All exceptions are caught.
<Side Effects>
None.
<Returns>
None.
"""
# setup the gmail user/password to use when sending email
success, explanation_str = send_gmail.init_gmail()
if not success:
integrationtestlib.log(explanation_str)
sys.exit(0)
integrationtestlib.notify_list.append("*****@*****.**")
key = str(random.randint(4, 2**30))
value = str(random.randint(4, 2**30))
ttlval = 60
subject = "DOR with repy test failed"
# put(key,value) with ttlval into DOR
integrationtestlib.log("calling DORadvertise_announce(key: " + str(key) +
", val: " + str(value) + ", ttl: " + str(ttlval) +
")")
try:
DORadvertise_announce(key, value, ttlval)
except:
message = "DORadvertise_lookup() failed.\nFailed while doing DORadvertise_announce(). "
message = message + "Anouncing with key: " + key + ", value: " + value + ", ttlval: " + str(
ttlval)
integrationtestlib.handle_exception("DORadvertise_announce() failed",
subject)
sys.exit(0)
# a 60 second timer to email the notify_list on slow lookups
lookup_timedout_timer = threading.Timer(60, lookup_timedout)
# start the lookup timer
lookup_timedout_timer.start()
# get(key) from DOR
integrationtestlib.log("calling DORadvertise_lookup(key: " + str(key) +
")")
try:
ret_value = DORadvertise_lookup(key)
# TODO: check the return value as well
# ret_value = int(ret_value[0])
except:
message = "DORadvertise_lookup() failed.\nFailed while doing DORadvertise_lookup(). "
message = message + "Looking up with key: " + key
integrationtestlib.handle_exception(message, subject)
sys.exit(0)
lookup_timedout_timer.cancel()
lookup_done_event.set()
return
