def run_on_vessels(self, vessel_handlers, filename, *vessel_args):
"""
<Purpose>
Runs a program on a set of vessels. A batch wrapper around the Experiment
Library function run_parallelized, with logging and parallelization support.
<Arguments>
vesselhandle_list
A list of vesselhandles of vessels to which a file is to be uploaded.
filename
The filename of the program to run.
*vessel_args
Optional additional arguments required by the program to be run on
vessels.
<Exceptions>
None
<Side Effects>
Logger will make an entry for each successful and failed start-ups
Process is parallelized
<Returns>
A list of vesselhandles of vessels that started the program successfully
"""
self.logger.info("Starting '" + filename + "' on " +
str(len(vessel_handlers)) + " vessels")
# Clear the list of successful_handlers for a new operation
self.successful_handlers = []
explib.run_parallelized(vessel_handlers, self._run_on_vessels,
filename, *vessel_args)
return self.successful_handlers
def run_on_vessels(self, vessel_handlers, filename, *vessel_args):
"""
<Purpose>
Runs a program on a set of vessels. A batch wrapper around the Experiment
Library function run_parallelized, with logging and parallelization support.
<Arguments>
vesselhandle_list
A list of vesselhandles of vessels to which a file is to be uploaded.
filename
The filename of the program to run.
*vessel_args
Optional additional arguments required by the program to be run on
vessels.
<Exceptions>
None
<Side Effects>
Logger will make an entry for each successful and failed start-ups
Process is parallelized
<Returns>
A list of vesselhandles of vessels that started the program successfully
"""
self.logger.info("Starting '" + filename + "' on " + str(len(vessel_handlers)) + " vessels")
# Clear the list of successful_handlers for a new operation
self.successful_handlers = []
explib.run_parallelized(vessel_handlers, self._run_on_vessels, filename, *vessel_args)
return self.successful_handlers
def upload_to_vessels(self, vessel_handlers, filename_list):
"""
<Purpose>
Uploads a list of file to a set of vessels. A batch wrapper around the Experiment
Library function upload_file_to_vessel, with logging and parallelization support.
<Arguments>
vesselhandle_list
A list of vesselhandles of vessels to which the file is to be uploaded.
filename
The filename of the file to be uploaded.
<Exceptions>
None
<Side Effects>
None
<Returns>
A list of vessels to which the upload succeeded.
"""
self.logger.info("Uploading '" + str(filename_list) + "' to " +
str(len(vessel_handlers)) + " vessels")
# Clear the list of successful_handlers for a new operation
self.successful_handlers = []
explib.run_parallelized(vessel_handlers,
self._upload_to_vessels_helper, filename_list)
return self.successful_handlers
def upload_to_vessels(self, vessel_handlers, filename_list):
"""
<Purpose>
Uploads a list of file to a set of vessels. A batch wrapper around the Experiment
Library function upload_file_to_vessel, with logging and parallelization support.
<Arguments>
vesselhandle_list
A list of vesselhandles of vessels to which the file is to be uploaded.
filename
The filename of the file to be uploaded.
<Exceptions>
None
<Side Effects>
None
<Returns>
A list of vessels to which the upload succeeded.
"""
self.logger.info("Uploading '" + str(filename_list) + "' to " + str(len(vessel_handlers)) + " vessels")
# Clear the list of successful_handlers for a new operation
self.successful_handlers = []
explib.run_parallelized(vessel_handlers, self._upload_to_vessels_helper, filename_list)
return self.successful_handlers
def _run_monitor(monitordict): """Performs the actual monitoring of vessels.""" # Copy the vesselhandle_list so that changes made to the list can't happen # while the parallelized call is being done. vesselhandle_list = monitordict['vesselhandle_list'][:] experimentlib.run_parallelized(vesselhandle_list, _check_vessel_status_change, monitordict) # We finished the last run, now schedule another. monitordict['timerhandle'] = settimer(monitordict['waittime'], _run_monitor, [monitordict])
def _run_monitor(monitordict):
"""Performs the actual monitoring of vessels."""
# Copy the vesselhandle_list so that changes made to the list can't happen
# while the parallelized call is being done.
vesselhandle_list = monitordict['vesselhandle_list'][:]
experimentlib.run_parallelized(vesselhandle_list,
_check_vessel_status_change, monitordict)
# We finished the last run, now schedule another.
monitordict['timerhandle'] = settimer(monitordict['waittime'],
_run_monitor, [monitordict])
def main():
identity = experimentlib.create_identity_from_key_files(
SEATTLECLEARINGHOUSE_PUBLICKEY_FILENAME)
nodelocation_list = experimentlib.lookup_node_locations_by_identity(
identity)
print("Number of advertising nodes: " + str(len(nodelocation_list)))
if MAX_NODES_TO_LOOK_AT is not None:
print("DEBUG: only looking at " + str(MAX_NODES_TO_LOOK_AT) +
" nodes.")
nodelocation_list = nodelocation_list[:MAX_NODES_TO_LOOK_AT]
# Talk to each nodemanager to find out vessel information.
browse_successlist, failurelist = \
experimentlib.run_parallelized(nodelocation_list, browse_node_for_available_vessels)
# Create a dictionary whose keys are the nodeids and values are lists of
# vesseldicts of the available vessels on that node.
available_vesseldicts_by_node = {}
for (nodeid, available_vesseldicts) in browse_successlist:
if available_vesseldicts:
available_vesseldicts_by_node[nodeid] = available_vesseldicts
print("Number of nodes that SeattleGENI vessels are available on: " +
str(len(available_vesseldicts_by_node.keys())))
def main():
identity = experimentlib.create_identity_from_key_files(PUBLICKEY_FILENAME, PRIVATEKEY_FILENAME)
nodelocation_list = experimentlib.lookup_node_locations_by_identity(identity)
print("Number of advertising nodes: " + str(len(nodelocation_list)))
browse_successlist, browse_failurelist = experimentlib.run_parallelized(nodelocation_list,
experimentlib.browse_node, identity)
vesseldict_list = []
for (nodeid, vesseldicts_of_node) in browse_successlist:
vesseldict_list += vesseldicts_of_node
print("Good vessels: " + str(len(vesseldict_list)))
set_keys_successlist, set_keys_failure_list = experimentlib.run_parallelized(vesseldict_list, set_keys, identity)
print("Vessels with proper user keys: " + str(len(set_keys_successlist)))
print("Vessels that failed user key setting: " + str(len(set_keys_failure_list)))
print("Done.")
def main():
identity = experimentlib.create_identity_from_key_files(SEATTLEGENI_PUBLICKEY_FILENAME)
nodelocation_list = experimentlib.lookup_node_locations_by_identity(identity)
print("Number of advertising nodes: " + str(len(nodelocation_list)))
if MAX_NODES_TO_LOOK_AT is not None:
print("DEBUG: only looking at " + str(MAX_NODES_TO_LOOK_AT) + " nodes.")
nodelocation_list = nodelocation_list[:MAX_NODES_TO_LOOK_AT]
# Talk to each nodemanager to find out vessel information.
browse_successlist, failurelist = \
experimentlib.run_parallelized(nodelocation_list, browse_node_for_available_vessels)
# Create a dictionary whose keys are the nodeids and values are lists of
# vesseldicts of the available vessels on that node.
available_vesseldicts_by_node = {}
for (nodeid, available_vesseldicts) in browse_successlist:
if available_vesseldicts:
available_vesseldicts_by_node[nodeid] = available_vesseldicts
print("Number of nodes that SeattleGENI vessels are available on: " +
str(len(available_vesseldicts_by_node.keys())))
def check_nodes(identitydict, nodelogfile, vessellogfile):
"""
<Purpose>
Determines which nodes are advertising under the keys in identitydict and
contacts any nodes that need to be contacted.
<Arguments>
identitydict:
A dictionary where the keys are public key names and the value is an
Experiment Library identity for that key.
nodelogfile:
An open file for logging new advertising and contactability events.
vessellogfile:
An open file for logging the node dictionaries returned by browse_node.
<Exceptions>
None.
<Side Effects>
Writes to nodelogfile and vessellogfile.
<Returns>
None.
"""
currenttime = time.time()
# Currently advertising node locations.
advertisingset = set()
# Locations that started advertising under some key.
addedlocationset = set()
# Locations that stopped advertising under some key.
droppedlocationset = set()
for keyname in identitydict:
# Locations that were advertising last time we checked this publickey.
oldlocationlist = oldlocationdict[keyname]
# Find all the node locations advertising under this key. If we don't expect
# there to be many nodes advertising, then don't bother retrying many times.
locationlist = advertising_lookup(identitydict[keyname],
min(len(oldlocationlist), 10))
if locationlist is None:
print currenttime, "ERROR: advertising lookup failed for", keyname
continue
oldlocationdict[keyname] = locationlist
advertisingset = advertisingset.union(locationlist)
# Find any changes in the list of advertising nodes.
startedadvertisingset = set(locationlist).difference(oldlocationlist)
stoppedadvertisingset = set(oldlocationlist).difference(locationlist)
addedlocationset = addedlocationset.union(startedadvertisingset)
droppedlocationset = droppedlocationset.union(stoppedadvertisingset)
for nodelocation in startedadvertisingset:
nodelogfile.write(
str(currenttime) + " " + nodelocation + " advertising " +
keyname + "\n")
for nodelocation in stoppedadvertisingset:
nodelogfile.write(
str(currenttime) + " " + nodelocation + " not_advertising " +
keyname + "\n")
for location in addedlocationset:
# Contact this node immediately since it started advertising.
nextlookupdict[location] = 0
for location in droppedlocationset:
if location in contactabledict or location in advertisingset:
# Contact this node immediately since it stopped advertising.
nextlookupdict[location] = 0
else:
# This node isn't advertising or contactable, so there's no real reason
# to contact it until it starts advertising again.
if location in nextlookupdict:
del nextlookupdict[location]
else:
print currenttime, "ERROR:", location, \
"not in nextlookupdict when it stopped advertising"
# The list of node locations to try to contact at this time.
currentlookuplist = []
for location in nextlookupdict:
if nextlookupdict[location] < currenttime:
currentlookuplist.append(location)
# Contact all list of node we're currently interested in. This lookup is
# parallelized to improve speed. Especially for the mass lookup that occurs
# when the script starts, increasing experimentlib.num_worker_threads to a
# larger number will further increase the speed of this lookup.
successlist, failurelist = experimentlib.run_parallelized(
currentlookuplist, browse_node)
# These are the nodes for which contacting them failed
for nodelocation, errorstring in failurelist:
if nodelocation in contactabledict:
nodeid = contactabledict.pop(nodelocation)
nodelogfile.write(
str(currenttime) + " " + nodelocation + " not_contactable " +
nodeid + "\n")
if nodelocation in advertisingset:
# Node is advertising but not contactable, so try again later.
nextlookupdict[nodelocation] = currenttime + \
(random.random() + 0.5) * ONLY_ADVERTISING_POLLING_INTERVAL
else:
# This node isn't advertising or contactable, so there's no real reason
# to contact it until it starts advertising again.
if nodelocation in nextlookupdict:
del nextlookupdict[nodelocation]
else:
print currenttime, "ERROR:", nodelocation, \
"not in nextlookupdict when it stopped being contactable"
# These are the nodes we successfully contacted.
for nodelocation, nodedict in successlist:
# Log the retrieved node information.
vessellogfile.write(str(nodedict) + "\n")
nodeid = nodedict['nodeid']
if nodelocation in contactabledict:
# The node changed nodeid while still advertising. This probably won't
# ever happen, but we might as well check. (Note: apparently this does
# happen every once in a while.)
if nodeid != contactabledict[nodelocation]:
nodelogfile.write(
str(currenttime) + " " + nodelocation +
" not_contactable " + contactabledict[nodelocation] + "\n")
del contactabledict[nodelocation]
if nodelocation not in contactabledict:
contactabledict[nodelocation] = nodeid
nodelogfile.write(
str(currenttime) + " " + nodelocation + " contactable " +
nodeid + "\n")
# Pick a semi-random time to do the next lookup so we aren't contacting all
# the nodes at once.
if nodelocation in advertisingset:
nextlookupdict[nodelocation] = currenttime + \
(random.random() + 0.5) * CONSISTENT_POLLING_INTERVAL
else:
nextlookupdict[nodelocation] = currenttime + \
(random.random() + 0.5) * ONLY_CONTACTABLE_POLLING_INTERVAL
def check_nodes(identitydict, nodelogfile, vessellogfile):
"""
<Purpose>
Determines which nodes are advertising under the keys in identitydict and
contacts any nodes that need to be contacted.
<Arguments>
identitydict:
A dictionary where the keys are public key names and the value is an
Experiment Library identity for that key.
nodelogfile:
An open file for logging new advertising and contactability events.
vessellogfile:
An open file for logging the node dictionaries returned by browse_node.
<Exceptions>
None.
<Side Effects>
Writes to nodelogfile and vessellogfile.
<Returns>
None.
"""
currenttime = time.time()
# Currently advertising node locations.
advertisingset = set()
# Locations that started advertising under some key.
addedlocationset = set()
# Locations that stopped advertising under some key.
droppedlocationset = set()
for keyname in identitydict:
# Locations that were advertising last time we checked this publickey.
oldlocationlist = oldlocationdict[keyname]
# Find all the node locations advertising under this key. If we don't expect
# there to be many nodes advertising, then don't bother retrying many times.
locationlist = advertising_lookup(identitydict[keyname],
min(len(oldlocationlist), 10))
if locationlist is None:
print currenttime, "ERROR: advertising lookup failed for", keyname
continue
oldlocationdict[keyname] = locationlist
advertisingset = advertisingset.union(locationlist)
# Find any changes in the list of advertising nodes.
startedadvertisingset = set(locationlist).difference(oldlocationlist)
stoppedadvertisingset = set(oldlocationlist).difference(locationlist)
addedlocationset = addedlocationset.union(startedadvertisingset)
droppedlocationset = droppedlocationset.union(stoppedadvertisingset)
for nodelocation in startedadvertisingset:
nodelogfile.write(str(currenttime) + " " + nodelocation +
" advertising " + keyname + "\n")
for nodelocation in stoppedadvertisingset:
nodelogfile.write(str(currenttime) + " " + nodelocation +
" not_advertising " + keyname + "\n")
for location in addedlocationset:
# Contact this node immediately since it started advertising.
nextlookupdict[location] = 0
for location in droppedlocationset:
if location in contactabledict or location in advertisingset:
# Contact this node immediately since it stopped advertising.
nextlookupdict[location] = 0
else:
# This node isn't advertising or contactable, so there's no real reason
# to contact it until it starts advertising again.
if location in nextlookupdict:
del nextlookupdict[location]
else:
print currenttime, "ERROR:", location, \
"not in nextlookupdict when it stopped advertising"
# The list of node locations to try to contact at this time.
currentlookuplist = []
for location in nextlookupdict:
if nextlookupdict[location] < currenttime:
currentlookuplist.append(location)
# Contact all list of node we're currently interested in. This lookup is
# parallelized to improve speed. Especially for the mass lookup that occurs
# when the script starts, increasing experimentlib.num_worker_threads to a
# larger number will further increase the speed of this lookup.
successlist, failurelist = experimentlib.run_parallelized(
currentlookuplist, browse_node)
# These are the nodes for which contacting them failed
for nodelocation, errorstring in failurelist:
if nodelocation in contactabledict:
nodeid = contactabledict.pop(nodelocation)
nodelogfile.write(str(currenttime) + " " + nodelocation +
" not_contactable " + nodeid + "\n")
if nodelocation in advertisingset:
# Node is advertising but not contactable, so try again later.
nextlookupdict[nodelocation] = currenttime + \
(random.random() + 0.5) * ONLY_ADVERTISING_POLLING_INTERVAL
else:
# This node isn't advertising or contactable, so there's no real reason
# to contact it until it starts advertising again.
if nodelocation in nextlookupdict:
del nextlookupdict[nodelocation]
else:
print currenttime, "ERROR:", nodelocation, \
"not in nextlookupdict when it stopped being contactable"
# These are the nodes we successfully contacted.
for nodelocation, nodedict in successlist:
# Log the retrieved node information.
vessellogfile.write(str(nodedict) + "\n")
nodeid = nodedict['nodeid']
if nodelocation in contactabledict:
# The node changed nodeid while still advertising. This probably won't
# ever happen, but we might as well check. (Note: apparently this does
# happen every once in a while.)
if nodeid != contactabledict[nodelocation]:
nodelogfile.write(str(currenttime) + " " + nodelocation +
" not_contactable " + contactabledict[nodelocation] + "\n")
del contactabledict[nodelocation]
if nodelocation not in contactabledict:
contactabledict[nodelocation] = nodeid
nodelogfile.write(str(currenttime) + " " + nodelocation +
" contactable " + nodeid + "\n")
# Pick a semi-random time to do the next lookup so we aren't contacting all
# the nodes at once.
if nodelocation in advertisingset:
nextlookupdict[nodelocation] = currenttime + \
(random.random() + 0.5) * CONSISTENT_POLLING_INTERVAL
else:
nextlookupdict[nodelocation] = currenttime + \
(random.random() + 0.5) * ONLY_CONTACTABLE_POLLING_INTERVAL
for vessel in vessel_details:
nodelocation_list.append(vessel["nodelocation"])
print("Number of vessels acquired: " + str(len(CBFUNC_CONTEXT['vesselhandle_list'])))
# Write node IPs to file to be uploaded
neighborip_fo = open("neighboriplist.txt", "w")
for ipport in nodelocation_list:
ip = ipport.split(":")[0]
NODEIPLIST.append(ip)
neighborip_fo.write(ip + "\n")
neighborip_fo.close()
# Using experimentlib.run_parallelized() to upload and run programs
uploadips_success, uploadips_fail = experimentlib.run_parallelized(CBFUNC_CONTEXT['vesselhandle_list'], experimentlib.upload_file_to_vessel, CBFUNC_CONTEXT['identity'], "neighboriplist.txt")
uploadprog_success, uploadprog_fail = experimentlib.run_parallelized(CBFUNC_CONTEXT['vesselhandle_list'], experimentlib.upload_file_to_vessel, CBFUNC_CONTEXT['identity'], "pingneighbors.py")
start_success, start_fail = experimentlib.run_parallelized(CBFUNC_CONTEXT['vesselhandle_list'], experimentlib.start_vessel, CBFUNC_CONTEXT['identity'], "pingneighbors.py", [str(CBFUNC_CONTEXT['geniport'])])
# Handle failed nodes
for failed in listops_uniq(uploadips_fail + uploadprog_fail + start_fail):
nodeid, vesselname = experimentlib.get_nodeid_and_vesselname(failed[0])
for vessel in vessel_details:
if vessel["nodeid"] == nodeid:
NODEIPLIST.remove(vessel["nodelocation"][:-5])
print "Failure on vessel " + failed[0] + ". Error was: " + failed[1]
print "Files uploaded to nodes and programs running"
# Now that vessel setup is complete, handle output
geoip_init_client("http://geoip.cs.washington.edu:12679")
