def run_on_vessels(self, vessel_handlers, filename, *vessel_args): """ <Purpose> Runs a program on a set of vessels. A batch wrapper around the Experiment Library function run_parallelized, with logging and parallelization support. <Arguments> vesselhandle_list A list of vesselhandles of vessels to which a file is to be uploaded. filename The filename of the program to run. *vessel_args Optional additional arguments required by the program to be run on vessels. <Exceptions> None <Side Effects> Logger will make an entry for each successful and failed start-ups Process is parallelized <Returns> A list of vesselhandles of vessels that started the program successfully """ self.logger.info("Starting '" + filename + "' on " + str(len(vessel_handlers)) + " vessels") # Clear the list of successful_handlers for a new operation self.successful_handlers = [] explib.run_parallelized(vessel_handlers, self._run_on_vessels, filename, *vessel_args) return self.successful_handlers
def upload_to_vessels(self, vessel_handlers, filename_list): """ <Purpose> Uploads a list of file to a set of vessels. A batch wrapper around the Experiment Library function upload_file_to_vessel, with logging and parallelization support. <Arguments> vesselhandle_list A list of vesselhandles of vessels to which the file is to be uploaded. filename The filename of the file to be uploaded. <Exceptions> None <Side Effects> None <Returns> A list of vessels to which the upload succeeded. """ self.logger.info("Uploading '" + str(filename_list) + "' to " + str(len(vessel_handlers)) + " vessels") # Clear the list of successful_handlers for a new operation self.successful_handlers = [] explib.run_parallelized(vessel_handlers, self._upload_to_vessels_helper, filename_list) return self.successful_handlers
def _run_monitor(monitordict): """Performs the actual monitoring of vessels.""" # Copy the vesselhandle_list so that changes made to the list can't happen # while the parallelized call is being done. vesselhandle_list = monitordict['vesselhandle_list'][:] experimentlib.run_parallelized(vesselhandle_list, _check_vessel_status_change, monitordict) # We finished the last run, now schedule another. monitordict['timerhandle'] = settimer(monitordict['waittime'], _run_monitor, [monitordict])
def main(): identity = experimentlib.create_identity_from_key_files( SEATTLECLEARINGHOUSE_PUBLICKEY_FILENAME) nodelocation_list = experimentlib.lookup_node_locations_by_identity( identity) print("Number of advertising nodes: " + str(len(nodelocation_list))) if MAX_NODES_TO_LOOK_AT is not None: print("DEBUG: only looking at " + str(MAX_NODES_TO_LOOK_AT) + " nodes.") nodelocation_list = nodelocation_list[:MAX_NODES_TO_LOOK_AT] # Talk to each nodemanager to find out vessel information. browse_successlist, failurelist = \ experimentlib.run_parallelized(nodelocation_list, browse_node_for_available_vessels) # Create a dictionary whose keys are the nodeids and values are lists of # vesseldicts of the available vessels on that node. available_vesseldicts_by_node = {} for (nodeid, available_vesseldicts) in browse_successlist: if available_vesseldicts: available_vesseldicts_by_node[nodeid] = available_vesseldicts print("Number of nodes that SeattleGENI vessels are available on: " + str(len(available_vesseldicts_by_node.keys())))
def main(): identity = experimentlib.create_identity_from_key_files(PUBLICKEY_FILENAME, PRIVATEKEY_FILENAME) nodelocation_list = experimentlib.lookup_node_locations_by_identity(identity) print("Number of advertising nodes: " + str(len(nodelocation_list))) browse_successlist, browse_failurelist = experimentlib.run_parallelized(nodelocation_list, experimentlib.browse_node, identity) vesseldict_list = [] for (nodeid, vesseldicts_of_node) in browse_successlist: vesseldict_list += vesseldicts_of_node print("Good vessels: " + str(len(vesseldict_list))) set_keys_successlist, set_keys_failure_list = experimentlib.run_parallelized(vesseldict_list, set_keys, identity) print("Vessels with proper user keys: " + str(len(set_keys_successlist))) print("Vessels that failed user key setting: " + str(len(set_keys_failure_list))) print("Done.")
def main(): identity = experimentlib.create_identity_from_key_files(SEATTLEGENI_PUBLICKEY_FILENAME) nodelocation_list = experimentlib.lookup_node_locations_by_identity(identity) print("Number of advertising nodes: " + str(len(nodelocation_list))) if MAX_NODES_TO_LOOK_AT is not None: print("DEBUG: only looking at " + str(MAX_NODES_TO_LOOK_AT) + " nodes.") nodelocation_list = nodelocation_list[:MAX_NODES_TO_LOOK_AT] # Talk to each nodemanager to find out vessel information. browse_successlist, failurelist = \ experimentlib.run_parallelized(nodelocation_list, browse_node_for_available_vessels) # Create a dictionary whose keys are the nodeids and values are lists of # vesseldicts of the available vessels on that node. available_vesseldicts_by_node = {} for (nodeid, available_vesseldicts) in browse_successlist: if available_vesseldicts: available_vesseldicts_by_node[nodeid] = available_vesseldicts print("Number of nodes that SeattleGENI vessels are available on: " + str(len(available_vesseldicts_by_node.keys())))
def check_nodes(identitydict, nodelogfile, vessellogfile): """ <Purpose> Determines which nodes are advertising under the keys in identitydict and contacts any nodes that need to be contacted. <Arguments> identitydict: A dictionary where the keys are public key names and the value is an Experiment Library identity for that key. nodelogfile: An open file for logging new advertising and contactability events. vessellogfile: An open file for logging the node dictionaries returned by browse_node. <Exceptions> None. <Side Effects> Writes to nodelogfile and vessellogfile. <Returns> None. """ currenttime = time.time() # Currently advertising node locations. advertisingset = set() # Locations that started advertising under some key. addedlocationset = set() # Locations that stopped advertising under some key. droppedlocationset = set() for keyname in identitydict: # Locations that were advertising last time we checked this publickey. oldlocationlist = oldlocationdict[keyname] # Find all the node locations advertising under this key. If we don't expect # there to be many nodes advertising, then don't bother retrying many times. locationlist = advertising_lookup(identitydict[keyname], min(len(oldlocationlist), 10)) if locationlist is None: print currenttime, "ERROR: advertising lookup failed for", keyname continue oldlocationdict[keyname] = locationlist advertisingset = advertisingset.union(locationlist) # Find any changes in the list of advertising nodes. startedadvertisingset = set(locationlist).difference(oldlocationlist) stoppedadvertisingset = set(oldlocationlist).difference(locationlist) addedlocationset = addedlocationset.union(startedadvertisingset) droppedlocationset = droppedlocationset.union(stoppedadvertisingset) for nodelocation in startedadvertisingset: nodelogfile.write( str(currenttime) + " " + nodelocation + " advertising " + keyname + "\n") for nodelocation in stoppedadvertisingset: nodelogfile.write( str(currenttime) + " " + nodelocation + " not_advertising " + keyname + "\n") for location in addedlocationset: # Contact this node immediately since it started advertising. nextlookupdict[location] = 0 for location in droppedlocationset: if location in contactabledict or location in advertisingset: # Contact this node immediately since it stopped advertising. nextlookupdict[location] = 0 else: # This node isn't advertising or contactable, so there's no real reason # to contact it until it starts advertising again. if location in nextlookupdict: del nextlookupdict[location] else: print currenttime, "ERROR:", location, \ "not in nextlookupdict when it stopped advertising" # The list of node locations to try to contact at this time. currentlookuplist = [] for location in nextlookupdict: if nextlookupdict[location] < currenttime: currentlookuplist.append(location) # Contact all list of node we're currently interested in. This lookup is # parallelized to improve speed. Especially for the mass lookup that occurs # when the script starts, increasing experimentlib.num_worker_threads to a # larger number will further increase the speed of this lookup. successlist, failurelist = experimentlib.run_parallelized( currentlookuplist, browse_node) # These are the nodes for which contacting them failed for nodelocation, errorstring in failurelist: if nodelocation in contactabledict: nodeid = contactabledict.pop(nodelocation) nodelogfile.write( str(currenttime) + " " + nodelocation + " not_contactable " + nodeid + "\n") if nodelocation in advertisingset: # Node is advertising but not contactable, so try again later. nextlookupdict[nodelocation] = currenttime + \ (random.random() + 0.5) * ONLY_ADVERTISING_POLLING_INTERVAL else: # This node isn't advertising or contactable, so there's no real reason # to contact it until it starts advertising again. if nodelocation in nextlookupdict: del nextlookupdict[nodelocation] else: print currenttime, "ERROR:", nodelocation, \ "not in nextlookupdict when it stopped being contactable" # These are the nodes we successfully contacted. for nodelocation, nodedict in successlist: # Log the retrieved node information. vessellogfile.write(str(nodedict) + "\n") nodeid = nodedict['nodeid'] if nodelocation in contactabledict: # The node changed nodeid while still advertising. This probably won't # ever happen, but we might as well check. (Note: apparently this does # happen every once in a while.) if nodeid != contactabledict[nodelocation]: nodelogfile.write( str(currenttime) + " " + nodelocation + " not_contactable " + contactabledict[nodelocation] + "\n") del contactabledict[nodelocation] if nodelocation not in contactabledict: contactabledict[nodelocation] = nodeid nodelogfile.write( str(currenttime) + " " + nodelocation + " contactable " + nodeid + "\n") # Pick a semi-random time to do the next lookup so we aren't contacting all # the nodes at once. if nodelocation in advertisingset: nextlookupdict[nodelocation] = currenttime + \ (random.random() + 0.5) * CONSISTENT_POLLING_INTERVAL else: nextlookupdict[nodelocation] = currenttime + \ (random.random() + 0.5) * ONLY_CONTACTABLE_POLLING_INTERVAL
def check_nodes(identitydict, nodelogfile, vessellogfile): """ <Purpose> Determines which nodes are advertising under the keys in identitydict and contacts any nodes that need to be contacted. <Arguments> identitydict: A dictionary where the keys are public key names and the value is an Experiment Library identity for that key. nodelogfile: An open file for logging new advertising and contactability events. vessellogfile: An open file for logging the node dictionaries returned by browse_node. <Exceptions> None. <Side Effects> Writes to nodelogfile and vessellogfile. <Returns> None. """ currenttime = time.time() # Currently advertising node locations. advertisingset = set() # Locations that started advertising under some key. addedlocationset = set() # Locations that stopped advertising under some key. droppedlocationset = set() for keyname in identitydict: # Locations that were advertising last time we checked this publickey. oldlocationlist = oldlocationdict[keyname] # Find all the node locations advertising under this key. If we don't expect # there to be many nodes advertising, then don't bother retrying many times. locationlist = advertising_lookup(identitydict[keyname], min(len(oldlocationlist), 10)) if locationlist is None: print currenttime, "ERROR: advertising lookup failed for", keyname continue oldlocationdict[keyname] = locationlist advertisingset = advertisingset.union(locationlist) # Find any changes in the list of advertising nodes. startedadvertisingset = set(locationlist).difference(oldlocationlist) stoppedadvertisingset = set(oldlocationlist).difference(locationlist) addedlocationset = addedlocationset.union(startedadvertisingset) droppedlocationset = droppedlocationset.union(stoppedadvertisingset) for nodelocation in startedadvertisingset: nodelogfile.write(str(currenttime) + " " + nodelocation + " advertising " + keyname + "\n") for nodelocation in stoppedadvertisingset: nodelogfile.write(str(currenttime) + " " + nodelocation + " not_advertising " + keyname + "\n") for location in addedlocationset: # Contact this node immediately since it started advertising. nextlookupdict[location] = 0 for location in droppedlocationset: if location in contactabledict or location in advertisingset: # Contact this node immediately since it stopped advertising. nextlookupdict[location] = 0 else: # This node isn't advertising or contactable, so there's no real reason # to contact it until it starts advertising again. if location in nextlookupdict: del nextlookupdict[location] else: print currenttime, "ERROR:", location, \ "not in nextlookupdict when it stopped advertising" # The list of node locations to try to contact at this time. currentlookuplist = [] for location in nextlookupdict: if nextlookupdict[location] < currenttime: currentlookuplist.append(location) # Contact all list of node we're currently interested in. This lookup is # parallelized to improve speed. Especially for the mass lookup that occurs # when the script starts, increasing experimentlib.num_worker_threads to a # larger number will further increase the speed of this lookup. successlist, failurelist = experimentlib.run_parallelized( currentlookuplist, browse_node) # These are the nodes for which contacting them failed for nodelocation, errorstring in failurelist: if nodelocation in contactabledict: nodeid = contactabledict.pop(nodelocation) nodelogfile.write(str(currenttime) + " " + nodelocation + " not_contactable " + nodeid + "\n") if nodelocation in advertisingset: # Node is advertising but not contactable, so try again later. nextlookupdict[nodelocation] = currenttime + \ (random.random() + 0.5) * ONLY_ADVERTISING_POLLING_INTERVAL else: # This node isn't advertising or contactable, so there's no real reason # to contact it until it starts advertising again. if nodelocation in nextlookupdict: del nextlookupdict[nodelocation] else: print currenttime, "ERROR:", nodelocation, \ "not in nextlookupdict when it stopped being contactable" # These are the nodes we successfully contacted. for nodelocation, nodedict in successlist: # Log the retrieved node information. vessellogfile.write(str(nodedict) + "\n") nodeid = nodedict['nodeid'] if nodelocation in contactabledict: # The node changed nodeid while still advertising. This probably won't # ever happen, but we might as well check. (Note: apparently this does # happen every once in a while.) if nodeid != contactabledict[nodelocation]: nodelogfile.write(str(currenttime) + " " + nodelocation + " not_contactable " + contactabledict[nodelocation] + "\n") del contactabledict[nodelocation] if nodelocation not in contactabledict: contactabledict[nodelocation] = nodeid nodelogfile.write(str(currenttime) + " " + nodelocation + " contactable " + nodeid + "\n") # Pick a semi-random time to do the next lookup so we aren't contacting all # the nodes at once. if nodelocation in advertisingset: nextlookupdict[nodelocation] = currenttime + \ (random.random() + 0.5) * CONSISTENT_POLLING_INTERVAL else: nextlookupdict[nodelocation] = currenttime + \ (random.random() + 0.5) * ONLY_CONTACTABLE_POLLING_INTERVAL
for vessel in vessel_details: nodelocation_list.append(vessel["nodelocation"]) print("Number of vessels acquired: " + str(len(CBFUNC_CONTEXT['vesselhandle_list']))) # Write node IPs to file to be uploaded neighborip_fo = open("neighboriplist.txt", "w") for ipport in nodelocation_list: ip = ipport.split(":")[0] NODEIPLIST.append(ip) neighborip_fo.write(ip + "\n") neighborip_fo.close() # Using experimentlib.run_parallelized() to upload and run programs uploadips_success, uploadips_fail = experimentlib.run_parallelized(CBFUNC_CONTEXT['vesselhandle_list'], experimentlib.upload_file_to_vessel, CBFUNC_CONTEXT['identity'], "neighboriplist.txt") uploadprog_success, uploadprog_fail = experimentlib.run_parallelized(CBFUNC_CONTEXT['vesselhandle_list'], experimentlib.upload_file_to_vessel, CBFUNC_CONTEXT['identity'], "pingneighbors.py") start_success, start_fail = experimentlib.run_parallelized(CBFUNC_CONTEXT['vesselhandle_list'], experimentlib.start_vessel, CBFUNC_CONTEXT['identity'], "pingneighbors.py", [str(CBFUNC_CONTEXT['geniport'])]) # Handle failed nodes for failed in listops_uniq(uploadips_fail + uploadprog_fail + start_fail): nodeid, vesselname = experimentlib.get_nodeid_and_vesselname(failed[0]) for vessel in vessel_details: if vessel["nodeid"] == nodeid: NODEIPLIST.remove(vessel["nodelocation"][:-5]) print "Failure on vessel " + failed[0] + ". Error was: " + failed[1] print "Files uploaded to nodes and programs running" # Now that vessel setup is complete, handle output geoip_init_client("http://geoip.cs.washington.edu:12679")