def main(): identity = experimentlib.create_identity_from_key_files(PUBLICKEY_FILENAME, PRIVATEKEY_FILENAME) nodelocation_list = experimentlib.lookup_node_locations_by_identity(identity) print("Number of advertising nodes: " + str(len(nodelocation_list))) vesselhandle_list = experimentlib.find_vessels_on_nodes(identity, nodelocation_list) print("Number of active vessels: " + str(len(vesselhandle_list))) # Note that we could use experimentlib.run_parallelized() to parallelize # this, but for simplicity we do each sequentially. for vesselhandle in vesselhandle_list: try: nodeid, vesselname = experimentlib.get_nodeid_and_vesselname(vesselhandle) nodelocation = experimentlib.get_node_location(nodeid) # Note: you may want to reset_vessel(). experimentlib.upload_file_to_vessel(vesselhandle, identity, PROGRAM_FILE) print("Uploaded " + PROGRAM_FILE + " to " + nodelocation + " vessel " + vesselname) experimentlib.start_vessel(vesselhandle, identity, PROGRAM_FILE, ARGUMENTS_TO_START_PROGRAM_WITH) print("Program started on " + nodelocation + " vessel " + vesselname) except experimentlib.SeattleExperimentError, e: print("Failure on vessel " + vesselhandle + ". Error was: " + str(e))
def main(): identity = experimentlib.create_identity_from_key_files(PUBLICKEY_FILENAME, PRIVATEKEY_FILENAME) nodelocation_list = experimentlib.lookup_node_locations_by_identity(identity) print("Number of advertising nodes: " + str(len(nodelocation_list))) vesselhandle_list = experimentlib.find_vessels_on_nodes(identity, nodelocation_list) print("Number of active vessels: " + str(len(vesselhandle_list))) # Note that we could use experimentlib.run_parallelized() to parallelize the # uploading of files and speed things up, but for simplicity we do each # upload sequentially. for vesselhandle in vesselhandle_list: try: nodeid, vesselname = experimentlib.get_nodeid_and_vesselname(vesselhandle) nodelocation = experimentlib.get_node_location(nodeid) for local_filename in FILES_TO_UPLOAD: experimentlib.upload_file_to_vessel(vesselhandle, identity, local_filename) print("Uploaded " + local_filename + " to " + nodelocation + " vessel " + vesselname) except experimentlib.SeattleExperimentError, e: print("Failure on vessel " + vesselhandle + ". Error was: " + str(e))
def get_node_location_or_unknown(vesselhandle): """ A helper function that will return the location of the node a vessel is on or the string "Unknown" if the location of a vessel cannot be determined. """ nodeid, vesselname = experimentlib.get_nodeid_and_vesselname(vesselhandle) try: return experimentlib.get_node_location(nodeid) except experimentlib.NodeLocationNotAdvertisedError, e: print(str(e)) return "Unknown"
def main(): identity = experimentlib.create_identity_from_key_files( PUBLICKEY_FILENAME, PRIVATEKEY_FILENAME) nodelocation_list = experimentlib.lookup_node_locations_by_identity( identity) print("Number of advertising nodes: " + str(len(nodelocation_list))) vesselhandle_list = experimentlib.find_vessels_on_nodes( identity, nodelocation_list) print("Number of active vessels: " + str(len(vesselhandle_list))) for vesselhandle in vesselhandle_list: try: # Generate the vessel directory name with format: host_port_vesselname nodeid, vesselname = experimentlib.get_nodeid_and_vesselname( vesselhandle) nodelocation = experimentlib.get_node_location(nodeid) host, port = experimentlib.get_host_and_port(nodelocation) dirname = host + "_" + str(port) + "_" + vesselname # Create the vessel directory if it doesn't already exist. vessel_directory = os.path.join(DOWNLOAD_DIRECTORY, dirname) if not os.path.exists(vessel_directory): os.mkdir(vessel_directory) # Get a list of files on the vessel. filelist = experimentlib.get_vessel_file_list( vesselhandle, identity) print("Files on " + nodelocation + " " + vesselname + ": " + str(filelist)) for remote_filename in filelist: local_filename = os.path.join(vessel_directory, remote_filename) experimentlib.download_file_from_vessel( vesselhandle, identity, remote_filename, local_filename) print("Downloaded " + local_filename) except experimentlib.SeattleExperimentError, e: print("Failure on vessel " + vesselhandle + ". Error was: " + str(e))
def upload_to_vessels(vesselhandle_list, filename): """ <Purpose> Uploads a file to a set of vessels. A batch wrapper around the Experiment Library function upload_file_to_vessel, with logging support. <Arguments> vesselhandle_list A list of vesselhandles of vessels to which the file is to be uploaded. filename The filename of the file to be uploaded. <Exceptions> None <Side Effects> None <Returns> A list of vessels to which the upload succeeded. """ # Log the fact that uploads are occurring config['logfile'].write(str(time.time()) + ': Uploading ' + filename + ' to '+ str(len(vesselhandle_list)) + ' vessel(s)...\n') config['logfile'].flush() # Initially set return list equal to argument vesselhandle list success_list = vesselhandle_list failed_list = [] # For each vesselhandle, attempt an upload for vh in vesselhandle_list: try: print "UPLOADING:%s - %s" % (vh, filename) explib.upload_file_to_vessel(vh, config['identity'], filename) except explib.NodeCommunicationError, e: # If upload failed, remove from vesselhandle_list... success_list.remove(vh) # ...add to failed_list... failed_list.append(vh) # ...and lookup the nodelocation so it can be logged nodeid, vesselname = explib.get_nodeid_and_vesselname(vh) nodelocation = explib.get_node_location(nodeid) config['logfile'].write('Failure on vessel ' + nodelocation + '\n') config['logfile'].write('Error was: ' + str(e) + '\n') config['logfile'].flush()
def vessel_location(self, vessel_handler): """ <Purpose> Determine the node location of the given vessel handle <Arguments> vessel_handler: The vesselhandle of the vessel whose location information is needed <Exceptions> None <Side Effects> None <Returns> A nodelocation """ node_id = explib.get_nodeid_and_vesselname(vessel_handler)[0] return explib.get_node_location(node_id)
def main(): identity = experimentlib.create_identity_from_key_files( PUBLICKEY_FILENAME, PRIVATEKEY_FILENAME) nodelocation_list = experimentlib.lookup_node_locations_by_identity( identity) print("Number of advertising nodes: " + str(len(nodelocation_list))) vesselhandle_list = experimentlib.find_vessels_on_nodes( identity, nodelocation_list) print("Number of active vessels: " + str(len(vesselhandle_list))) # Note that we could use experimentlib.run_parallelized() to parallelize # this, but for simplicity we do each sequentially. for vesselhandle in vesselhandle_list: try: nodeid, vesselname = experimentlib.get_nodeid_and_vesselname( vesselhandle) nodelocation = experimentlib.get_node_location(nodeid) # Note: you may want to reset_vessel(). experimentlib.upload_file_to_vessel(vesselhandle, identity, PROGRAM_FILE) print("Uploaded " + PROGRAM_FILE + " to " + nodelocation + " vessel " + vesselname) experimentlib.start_vessel(vesselhandle, identity, PROGRAM_FILE, ARGUMENTS_TO_START_PROGRAM_WITH) print("Program started on " + nodelocation + " vessel " + vesselname) except experimentlib.SeattleExperimentError, e: print("Failure on vessel " + vesselhandle + ". Error was: " + str(e))
def main(): identity = experimentlib.create_identity_from_key_files(PUBLICKEY_FILENAME, PRIVATEKEY_FILENAME) nodelocation_list = experimentlib.lookup_node_locations_by_identity(identity) print("Number of advertising nodes: " + str(len(nodelocation_list))) vesselhandle_list = experimentlib.find_vessels_on_nodes(identity, nodelocation_list) print("Number of active vessels: " + str(len(vesselhandle_list))) for vesselhandle in vesselhandle_list: try: # Generate the vessel directory name with format: host_port_vesselname nodeid, vesselname = experimentlib.get_nodeid_and_vesselname(vesselhandle) nodelocation = experimentlib.get_node_location(nodeid) host, port = experimentlib.get_host_and_port(nodelocation) dirname = host + "_" + str(port) + "_" + vesselname # Create the vessel directory if it doesn't already exist. vessel_directory = os.path.join(DOWNLOAD_DIRECTORY, dirname) if not os.path.exists(vessel_directory): os.mkdir(vessel_directory) # Get a list of files on the vessel. filelist = experimentlib.get_vessel_file_list(vesselhandle, identity) print("Files on " + nodelocation + " " + vesselname + ": " + str(filelist)) for remote_filename in filelist: local_filename = os.path.join(vessel_directory, remote_filename) experimentlib.download_file_from_vessel(vesselhandle, identity, remote_filename, local_filename) print("Downloaded " + local_filename) except experimentlib.SeattleExperimentError, e: print("Failure on vessel " + vesselhandle + ". Error was: " + str(e))
def main(): identity = experimentlib.create_identity_from_key_files(PUBLICKEY_FILENAME, PRIVATEKEY_FILENAME) nodelocation_list = experimentlib.lookup_node_locations_by_identity(identity) print("Number of advertising nodes: " + str(len(nodelocation_list))) vesselhandle_list = experimentlib.find_vessels_on_nodes(identity, nodelocation_list) print("Number of active vessels: " + str(len(vesselhandle_list))) for vesselhandle in vesselhandle_list: try: nodeid, vesselname = experimentlib.get_nodeid_and_vesselname(vesselhandle) nodelocation = experimentlib.get_node_location(nodeid) print("====================================================================") print("Log from " + nodelocation + " " + vesselname) logcontents = experimentlib.get_vessel_log(vesselhandle, identity) print(logcontents) except experimentlib.SeattleExperimentError, e: print("Failure on vessel " + vesselhandle + ". Error was: " + str(e))
# Tell overlord (the slightly modified version) to distribute 10 instances of the newproxy(pre-processed) to various seattle nodes init_dict = overlord.init(user, 10, 'wan', 'newproxypp.repy') config = overlord.get_config() run = threading.Thread(target=overlord.run, args=[init_dict['geni_port']]) run.start() # Wait a bit, overlord needs its time if you don't want it to throw up everywhere time.sleep(60) # Get the geoipserver so that we can map the ips of our nodes to actual locations locations = {} geoipserver = xmlrpclib.ServerProxy(GEO_IP_SERVER) for vessel in overlord.get_vessels(): nodeid, vesselname = vessel.split(":") location = experimentlib.get_node_location(nodeid).split(":")[0] locations[location] = [vessel] if debug: try: log = experimentlib.get_vessel_log(vessel, config['identity']) print "Log for %s: %s" (location, log) except: print "Unnexpected Error: %s" % sys.exc_info()[0] try: loc = geoipserver.record_by_addr(location) locations[location].append("%s - %s" % (loc['city'], loc['country_name'])) except: locations[location].append("%s (No Location Data)" % location) conn = None # Connect to our local database, which contains the user agent strings and their associated OSs.
def run(*args): """ <Purpose> Starts the deployment and monitoring of a service on a number of vessels. Handles all acquisition of, uploading to, starting, and release of vessels. Contains the main loop of this program, and is thus the final function to call in all client programs. Requires init() to have been called prior to running. <Arguments> *args <Exceptions> None <Side Effects> Persistently writes to a log file. <Returns> None """ # Write logfile header config['logfile'] = open(config['logfilename'], 'w') config['logfile'].write('################################################\n') config['logfile'].write('## Overlord Deployment and Monitoring Log ##\n') config['logfile'].write('################################################\n\n') config['logfile'].write('GENI user: '******'identity']['username'] + '\n') config['logfile'].write('Vessels to monitor: ' + str(config['vesselcount']) + '\n') config['logfile'].write('Time of script start: ' + str(time.time()) + '\n\n') config['logfile'].flush() # Release any preallocated vessels vesselhandle_list = explib.seattlegeni_get_acquired_vessels(config['identity']) release_vessels(vesselhandle_list, 'Releasing ' + str(len(vesselhandle_list)) + ' preallocated vessels...') # Acquire an initial sample of vessels config['logfile'].write(str(time.time()) + ': Fetching initial batch of ' + str(config['vesselcount']) + ' vessels:\n') config['logfile'].flush() vesselhandle_list = [] while not vesselhandle_list: vesselhandle_list = acquire_vessels(config['vesselcount']) # Upload program to vessels vesselhandle_list = upload_to_vessels(vesselhandle_list, config['program_filename']) # Run program on vessels vesselhandle_list, failed_list = run_on_vessels(vesselhandle_list, config['program_filename'], *args) # Release any failed vessels if failed_list: config['logfile'].write(str(time.time()) + ': Running ' + config['program_filename'] + ' failed on ' + str(len(failed_list)) + ' vessels\n') # Get details about failed vessel(s) and log them for vh in failed_list: try: vessel_log = explib.get_vessel_log(vh, config['identity']) except: vessel_log = '[ERROR: vessel log fetch failed]' nodeid, vesselname = explib.get_nodeid_and_vesselname(vh) nodelocation = explib.get_node_location(nodeid) # Log the vessel's log contents config['logfile'].write('Log contents of failed vessel at ' + nodelocation + ': ' + vessel_log + '\n') config['logfile'].flush() # Release the failed vessels release_vessels(failed_list, 'Releasing failed vessel(s)...') # Initialize counter variable for loop iterations loop_iterations = 0 PREPPED = True print "PREPPED!" print "Vessel Handles: %s" % vesselhandle_list # Main loop while KEEP_RUNNING == True: print "Starting Loop!" # Check for vessels not in started state stopped_vessel_list = [] for vh in vesselhandle_list: try: vessel_status = explib.get_vessel_status(vh, config['identity']) log = explib.get_vessel_log(vh, config['identity']) print "Loop Log: %s" % log except: # Node lookup failed, so remove vessel from vesselhandle_list # TODO: proper way to handle failed advertisements? stopped_vessel_list.append(vh) else: if vessel_status != explib.VESSEL_STATUS_STARTED: stopped_vessel_list.append(vh) # Release and replace any stopped vessels if stopped_vessel_list: # Release any stopped vessels release_vessels(stopped_vessel_list, 'Releasing ' + str(len(stopped_vessel_list)) + ' stopped vessel(s)...') # Remove released vessels from vesselhandle_list vesselhandle_list = list_difference(vesselhandle_list, stopped_vessel_list) # Ensure that enough vessels are running if len(vesselhandle_list) < config['vesselcount']: # If there aren't enough active vessels, acquire some config['logfile'].write(str(time.time()) + ': Only ' + str(len(vesselhandle_list)) + ' vessel(s) out of target ' + str(config['vesselcount']) + ' detected\n') config['logfile'].flush() fresh_vessels = acquire_vessels(config['vesselcount'] - len(vesselhandle_list)) # Upload and run program to/on fresh vessels fresh_vessels = upload_to_vessels(fresh_vessels, config['program_filename']) success_list, failed_list = run_on_vessels(fresh_vessels, config['program_filename'], *args) # Release any failed vessels if failed_list: config['logfile'].write(str(time.time()) + ': Running ' + config['program_filename'] + ' failed on ' + str(len(failed_list)) + ' vessels\n') # Get details about failed vessel(s) and log them for vh in failed_list: try: vessel_log = explib.get_vessel_log(vh, config['identity']) except: vessel_log = '[ERROR: vessel log fetch failed]' nodeid, vesselname = explib.get_nodeid_and_vesselname(vh) nodelocation = explib.get_node_location(nodeid) # Log the vessel's log contents config['logfile'].write('Log contents of failed vessel at ' + nodelocation + ': ' + vessel_log + '\n') config['logfile'].flush() # Release the failed vessels release_vessels(failed_list, 'Releasing failed vessel(s)...') # Remove released vessels from fresh_vessels list fresh_vessels = list_difference(fresh_vessels, failed_list) # Add fresh_vessels to vesselhandle_list vesselhandle_list.extend(fresh_vessels) # Sleep for parameterized amount of time time.sleep(VESSEL_POLLING_TIME) # Log a liveness message every certain number of iterations loop_iterations += 1 if loop_iterations % LOG_AFTER_THIS_MANY_LOOPS == 0: config['logfile'].write(str(time.time()) + ': Still alive...\n') config['logfile'].flush() # Renew vessels according to constant period if loop_iterations * VESSEL_POLLING_TIME > VESSEL_RENEWAL_PERIOD: explib.seattlegeni_renew_vessels(config['identity'], vesselhandle_list) loop_iterations = 0