def default_init_overlord(overlord): # Release any pre-allocated vessels vessel_handlers = explib.seattlegeni_get_acquired_vessels( overlord.config['identity']) overlord.logger.info('Releasing ' + str(len(vessel_handlers)) + ' pre-allocated vessels') overlord.release_vessels(vessel_handlers)
def run(*args): """ <Purpose> Starts the deployment and monitoring of a service on a number of vessels. Handles all acquisition of, uploading to, starting, and release of vessels. Contains the main loop of this program, and is thus the final function to call in all client programs. Requires init() to have been called prior to running. <Arguments> *args <Exceptions> None <Side Effects> Persistently writes to a log file. <Returns> None """ # Write logfile header config['logfile'] = open(config['logfilename'], 'w') config['logfile'].write('################################################\n') config['logfile'].write('## Overlord Deployment and Monitoring Log ##\n') config['logfile'].write('################################################\n\n') config['logfile'].write('GENI user: '******'identity']['username'] + '\n') config['logfile'].write('Vessels to monitor: ' + str(config['vesselcount']) + '\n') config['logfile'].write('Time of script start: ' + str(time.time()) + '\n\n') config['logfile'].flush() # Release any preallocated vessels vesselhandle_list = explib.seattlegeni_get_acquired_vessels(config['identity']) release_vessels(vesselhandle_list, 'Releasing ' + str(len(vesselhandle_list)) + ' preallocated vessels...') # Acquire an initial sample of vessels config['logfile'].write(str(time.time()) + ': Fetching initial batch of ' + str(config['vesselcount']) + ' vessels:\n') config['logfile'].flush() vesselhandle_list = [] while not vesselhandle_list: vesselhandle_list = acquire_vessels(config['vesselcount']) # Upload program to vessels vesselhandle_list = upload_to_vessels(vesselhandle_list, config['program_filename']) # Run program on vessels vesselhandle_list, failed_list = run_on_vessels(vesselhandle_list, config['program_filename'], *args) # Release any failed vessels if failed_list: config['logfile'].write(str(time.time()) + ': Running ' + config['program_filename'] + ' failed on ' + str(len(failed_list)) + ' vessels\n') # Get details about failed vessel(s) and log them for vh in failed_list: try: vessel_log = explib.get_vessel_log(vh, config['identity']) except: vessel_log = '[ERROR: vessel log fetch failed]' nodeid, vesselname = explib.get_nodeid_and_vesselname(vh) nodelocation = explib.get_node_location(nodeid) # Log the vessel's log contents config['logfile'].write('Log contents of failed vessel at ' + nodelocation + ': ' + vessel_log + '\n') config['logfile'].flush() # Release the failed vessels release_vessels(failed_list, 'Releasing failed vessel(s)...') # Initialize counter variable for loop iterations loop_iterations = 0 PREPPED = True print "PREPPED!" print "Vessel Handles: %s" % vesselhandle_list # Main loop while KEEP_RUNNING == True: print "Starting Loop!" # Check for vessels not in started state stopped_vessel_list = [] for vh in vesselhandle_list: try: vessel_status = explib.get_vessel_status(vh, config['identity']) log = explib.get_vessel_log(vh, config['identity']) print "Loop Log: %s" % log except: # Node lookup failed, so remove vessel from vesselhandle_list # TODO: proper way to handle failed advertisements? stopped_vessel_list.append(vh) else: if vessel_status != explib.VESSEL_STATUS_STARTED: stopped_vessel_list.append(vh) # Release and replace any stopped vessels if stopped_vessel_list: # Release any stopped vessels release_vessels(stopped_vessel_list, 'Releasing ' + str(len(stopped_vessel_list)) + ' stopped vessel(s)...') # Remove released vessels from vesselhandle_list vesselhandle_list = list_difference(vesselhandle_list, stopped_vessel_list) # Ensure that enough vessels are running if len(vesselhandle_list) < config['vesselcount']: # If there aren't enough active vessels, acquire some config['logfile'].write(str(time.time()) + ': Only ' + str(len(vesselhandle_list)) + ' vessel(s) out of target ' + str(config['vesselcount']) + ' detected\n') config['logfile'].flush() fresh_vessels = acquire_vessels(config['vesselcount'] - len(vesselhandle_list)) # Upload and run program to/on fresh vessels fresh_vessels = upload_to_vessels(fresh_vessels, config['program_filename']) success_list, failed_list = run_on_vessels(fresh_vessels, config['program_filename'], *args) # Release any failed vessels if failed_list: config['logfile'].write(str(time.time()) + ': Running ' + config['program_filename'] + ' failed on ' + str(len(failed_list)) + ' vessels\n') # Get details about failed vessel(s) and log them for vh in failed_list: try: vessel_log = explib.get_vessel_log(vh, config['identity']) except: vessel_log = '[ERROR: vessel log fetch failed]' nodeid, vesselname = explib.get_nodeid_and_vesselname(vh) nodelocation = explib.get_node_location(nodeid) # Log the vessel's log contents config['logfile'].write('Log contents of failed vessel at ' + nodelocation + ': ' + vessel_log + '\n') config['logfile'].flush() # Release the failed vessels release_vessels(failed_list, 'Releasing failed vessel(s)...') # Remove released vessels from fresh_vessels list fresh_vessels = list_difference(fresh_vessels, failed_list) # Add fresh_vessels to vesselhandle_list vesselhandle_list.extend(fresh_vessels) # Sleep for parameterized amount of time time.sleep(VESSEL_POLLING_TIME) # Log a liveness message every certain number of iterations loop_iterations += 1 if loop_iterations % LOG_AFTER_THIS_MANY_LOOPS == 0: config['logfile'].write(str(time.time()) + ': Still alive...\n') config['logfile'].flush() # Renew vessels according to constant period if loop_iterations * VESSEL_POLLING_TIME > VESSEL_RENEWAL_PERIOD: explib.seattlegeni_renew_vessels(config['identity'], vesselhandle_list) loop_iterations = 0
def run(self, *args): """ <Purpose> Starts the deployment and monitoring of a service on a number of vessels. Handles all acquisition of, uploading to, starting, and release of vessels. Contains the main loop of this program, and is thus the final function to call in all client programs. <Arguments> *args: Optional arguments that well be passed when starting the uploaded program file, by default <Exceptions> Common SeattleGENI exceptions described in the module comments. <Side Effects> Unless overridden beforehand, persistently logs to overlord_instance.log and console Returns if a 'stop' file is found in directory of the program importing the API <Returns> None """ # Log setup information for user reference. self.logger.info("Performing actions as GENI user '" + self.config['identity']['username'] + "'") self.logger.info('Vessel status will be polled every ' + str(self.VESSEL_POLLING_PERIOD) + ' seconds') self.logger.info('Vessels will be renewed every ' + str(self.VESSEL_RENEWAL_PERIOD) + ' seconds') # Run the start-up function for overlord self.init_overlord_func(self) # Retrieve the current list of vessel handles owned by identity, regardless # of the results of overlord's start-up function vessel_handlers = explib.seattlegeni_get_acquired_vessels( self.config['identity']) # If there are still vessels allocated to the identity, renew them to keep # renewal periods consistent with newly acquired vessels later on if vessel_handlers: self.logger.info('Renewing ' + str(len(vessel_handlers)) + ' vessels') explib.seattlegeni_renew_vessels(self.config['identity'], vessel_handlers) # Turn VESSEL_RENEWAL_PERIOD into a timedelta object. renewal_delay = timedelta(seconds=self.VESSEL_RENEWAL_PERIOD) # Assume that the vessels have been renewed. last_renewal = datetime.utcnow() # Main loop while True: # End operations if "stop" file is found in same directory as overlord.py if os.path.isfile("./stop"): self.logger.info("Stop file found; Discontinuing operations.") return # Call the acquire vessels function and set the list of the handles of the # newly acquired vessels fresh_handlers = self.acquire_vessels_func(self, vessel_handlers) # If the list is not empty, call the initiate vessel function, and an updated # list of currently owned vessel handles that includes the newly acquired ones # should be returned if fresh_handlers: vessel_handlers = self.init_vessels_func( self, fresh_handlers, vessel_handlers, *args) # Weed out any vessels based on the remove vessels function vessel_handlers = self.remove_vessels_func(self, vessel_handlers) # Perform any maintenance on the vessels, including renewals if needed, # and return the relevant information needed for next maintenance. # Currently coded to fit with default behavior, but should somehow be made a # little more flexible for user's function overrides? last_renewal = self.maintenance_func(self, renewal_delay, last_renewal, vessel_handlers) # If the list of running vessels does not match the requested vessel count, # repeat the loop until it does if not len(vessel_handlers) == self.config['vessel_count']: continue # Sleep between polling vessel statuses. time.sleep(self.VESSEL_POLLING_PERIOD)
def reset_vessels(): vesselhandle_list = explib.seattlegeni_get_acquired_vessels(config['identity']) release_vessels(vesselhandle_list, 'Releasing ' + str(len(vesselhandle_list)) + ' preallocated vessels...')
def get_vessels(): print "Config: %s" % config return explib.seattlegeni_get_acquired_vessels(config['identity'])
def main(): identity = experimentlib.create_identity_from_key_files(PUBLICKEY_FILENAME, PRIVATEKEY_FILENAME) # Get a list of nodes advertising under the public key. Some of these # nodes may be unreachable or may have gone offline since they advertised. # This won't try to communicate with the actual nodes. nodelocation_list = experimentlib.lookup_node_locations_by_identity(identity) print("nodelocation_list:" + str(nodelocation_list)) # Talk to each advertising node to find out which vessels we have on each. # We get back a list of dictionaries, where each dictionary describes a # vessel. There may be multiple vessels from any single node in this list. active_vesselhandle_list = experimentlib.find_vessels_on_nodes(identity, nodelocation_list) print("active_vesselhandle_list:" + str(active_vesselhandle_list)) # Now we want to find out which vessels we've acquired through seattlegeni # that are not in our list. We may, for example, want to release those # vessels because we consider them unusable. try: expected_vesselhandle_list = experimentlib.seattlegeni_get_acquired_vessels(identity) print("expected_vesselhandle_list:" + str(expected_vesselhandle_list)) # If we already have enough usable vessels, we're done. if len(active_vesselhandle_list) >= MIN_VESSELS_TO_KEEP: print( "There are already " + str(len(active_vesselhandle_list)) + " active vessels " + "and our MIN_VESSELS_TO_KEEP is " + str(MIN_VESSELS_TO_KEEP) ) return # We assume all of our vessels come from seattlegeni, so if any vessels we # should have access to according to seattlegeni aren't accessible or # otherwise usable, then release them. vesselhandles_to_release = [] for vesselhandle in expected_vesselhandle_list: if vesselhandle not in active_vesselhandle_list: vesselhandles_to_release.append(vesselhandle) if vesselhandles_to_release: print( str(len(vesselhandles_to_release)) + " vessels were inaccessible, so will try to release them:" + str(vesselhandles_to_release) ) try: experimentlib.seattlegeni_release_vessels(identity, vesselhandles_to_release) except experimentlib.SeattleClearinghouseError, e: print("Failed to release vessels: " + str(e)) else: print("Vessels successfully released.") # Determine the maximum number of vessels we can acquire through seattlegeni. max_vessels_allowed = experimentlib.seattlegeni_max_vessels_allowed(identity) print("max_vessels_allowed: " + str(max_vessels_allowed)) # Determine the number of vessels we already have acquired through seattlegeni. num_currently_acquired = len(experimentlib.seattlegeni_get_acquired_vessels(identity)) print("currently_acquired_count: " + str(num_currently_acquired)) # Let's try to get as close to MIN_VESSELS_TO_KEEP without requesting more # than is allowed by this account. num_vessels_to_request = min(max_vessels_allowed, MIN_VESSELS_TO_KEEP) - num_currently_acquired if num_vessels_to_request <= 0: print("This account doesn't have enough vessel credits to request more vessels.") return else: print("Will try to acquire " + str(num_vessels_to_request) + " vessels.") vessel_type = experimentlib.SEATTLECLEARINGHOUSE_VESSEL_TYPE_WAN acquired_vessels = experimentlib.seattlegeni_acquire_vessels(identity, vessel_type, num_vessels_to_request) print("Acquired " + str(num_vessels_to_request) + " vessels: " + str(acquired_vessels))
def default_init_overlord(overlord): # Release any pre-allocated vessels vessel_handlers = explib.seattlegeni_get_acquired_vessels(overlord.config["identity"]) overlord.logger.info("Releasing " + str(len(vessel_handlers)) + " pre-allocated vessels") overlord.release_vessels(vessel_handlers)
def run(self, *args): """ <Purpose> Starts the deployment and monitoring of a service on a number of vessels. Handles all acquisition of, uploading to, starting, and release of vessels. Contains the main loop of this program, and is thus the final function to call in all client programs. <Arguments> *args: Optional arguments that well be passed when starting the uploaded program file, by default <Exceptions> Common SeattleGENI exceptions described in the module comments. <Side Effects> Unless overridden beforehand, persistently logs to overlord_instance.log and console Returns if a 'stop' file is found in directory of the program importing the API <Returns> None """ # Log setup information for user reference. self.logger.info("Performing actions as GENI user '" + self.config["identity"]["username"] + "'") self.logger.info("Vessel status will be polled every " + str(self.VESSEL_POLLING_PERIOD) + " seconds") self.logger.info("Vessels will be renewed every " + str(self.VESSEL_RENEWAL_PERIOD) + " seconds") # Run the start-up function for overlord self.init_overlord_func(self) # Retrieve the current list of vessel handles owned by identity, regardless # of the results of overlord's start-up function vessel_handlers = explib.seattlegeni_get_acquired_vessels(self.config["identity"]) # If there are still vessels allocated to the identity, renew them to keep # renewal periods consistent with newly acquired vessels later on if vessel_handlers: self.logger.info("Renewing " + str(len(vessel_handlers)) + " vessels") explib.seattlegeni_renew_vessels(self.config["identity"], vessel_handlers) # Turn VESSEL_RENEWAL_PERIOD into a timedelta object. renewal_delay = timedelta(seconds=self.VESSEL_RENEWAL_PERIOD) # Assume that the vessels have been renewed. last_renewal = datetime.utcnow() # Main loop while True: # End operations if "stop" file is found in same directory as overlord.py if os.path.isfile("./stop"): self.logger.info("Stop file found; Discontinuing operations.") return # Call the acquire vessels function and set the list of the handles of the # newly acquired vessels fresh_handlers = self.acquire_vessels_func(self, vessel_handlers) # If the list is not empty, call the initiate vessel function, and an updated # list of currently owned vessel handles that includes the newly acquired ones # should be returned if fresh_handlers: vessel_handlers = self.init_vessels_func(self, fresh_handlers, vessel_handlers, *args) # Weed out any vessels based on the remove vessels function vessel_handlers = self.remove_vessels_func(self, vessel_handlers) # Perform any maintenance on the vessels, including renewals if needed, # and return the relevant information needed for next maintenance. # Currently coded to fit with default behavior, but should somehow be made a # little more flexible for user's function overrides? last_renewal = self.maintenance_func(self, renewal_delay, last_renewal, vessel_handlers) # If the list of running vessels does not match the requested vessel count, # repeat the loop until it does if not len(vessel_handlers) == self.config["vessel_count"]: continue # Sleep between polling vessel statuses. time.sleep(self.VESSEL_POLLING_PERIOD)
# SeattleGENI. If SeattleClearinghouseError exception thrown, print warning. try: CBFUNC_CONTEXT['geniport'] = experimentlib.seattlegeni_user_port(CBFUNC_CONTEXT['identity']) except Exception, e: if repr(e).startswith("SeattleClearinghouseError"): print """Error: Cannot make secure SSL connection to SeattleGENI Please make sure that: * M2Crypto is installed * the current directory contains a valid PEM file containing CA certificates To run appmap.py without SSL authentication, use the `--insecure` switch. Check documentation for more details.""" sys.exit() # Get list of vessels user has previously allocated CBFUNC_CONTEXT['vesselhandle_list'] = experimentlib.seattlegeni_get_acquired_vessels(CBFUNC_CONTEXT['identity']) if appmap_config['num_nodes'] > 0 and len(CBFUNC_CONTEXT['vesselhandle_list']) > 0: # Release any vessels previously acquired from GENI experimentlib.seattlegeni_release_vessels(CBFUNC_CONTEXT['identity'], CBFUNC_CONTEXT['vesselhandle_list']) if appmap_config['num_nodes'] > 0: # Acquire argument number of vessels using seattlegeni interface CBFUNC_CONTEXT['vesselhandle_list'] = experimentlib.seattlegeni_acquire_vessels(CBFUNC_CONTEXT['identity'], "wan", int(appmap_config['num_nodes'])) # If no vessels were acquired, an error occurred and script exits if len(CBFUNC_CONTEXT['vesselhandle_list']) == 0: print "Error: No vessels acquired. Try rerunning " + sys.argv[0] sys.exit() nodelocation_list = []
def main(): identity = experimentlib.create_identity_from_key_files( PUBLICKEY_FILENAME, PRIVATEKEY_FILENAME) # Get a list of nodes advertising under the public key. Some of these # nodes may be unreachable or may have gone offline since they advertised. # This won't try to communicate with the actual nodes. nodelocation_list = experimentlib.lookup_node_locations_by_identity( identity) print("nodelocation_list:" + str(nodelocation_list)) # Talk to each advertising node to find out which vessels we have on each. # We get back a list of dictionaries, where each dictionary describes a # vessel. There may be multiple vessels from any single node in this list. active_vesselhandle_list = experimentlib.find_vessels_on_nodes( identity, nodelocation_list) print("active_vesselhandle_list:" + str(active_vesselhandle_list)) # Now we want to find out which vessels we've acquired through seattlegeni # that are not in our list. We may, for example, want to release those # vessels because we consider them unusable. try: expected_vesselhandle_list = experimentlib.seattlegeni_get_acquired_vessels( identity) print("expected_vesselhandle_list:" + str(expected_vesselhandle_list)) # If we already have enough usable vessels, we're done. if len(active_vesselhandle_list) >= MIN_VESSELS_TO_KEEP: print("There are already " + str(len(active_vesselhandle_list)) + " active vessels " + "and our MIN_VESSELS_TO_KEEP is " + str(MIN_VESSELS_TO_KEEP)) return # We assume all of our vessels come from seattlegeni, so if any vessels we # should have access to according to seattlegeni aren't accessible or # otherwise usable, then release them. vesselhandles_to_release = [] for vesselhandle in expected_vesselhandle_list: if vesselhandle not in active_vesselhandle_list: vesselhandles_to_release.append(vesselhandle) if vesselhandles_to_release: print( str(len(vesselhandles_to_release)) + " vessels were inaccessible, so will try to release them:" + str(vesselhandles_to_release)) try: experimentlib.seattlegeni_release_vessels( identity, vesselhandles_to_release) except experimentlib.SeattleClearinghouseError, e: print("Failed to release vessels: " + str(e)) else: print("Vessels successfully released.") # Determine the maximum number of vessels we can acquire through seattlegeni. max_vessels_allowed = experimentlib.seattlegeni_max_vessels_allowed( identity) print("max_vessels_allowed: " + str(max_vessels_allowed)) # Determine the number of vessels we already have acquired through seattlegeni. num_currently_acquired = len( experimentlib.seattlegeni_get_acquired_vessels(identity)) print("currently_acquired_count: " + str(num_currently_acquired)) # Let's try to get as close to MIN_VESSELS_TO_KEEP without requesting more # than is allowed by this account. num_vessels_to_request = min( max_vessels_allowed, MIN_VESSELS_TO_KEEP) - num_currently_acquired if num_vessels_to_request <= 0: print( "This account doesn't have enough vessel credits to request more vessels." ) return else: print("Will try to acquire " + str(num_vessels_to_request) + " vessels.") vessel_type = experimentlib.SEATTLECLEARINGHOUSE_VESSEL_TYPE_WAN acquired_vessels = experimentlib.seattlegeni_acquire_vessels( identity, vessel_type, num_vessels_to_request) print("Acquired " + str(num_vessels_to_request) + " vessels: " + str(acquired_vessels))