def default_maintenance(overlord, renewal_delay, last_renewal, vessel_handlers):
# Renew vessels periodically.
time_elapsed = datetime.utcnow() - last_renewal
if time_elapsed > renewal_delay:
# Perform the vessel renewal.
overlord.logger.info("Renewing " + str(len(vessel_handlers)) + " vessels")
explib.seattlegeni_renew_vessels(overlord.config["identity"], vessel_handlers)
# Reset the time.
last_renewal = datetime.utcnow()
return last_renewal
def default_maintenance(overlord, renewal_delay, last_renewal,
vessel_handlers):
# Renew vessels periodically.
time_elapsed = datetime.utcnow() - last_renewal
if time_elapsed > renewal_delay:
# Perform the vessel renewal.
overlord.logger.info('Renewing ' + str(len(vessel_handlers)) +
' vessels')
explib.seattlegeni_renew_vessels(overlord.config['identity'],
vessel_handlers)
# Reset the time.
last_renewal = datetime.utcnow()
return last_renewal
def run(self, *args):
"""
<Purpose>
Starts the deployment and monitoring of a service on a number of vessels.
Handles all acquisition of, uploading to, starting, and release of vessels.
Contains the main loop of this program, and is thus the final function to
call in all client programs.
<Arguments>
*args:
Optional arguments that well be passed when starting the uploaded
program file, by default
<Exceptions>
Common SeattleGENI exceptions described in the module comments.
<Side Effects>
Unless overridden beforehand, persistently logs to overlord_instance.log and
console
Returns if a 'stop' file is found in directory of the program importing the API
<Returns>
None
"""
# Log setup information for user reference.
self.logger.info("Performing actions as GENI user '" +
self.config['identity']['username'] + "'")
self.logger.info('Vessel status will be polled every ' +
str(self.VESSEL_POLLING_PERIOD) + ' seconds')
self.logger.info('Vessels will be renewed every ' +
str(self.VESSEL_RENEWAL_PERIOD) + ' seconds')
# Run the start-up function for overlord
self.init_overlord_func(self)
# Retrieve the current list of vessel handles owned by identity, regardless
# of the results of overlord's start-up function
vessel_handlers = explib.seattlegeni_get_acquired_vessels(
self.config['identity'])
# If there are still vessels allocated to the identity, renew them to keep
# renewal periods consistent with newly acquired vessels later on
if vessel_handlers:
self.logger.info('Renewing ' + str(len(vessel_handlers)) +
' vessels')
explib.seattlegeni_renew_vessels(self.config['identity'],
vessel_handlers)
# Turn VESSEL_RENEWAL_PERIOD into a timedelta object.
renewal_delay = timedelta(seconds=self.VESSEL_RENEWAL_PERIOD)
# Assume that the vessels have been renewed.
last_renewal = datetime.utcnow()
# Main loop
while True:
# End operations if "stop" file is found in same directory as overlord.py
if os.path.isfile("./stop"):
self.logger.info("Stop file found; Discontinuing operations.")
return
# Call the acquire vessels function and set the list of the handles of the
# newly acquired vessels
fresh_handlers = self.acquire_vessels_func(self, vessel_handlers)
# If the list is not empty, call the initiate vessel function, and an updated
# list of currently owned vessel handles that includes the newly acquired ones
# should be returned
if fresh_handlers:
vessel_handlers = self.init_vessels_func(
self, fresh_handlers, vessel_handlers, *args)
# Weed out any vessels based on the remove vessels function
vessel_handlers = self.remove_vessels_func(self, vessel_handlers)
# Perform any maintenance on the vessels, including renewals if needed,
# and return the relevant information needed for next maintenance.
# Currently coded to fit with default behavior, but should somehow be made a
# little more flexible for user's function overrides?
last_renewal = self.maintenance_func(self, renewal_delay,
last_renewal, vessel_handlers)
# If the list of running vessels does not match the requested vessel count,
# repeat the loop until it does
if not len(vessel_handlers) == self.config['vessel_count']:
continue
# Sleep between polling vessel statuses.
time.sleep(self.VESSEL_POLLING_PERIOD)
vessel_handlers = explib.seattlegeni_acquire_vessels(
self.config['identity'], self.config['vessel_type'],
vessel_count)
except explib.SeattleClearinghouseError, e:
self.logger.error('Error while acquiring vessels: ' + str(e))
return []
else:
self.logger.debug('Successfully acquired ' + str(vessel_count) +
' vessels')
# Renew vessels to extend their expiration time.
self.logger.debug('Renewing ' + str(len(vessel_handlers)) +
' newly acquired vessels')
explib.seattlegeni_renew_vessels(self.config['identity'],
vessel_handlers)
return vessel_handlers
def upload_to_vessels(self, vessel_handlers, filename_list):
"""
<Purpose>
Uploads a list of file to a set of vessels. A batch wrapper around the Experiment
Library function upload_file_to_vessel, with logging and parallelization support.
<Arguments>
vesselhandle_list
A list of vesselhandles of vessels to which the file is to be uploaded.
filename
The filename of the file to be uploaded.
def run(*args):
"""
<Purpose>
Starts the deployment and monitoring of a service on a number of vessels.
Handles all acquisition of, uploading to, starting, and release of vessels.
Contains the main loop of this program, and is thus the final function to
call in all client programs. Requires init() to have been called prior to
running.
<Arguments>
*args
<Exceptions>
None
<Side Effects>
Persistently writes to a log file.
<Returns>
None
"""
# Write logfile header
config['logfile'] = open(config['logfilename'], 'w')
config['logfile'].write('################################################\n')
config['logfile'].write('## Overlord Deployment and Monitoring Log ##\n')
config['logfile'].write('################################################\n\n')
config['logfile'].write('GENI user: '******'identity']['username'] + '\n')
config['logfile'].write('Vessels to monitor: ' + str(config['vesselcount']) + '\n')
config['logfile'].write('Time of script start: ' + str(time.time()) + '\n\n')
config['logfile'].flush()
# Release any preallocated vessels
vesselhandle_list = explib.seattlegeni_get_acquired_vessels(config['identity'])
release_vessels(vesselhandle_list, 'Releasing ' + str(len(vesselhandle_list)) + ' preallocated vessels...')
# Acquire an initial sample of vessels
config['logfile'].write(str(time.time()) + ': Fetching initial batch of ' + str(config['vesselcount']) + ' vessels:\n')
config['logfile'].flush()
vesselhandle_list = []
while not vesselhandle_list:
vesselhandle_list = acquire_vessels(config['vesselcount'])
# Upload program to vessels
vesselhandle_list = upload_to_vessels(vesselhandle_list, config['program_filename'])
# Run program on vessels
vesselhandle_list, failed_list = run_on_vessels(vesselhandle_list,
config['program_filename'],
*args)
# Release any failed vessels
if failed_list:
config['logfile'].write(str(time.time()) + ': Running ' + config['program_filename'] + ' failed on ' + str(len(failed_list)) + ' vessels\n')
# Get details about failed vessel(s) and log them
for vh in failed_list:
try:
vessel_log = explib.get_vessel_log(vh, config['identity'])
except:
vessel_log = '[ERROR: vessel log fetch failed]'
nodeid, vesselname = explib.get_nodeid_and_vesselname(vh)
nodelocation = explib.get_node_location(nodeid)
# Log the vessel's log contents
config['logfile'].write('Log contents of failed vessel at ' + nodelocation + ': ' + vessel_log + '\n')
config['logfile'].flush()
# Release the failed vessels
release_vessels(failed_list, 'Releasing failed vessel(s)...')
# Initialize counter variable for loop iterations
loop_iterations = 0
PREPPED = True
print "PREPPED!"
print "Vessel Handles: %s" % vesselhandle_list
# Main loop
while KEEP_RUNNING == True:
print "Starting Loop!"
# Check for vessels not in started state
stopped_vessel_list = []
for vh in vesselhandle_list:
try:
vessel_status = explib.get_vessel_status(vh, config['identity'])
log = explib.get_vessel_log(vh, config['identity'])
print "Loop Log: %s" % log
except:
# Node lookup failed, so remove vessel from vesselhandle_list
# TODO: proper way to handle failed advertisements?
stopped_vessel_list.append(vh)
else:
if vessel_status != explib.VESSEL_STATUS_STARTED:
stopped_vessel_list.append(vh)
# Release and replace any stopped vessels
if stopped_vessel_list:
# Release any stopped vessels
release_vessels(stopped_vessel_list, 'Releasing ' + str(len(stopped_vessel_list)) + ' stopped vessel(s)...')
# Remove released vessels from vesselhandle_list
vesselhandle_list = list_difference(vesselhandle_list, stopped_vessel_list)
# Ensure that enough vessels are running
if len(vesselhandle_list) < config['vesselcount']:
# If there aren't enough active vessels, acquire some
config['logfile'].write(str(time.time()) + ': Only ' + str(len(vesselhandle_list)) + ' vessel(s) out of target ' + str(config['vesselcount']) + ' detected\n')
config['logfile'].flush()
fresh_vessels = acquire_vessels(config['vesselcount'] - len(vesselhandle_list))
# Upload and run program to/on fresh vessels
fresh_vessels = upload_to_vessels(fresh_vessels, config['program_filename'])
success_list, failed_list = run_on_vessels(fresh_vessels,
config['program_filename'],
*args)
# Release any failed vessels
if failed_list:
config['logfile'].write(str(time.time()) + ': Running ' + config['program_filename'] + ' failed on ' + str(len(failed_list)) + ' vessels\n')
# Get details about failed vessel(s) and log them
for vh in failed_list:
try:
vessel_log = explib.get_vessel_log(vh, config['identity'])
except:
vessel_log = '[ERROR: vessel log fetch failed]'
nodeid, vesselname = explib.get_nodeid_and_vesselname(vh)
nodelocation = explib.get_node_location(nodeid)
# Log the vessel's log contents
config['logfile'].write('Log contents of failed vessel at ' + nodelocation + ': ' + vessel_log + '\n')
config['logfile'].flush()
# Release the failed vessels
release_vessels(failed_list, 'Releasing failed vessel(s)...')
# Remove released vessels from fresh_vessels list
fresh_vessels = list_difference(fresh_vessels, failed_list)
# Add fresh_vessels to vesselhandle_list
vesselhandle_list.extend(fresh_vessels)
# Sleep for parameterized amount of time
time.sleep(VESSEL_POLLING_TIME)
# Log a liveness message every certain number of iterations
loop_iterations += 1
if loop_iterations % LOG_AFTER_THIS_MANY_LOOPS == 0:
config['logfile'].write(str(time.time()) + ': Still alive...\n')
config['logfile'].flush()
# Renew vessels according to constant period
if loop_iterations * VESSEL_POLLING_TIME > VESSEL_RENEWAL_PERIOD:
explib.seattlegeni_renew_vessels(config['identity'], vesselhandle_list)
loop_iterations = 0
# Attempt to acquire vessels. Log success or failure, accordingly.
try:
vesselhandle_list = explib.seattlegeni_acquire_vessels(config['identity'],
config['vesseltype'],
number)
except explib.SeattleGENIError, e:
config['logfile'].write('failure\n')
config['logfile'].write('Error was: ' + str(e) + '\n')
return []
else:
config['logfile'].write('success\n')
finally:
config['logfile'].flush()
# Renew vessels to maximum expiration time
explib.seattlegeni_renew_vessels(config['identity'], vesselhandle_list)
return vesselhandle_list
def upload_to_vessels(vesselhandle_list, filename):
"""
<Purpose>
Uploads a file to a set of vessels. A batch wrapper around the Experiment
Library function upload_file_to_vessel, with logging support.
<Arguments>
vesselhandle_list
def run(self, *args):
"""
<Purpose>
Starts the deployment and monitoring of a service on a number of vessels.
Handles all acquisition of, uploading to, starting, and release of vessels.
Contains the main loop of this program, and is thus the final function to
call in all client programs.
<Arguments>
*args:
Optional arguments that well be passed when starting the uploaded
program file, by default
<Exceptions>
Common SeattleGENI exceptions described in the module comments.
<Side Effects>
Unless overridden beforehand, persistently logs to overlord_instance.log and
console
Returns if a 'stop' file is found in directory of the program importing the API
<Returns>
None
"""
# Log setup information for user reference.
self.logger.info("Performing actions as GENI user '" + self.config["identity"]["username"] + "'")
self.logger.info("Vessel status will be polled every " + str(self.VESSEL_POLLING_PERIOD) + " seconds")
self.logger.info("Vessels will be renewed every " + str(self.VESSEL_RENEWAL_PERIOD) + " seconds")
# Run the start-up function for overlord
self.init_overlord_func(self)
# Retrieve the current list of vessel handles owned by identity, regardless
# of the results of overlord's start-up function
vessel_handlers = explib.seattlegeni_get_acquired_vessels(self.config["identity"])
# If there are still vessels allocated to the identity, renew them to keep
# renewal periods consistent with newly acquired vessels later on
if vessel_handlers:
self.logger.info("Renewing " + str(len(vessel_handlers)) + " vessels")
explib.seattlegeni_renew_vessels(self.config["identity"], vessel_handlers)
# Turn VESSEL_RENEWAL_PERIOD into a timedelta object.
renewal_delay = timedelta(seconds=self.VESSEL_RENEWAL_PERIOD)
# Assume that the vessels have been renewed.
last_renewal = datetime.utcnow()
# Main loop
while True:
# End operations if "stop" file is found in same directory as overlord.py
if os.path.isfile("./stop"):
self.logger.info("Stop file found; Discontinuing operations.")
return
# Call the acquire vessels function and set the list of the handles of the
# newly acquired vessels
fresh_handlers = self.acquire_vessels_func(self, vessel_handlers)
# If the list is not empty, call the initiate vessel function, and an updated
# list of currently owned vessel handles that includes the newly acquired ones
# should be returned
if fresh_handlers:
vessel_handlers = self.init_vessels_func(self, fresh_handlers, vessel_handlers, *args)
# Weed out any vessels based on the remove vessels function
vessel_handlers = self.remove_vessels_func(self, vessel_handlers)
# Perform any maintenance on the vessels, including renewals if needed,
# and return the relevant information needed for next maintenance.
# Currently coded to fit with default behavior, but should somehow be made a
# little more flexible for user's function overrides?
last_renewal = self.maintenance_func(self, renewal_delay, last_renewal, vessel_handlers)
# If the list of running vessels does not match the requested vessel count,
# repeat the loop until it does
if not len(vessel_handlers) == self.config["vessel_count"]:
continue
# Sleep between polling vessel statuses.
time.sleep(self.VESSEL_POLLING_PERIOD)
try:
vessel_handlers = explib.seattlegeni_acquire_vessels(
self.config["identity"], self.config["vessel_type"], vessel_count
)
except explib.SeattleClearinghouseError, e:
self.logger.error("Error while acquiring vessels: " + str(e))
return []
else:
self.logger.debug("Successfully acquired " + str(vessel_count) + " vessels")
# Renew vessels to extend their expiration time.
self.logger.debug("Renewing " + str(len(vessel_handlers)) + " newly acquired vessels")
explib.seattlegeni_renew_vessels(self.config["identity"], vessel_handlers)
return vessel_handlers
def upload_to_vessels(self, vessel_handlers, filename_list):
"""
<Purpose>
Uploads a list of file to a set of vessels. A batch wrapper around the Experiment
Library function upload_file_to_vessel, with logging and parallelization support.
<Arguments>
vesselhandle_list
A list of vesselhandles of vessels to which the file is to be uploaded.
filename
The filename of the file to be uploaded.
