def default_init_overlord(overlord):
# Release any pre-allocated vessels
vessel_handlers = explib.seattlegeni_get_acquired_vessels(
overlord.config['identity'])
overlord.logger.info('Releasing ' + str(len(vessel_handlers)) +
' pre-allocated vessels')
overlord.release_vessels(vessel_handlers)
def run(*args):
"""
<Purpose>
Starts the deployment and monitoring of a service on a number of vessels.
Handles all acquisition of, uploading to, starting, and release of vessels.
Contains the main loop of this program, and is thus the final function to
call in all client programs. Requires init() to have been called prior to
running.
<Arguments>
*args
<Exceptions>
None
<Side Effects>
Persistently writes to a log file.
<Returns>
None
"""
# Write logfile header
config['logfile'] = open(config['logfilename'], 'w')
config['logfile'].write('################################################\n')
config['logfile'].write('## Overlord Deployment and Monitoring Log ##\n')
config['logfile'].write('################################################\n\n')
config['logfile'].write('GENI user: '******'identity']['username'] + '\n')
config['logfile'].write('Vessels to monitor: ' + str(config['vesselcount']) + '\n')
config['logfile'].write('Time of script start: ' + str(time.time()) + '\n\n')
config['logfile'].flush()
# Release any preallocated vessels
vesselhandle_list = explib.seattlegeni_get_acquired_vessels(config['identity'])
release_vessels(vesselhandle_list, 'Releasing ' + str(len(vesselhandle_list)) + ' preallocated vessels...')
# Acquire an initial sample of vessels
config['logfile'].write(str(time.time()) + ': Fetching initial batch of ' + str(config['vesselcount']) + ' vessels:\n')
config['logfile'].flush()
vesselhandle_list = []
while not vesselhandle_list:
vesselhandle_list = acquire_vessels(config['vesselcount'])
# Upload program to vessels
vesselhandle_list = upload_to_vessels(vesselhandle_list, config['program_filename'])
# Run program on vessels
vesselhandle_list, failed_list = run_on_vessels(vesselhandle_list,
config['program_filename'],
*args)
# Release any failed vessels
if failed_list:
config['logfile'].write(str(time.time()) + ': Running ' + config['program_filename'] + ' failed on ' + str(len(failed_list)) + ' vessels\n')
# Get details about failed vessel(s) and log them
for vh in failed_list:
try:
vessel_log = explib.get_vessel_log(vh, config['identity'])
except:
vessel_log = '[ERROR: vessel log fetch failed]'
nodeid, vesselname = explib.get_nodeid_and_vesselname(vh)
nodelocation = explib.get_node_location(nodeid)
# Log the vessel's log contents
config['logfile'].write('Log contents of failed vessel at ' + nodelocation + ': ' + vessel_log + '\n')
config['logfile'].flush()
# Release the failed vessels
release_vessels(failed_list, 'Releasing failed vessel(s)...')
# Initialize counter variable for loop iterations
loop_iterations = 0
PREPPED = True
print "PREPPED!"
print "Vessel Handles: %s" % vesselhandle_list
# Main loop
while KEEP_RUNNING == True:
print "Starting Loop!"
# Check for vessels not in started state
stopped_vessel_list = []
for vh in vesselhandle_list:
try:
vessel_status = explib.get_vessel_status(vh, config['identity'])
log = explib.get_vessel_log(vh, config['identity'])
print "Loop Log: %s" % log
except:
# Node lookup failed, so remove vessel from vesselhandle_list
# TODO: proper way to handle failed advertisements?
stopped_vessel_list.append(vh)
else:
if vessel_status != explib.VESSEL_STATUS_STARTED:
stopped_vessel_list.append(vh)
# Release and replace any stopped vessels
if stopped_vessel_list:
# Release any stopped vessels
release_vessels(stopped_vessel_list, 'Releasing ' + str(len(stopped_vessel_list)) + ' stopped vessel(s)...')
# Remove released vessels from vesselhandle_list
vesselhandle_list = list_difference(vesselhandle_list, stopped_vessel_list)
# Ensure that enough vessels are running
if len(vesselhandle_list) < config['vesselcount']:
# If there aren't enough active vessels, acquire some
config['logfile'].write(str(time.time()) + ': Only ' + str(len(vesselhandle_list)) + ' vessel(s) out of target ' + str(config['vesselcount']) + ' detected\n')
config['logfile'].flush()
fresh_vessels = acquire_vessels(config['vesselcount'] - len(vesselhandle_list))
# Upload and run program to/on fresh vessels
fresh_vessels = upload_to_vessels(fresh_vessels, config['program_filename'])
success_list, failed_list = run_on_vessels(fresh_vessels,
config['program_filename'],
*args)
# Release any failed vessels
if failed_list:
config['logfile'].write(str(time.time()) + ': Running ' + config['program_filename'] + ' failed on ' + str(len(failed_list)) + ' vessels\n')
# Get details about failed vessel(s) and log them
for vh in failed_list:
try:
vessel_log = explib.get_vessel_log(vh, config['identity'])
except:
vessel_log = '[ERROR: vessel log fetch failed]'
nodeid, vesselname = explib.get_nodeid_and_vesselname(vh)
nodelocation = explib.get_node_location(nodeid)
# Log the vessel's log contents
config['logfile'].write('Log contents of failed vessel at ' + nodelocation + ': ' + vessel_log + '\n')
config['logfile'].flush()
# Release the failed vessels
release_vessels(failed_list, 'Releasing failed vessel(s)...')
# Remove released vessels from fresh_vessels list
fresh_vessels = list_difference(fresh_vessels, failed_list)
# Add fresh_vessels to vesselhandle_list
vesselhandle_list.extend(fresh_vessels)
# Sleep for parameterized amount of time
time.sleep(VESSEL_POLLING_TIME)
# Log a liveness message every certain number of iterations
loop_iterations += 1
if loop_iterations % LOG_AFTER_THIS_MANY_LOOPS == 0:
config['logfile'].write(str(time.time()) + ': Still alive...\n')
config['logfile'].flush()
# Renew vessels according to constant period
if loop_iterations * VESSEL_POLLING_TIME > VESSEL_RENEWAL_PERIOD:
explib.seattlegeni_renew_vessels(config['identity'], vesselhandle_list)
loop_iterations = 0
def run(self, *args):
"""
<Purpose>
Starts the deployment and monitoring of a service on a number of vessels.
Handles all acquisition of, uploading to, starting, and release of vessels.
Contains the main loop of this program, and is thus the final function to
call in all client programs.
<Arguments>
*args:
Optional arguments that well be passed when starting the uploaded
program file, by default
<Exceptions>
Common SeattleGENI exceptions described in the module comments.
<Side Effects>
Unless overridden beforehand, persistently logs to overlord_instance.log and
console
Returns if a 'stop' file is found in directory of the program importing the API
<Returns>
None
"""
# Log setup information for user reference.
self.logger.info("Performing actions as GENI user '" +
self.config['identity']['username'] + "'")
self.logger.info('Vessel status will be polled every ' +
str(self.VESSEL_POLLING_PERIOD) + ' seconds')
self.logger.info('Vessels will be renewed every ' +
str(self.VESSEL_RENEWAL_PERIOD) + ' seconds')
# Run the start-up function for overlord
self.init_overlord_func(self)
# Retrieve the current list of vessel handles owned by identity, regardless
# of the results of overlord's start-up function
vessel_handlers = explib.seattlegeni_get_acquired_vessels(
self.config['identity'])
# If there are still vessels allocated to the identity, renew them to keep
# renewal periods consistent with newly acquired vessels later on
if vessel_handlers:
self.logger.info('Renewing ' + str(len(vessel_handlers)) +
' vessels')
explib.seattlegeni_renew_vessels(self.config['identity'],
vessel_handlers)
# Turn VESSEL_RENEWAL_PERIOD into a timedelta object.
renewal_delay = timedelta(seconds=self.VESSEL_RENEWAL_PERIOD)
# Assume that the vessels have been renewed.
last_renewal = datetime.utcnow()
# Main loop
while True:
# End operations if "stop" file is found in same directory as overlord.py
if os.path.isfile("./stop"):
self.logger.info("Stop file found; Discontinuing operations.")
return
# Call the acquire vessels function and set the list of the handles of the
# newly acquired vessels
fresh_handlers = self.acquire_vessels_func(self, vessel_handlers)
# If the list is not empty, call the initiate vessel function, and an updated
# list of currently owned vessel handles that includes the newly acquired ones
# should be returned
if fresh_handlers:
vessel_handlers = self.init_vessels_func(
self, fresh_handlers, vessel_handlers, *args)
# Weed out any vessels based on the remove vessels function
vessel_handlers = self.remove_vessels_func(self, vessel_handlers)
# Perform any maintenance on the vessels, including renewals if needed,
# and return the relevant information needed for next maintenance.
# Currently coded to fit with default behavior, but should somehow be made a
# little more flexible for user's function overrides?
last_renewal = self.maintenance_func(self, renewal_delay,
last_renewal, vessel_handlers)
# If the list of running vessels does not match the requested vessel count,
# repeat the loop until it does
if not len(vessel_handlers) == self.config['vessel_count']:
continue
# Sleep between polling vessel statuses.
time.sleep(self.VESSEL_POLLING_PERIOD)
def reset_vessels(): vesselhandle_list = explib.seattlegeni_get_acquired_vessels(config['identity']) release_vessels(vesselhandle_list, 'Releasing ' + str(len(vesselhandle_list)) + ' preallocated vessels...')
def get_vessels(): print "Config: %s" % config return explib.seattlegeni_get_acquired_vessels(config['identity'])
def main():
identity = experimentlib.create_identity_from_key_files(PUBLICKEY_FILENAME, PRIVATEKEY_FILENAME)
# Get a list of nodes advertising under the public key. Some of these
# nodes may be unreachable or may have gone offline since they advertised.
# This won't try to communicate with the actual nodes.
nodelocation_list = experimentlib.lookup_node_locations_by_identity(identity)
print("nodelocation_list:" + str(nodelocation_list))
# Talk to each advertising node to find out which vessels we have on each.
# We get back a list of dictionaries, where each dictionary describes a
# vessel. There may be multiple vessels from any single node in this list.
active_vesselhandle_list = experimentlib.find_vessels_on_nodes(identity, nodelocation_list)
print("active_vesselhandle_list:" + str(active_vesselhandle_list))
# Now we want to find out which vessels we've acquired through seattlegeni
# that are not in our list. We may, for example, want to release those
# vessels because we consider them unusable.
try:
expected_vesselhandle_list = experimentlib.seattlegeni_get_acquired_vessels(identity)
print("expected_vesselhandle_list:" + str(expected_vesselhandle_list))
# If we already have enough usable vessels, we're done.
if len(active_vesselhandle_list) >= MIN_VESSELS_TO_KEEP:
print(
"There are already "
+ str(len(active_vesselhandle_list))
+ " active vessels "
+ "and our MIN_VESSELS_TO_KEEP is "
+ str(MIN_VESSELS_TO_KEEP)
)
return
# We assume all of our vessels come from seattlegeni, so if any vessels we
# should have access to according to seattlegeni aren't accessible or
# otherwise usable, then release them.
vesselhandles_to_release = []
for vesselhandle in expected_vesselhandle_list:
if vesselhandle not in active_vesselhandle_list:
vesselhandles_to_release.append(vesselhandle)
if vesselhandles_to_release:
print(
str(len(vesselhandles_to_release))
+ " vessels were inaccessible, so will try to release them:"
+ str(vesselhandles_to_release)
)
try:
experimentlib.seattlegeni_release_vessels(identity, vesselhandles_to_release)
except experimentlib.SeattleClearinghouseError, e:
print("Failed to release vessels: " + str(e))
else:
print("Vessels successfully released.")
# Determine the maximum number of vessels we can acquire through seattlegeni.
max_vessels_allowed = experimentlib.seattlegeni_max_vessels_allowed(identity)
print("max_vessels_allowed: " + str(max_vessels_allowed))
# Determine the number of vessels we already have acquired through seattlegeni.
num_currently_acquired = len(experimentlib.seattlegeni_get_acquired_vessels(identity))
print("currently_acquired_count: " + str(num_currently_acquired))
# Let's try to get as close to MIN_VESSELS_TO_KEEP without requesting more
# than is allowed by this account.
num_vessels_to_request = min(max_vessels_allowed, MIN_VESSELS_TO_KEEP) - num_currently_acquired
if num_vessels_to_request <= 0:
print("This account doesn't have enough vessel credits to request more vessels.")
return
else:
print("Will try to acquire " + str(num_vessels_to_request) + " vessels.")
vessel_type = experimentlib.SEATTLECLEARINGHOUSE_VESSEL_TYPE_WAN
acquired_vessels = experimentlib.seattlegeni_acquire_vessels(identity, vessel_type, num_vessels_to_request)
print("Acquired " + str(num_vessels_to_request) + " vessels: " + str(acquired_vessels))
def default_init_overlord(overlord):
# Release any pre-allocated vessels
vessel_handlers = explib.seattlegeni_get_acquired_vessels(overlord.config["identity"])
overlord.logger.info("Releasing " + str(len(vessel_handlers)) + " pre-allocated vessels")
overlord.release_vessels(vessel_handlers)
def run(self, *args):
"""
<Purpose>
Starts the deployment and monitoring of a service on a number of vessels.
Handles all acquisition of, uploading to, starting, and release of vessels.
Contains the main loop of this program, and is thus the final function to
call in all client programs.
<Arguments>
*args:
Optional arguments that well be passed when starting the uploaded
program file, by default
<Exceptions>
Common SeattleGENI exceptions described in the module comments.
<Side Effects>
Unless overridden beforehand, persistently logs to overlord_instance.log and
console
Returns if a 'stop' file is found in directory of the program importing the API
<Returns>
None
"""
# Log setup information for user reference.
self.logger.info("Performing actions as GENI user '" + self.config["identity"]["username"] + "'")
self.logger.info("Vessel status will be polled every " + str(self.VESSEL_POLLING_PERIOD) + " seconds")
self.logger.info("Vessels will be renewed every " + str(self.VESSEL_RENEWAL_PERIOD) + " seconds")
# Run the start-up function for overlord
self.init_overlord_func(self)
# Retrieve the current list of vessel handles owned by identity, regardless
# of the results of overlord's start-up function
vessel_handlers = explib.seattlegeni_get_acquired_vessels(self.config["identity"])
# If there are still vessels allocated to the identity, renew them to keep
# renewal periods consistent with newly acquired vessels later on
if vessel_handlers:
self.logger.info("Renewing " + str(len(vessel_handlers)) + " vessels")
explib.seattlegeni_renew_vessels(self.config["identity"], vessel_handlers)
# Turn VESSEL_RENEWAL_PERIOD into a timedelta object.
renewal_delay = timedelta(seconds=self.VESSEL_RENEWAL_PERIOD)
# Assume that the vessels have been renewed.
last_renewal = datetime.utcnow()
# Main loop
while True:
# End operations if "stop" file is found in same directory as overlord.py
if os.path.isfile("./stop"):
self.logger.info("Stop file found; Discontinuing operations.")
return
# Call the acquire vessels function and set the list of the handles of the
# newly acquired vessels
fresh_handlers = self.acquire_vessels_func(self, vessel_handlers)
# If the list is not empty, call the initiate vessel function, and an updated
# list of currently owned vessel handles that includes the newly acquired ones
# should be returned
if fresh_handlers:
vessel_handlers = self.init_vessels_func(self, fresh_handlers, vessel_handlers, *args)
# Weed out any vessels based on the remove vessels function
vessel_handlers = self.remove_vessels_func(self, vessel_handlers)
# Perform any maintenance on the vessels, including renewals if needed,
# and return the relevant information needed for next maintenance.
# Currently coded to fit with default behavior, but should somehow be made a
# little more flexible for user's function overrides?
last_renewal = self.maintenance_func(self, renewal_delay, last_renewal, vessel_handlers)
# If the list of running vessels does not match the requested vessel count,
# repeat the loop until it does
if not len(vessel_handlers) == self.config["vessel_count"]:
continue
# Sleep between polling vessel statuses.
time.sleep(self.VESSEL_POLLING_PERIOD)
# SeattleGENI. If SeattleClearinghouseError exception thrown, print warning.
try:
CBFUNC_CONTEXT['geniport'] = experimentlib.seattlegeni_user_port(CBFUNC_CONTEXT['identity'])
except Exception, e:
if repr(e).startswith("SeattleClearinghouseError"):
print """Error: Cannot make secure SSL connection to SeattleGENI
Please make sure that:
* M2Crypto is installed
* the current directory contains a valid PEM file containing CA certificates
To run appmap.py without SSL authentication, use the `--insecure` switch. Check documentation for more details."""
sys.exit()
# Get list of vessels user has previously allocated
CBFUNC_CONTEXT['vesselhandle_list'] = experimentlib.seattlegeni_get_acquired_vessels(CBFUNC_CONTEXT['identity'])
if appmap_config['num_nodes'] > 0 and len(CBFUNC_CONTEXT['vesselhandle_list']) > 0:
# Release any vessels previously acquired from GENI
experimentlib.seattlegeni_release_vessels(CBFUNC_CONTEXT['identity'], CBFUNC_CONTEXT['vesselhandle_list'])
if appmap_config['num_nodes'] > 0:
# Acquire argument number of vessels using seattlegeni interface
CBFUNC_CONTEXT['vesselhandle_list'] = experimentlib.seattlegeni_acquire_vessels(CBFUNC_CONTEXT['identity'], "wan", int(appmap_config['num_nodes']))
# If no vessels were acquired, an error occurred and script exits
if len(CBFUNC_CONTEXT['vesselhandle_list']) == 0:
print "Error: No vessels acquired. Try rerunning " + sys.argv[0]
sys.exit()
nodelocation_list = []
def main():
identity = experimentlib.create_identity_from_key_files(
PUBLICKEY_FILENAME, PRIVATEKEY_FILENAME)
# Get a list of nodes advertising under the public key. Some of these
# nodes may be unreachable or may have gone offline since they advertised.
# This won't try to communicate with the actual nodes.
nodelocation_list = experimentlib.lookup_node_locations_by_identity(
identity)
print("nodelocation_list:" + str(nodelocation_list))
# Talk to each advertising node to find out which vessels we have on each.
# We get back a list of dictionaries, where each dictionary describes a
# vessel. There may be multiple vessels from any single node in this list.
active_vesselhandle_list = experimentlib.find_vessels_on_nodes(
identity, nodelocation_list)
print("active_vesselhandle_list:" + str(active_vesselhandle_list))
# Now we want to find out which vessels we've acquired through seattlegeni
# that are not in our list. We may, for example, want to release those
# vessels because we consider them unusable.
try:
expected_vesselhandle_list = experimentlib.seattlegeni_get_acquired_vessels(
identity)
print("expected_vesselhandle_list:" + str(expected_vesselhandle_list))
# If we already have enough usable vessels, we're done.
if len(active_vesselhandle_list) >= MIN_VESSELS_TO_KEEP:
print("There are already " + str(len(active_vesselhandle_list)) +
" active vessels " + "and our MIN_VESSELS_TO_KEEP is " +
str(MIN_VESSELS_TO_KEEP))
return
# We assume all of our vessels come from seattlegeni, so if any vessels we
# should have access to according to seattlegeni aren't accessible or
# otherwise usable, then release them.
vesselhandles_to_release = []
for vesselhandle in expected_vesselhandle_list:
if vesselhandle not in active_vesselhandle_list:
vesselhandles_to_release.append(vesselhandle)
if vesselhandles_to_release:
print(
str(len(vesselhandles_to_release)) +
" vessels were inaccessible, so will try to release them:" +
str(vesselhandles_to_release))
try:
experimentlib.seattlegeni_release_vessels(
identity, vesselhandles_to_release)
except experimentlib.SeattleClearinghouseError, e:
print("Failed to release vessels: " + str(e))
else:
print("Vessels successfully released.")
# Determine the maximum number of vessels we can acquire through seattlegeni.
max_vessels_allowed = experimentlib.seattlegeni_max_vessels_allowed(
identity)
print("max_vessels_allowed: " + str(max_vessels_allowed))
# Determine the number of vessels we already have acquired through seattlegeni.
num_currently_acquired = len(
experimentlib.seattlegeni_get_acquired_vessels(identity))
print("currently_acquired_count: " + str(num_currently_acquired))
# Let's try to get as close to MIN_VESSELS_TO_KEEP without requesting more
# than is allowed by this account.
num_vessels_to_request = min(
max_vessels_allowed, MIN_VESSELS_TO_KEEP) - num_currently_acquired
if num_vessels_to_request <= 0:
print(
"This account doesn't have enough vessel credits to request more vessels."
)
return
else:
print("Will try to acquire " + str(num_vessels_to_request) +
" vessels.")
vessel_type = experimentlib.SEATTLECLEARINGHOUSE_VESSEL_TYPE_WAN
acquired_vessels = experimentlib.seattlegeni_acquire_vessels(
identity, vessel_type, num_vessels_to_request)
print("Acquired " + str(num_vessels_to_request) + " vessels: " +
str(acquired_vessels))
