def default_initiate_vessels(overlord, fresh_handlers, vessel_handlers, *args):
successful_handlers = overlord.upload_to_vessels(fresh_handlers, [overlord.config["program_filename"]])
successful_handlers = overlord.run_on_vessels(successful_handlers, overlord.config["program_filename"], *args)
# Identify and release any failed vessels.
failed_handlers = overlord.list_difference(fresh_handlers, successful_handlers)
if len(failed_handlers) > 0:
overlord.logger.info(
"Releasing "
+ str(len(failed_handlers))
+ " vessels because '"
+ overlord.config["program_filename"]
+ "' failed to upload or run"
)
for vessel in failed_handlers:
try:
vessel_log = explib.get_vessel_log(vessel, overlord.config["identity"])
if vessel_log == "":
vessel_log = "[empty vessel log]"
except:
vessel_log = "[no vessel log available]"
overlord.logger.error("Vessel " + overlord.vessel_location(vessel) + " failed: " + vessel_log)
# Release all the failed vessels.
overlord.release_vessels(failed_handlers)
# Merge the fresh vessels into the overall list.
vessel_handlers.extend(successful_handlers)
return vessel_handlers
def default_initiate_vessels(overlord, fresh_handlers, vessel_handlers, *args):
successful_handlers = overlord.upload_to_vessels(
fresh_handlers, [overlord.config['program_filename']])
successful_handlers = overlord.run_on_vessels(
successful_handlers, overlord.config['program_filename'], *args)
# Identify and release any failed vessels.
failed_handlers = overlord.list_difference(fresh_handlers,
successful_handlers)
if len(failed_handlers) > 0:
overlord.logger.info("Releasing " + str(len(failed_handlers)) +
" vessels because '" +
overlord.config['program_filename'] +
"' failed to upload or run")
for vessel in failed_handlers:
try:
vessel_log = explib.get_vessel_log(vessel,
overlord.config['identity'])
if vessel_log == '':
vessel_log = '[empty vessel log]'
except:
vessel_log = '[no vessel log available]'
overlord.logger.error('Vessel ' +
overlord.vessel_location(vessel) +
' failed: ' + vessel_log)
# Release all the failed vessels.
overlord.release_vessels(failed_handlers)
# Merge the fresh vessels into the overall list.
vessel_handlers.extend(successful_handlers)
return vessel_handlers
def main():
identity = experimentlib.create_identity_from_key_files(PUBLICKEY_FILENAME, PRIVATEKEY_FILENAME)
nodelocation_list = experimentlib.lookup_node_locations_by_identity(identity)
print("Number of advertising nodes: " + str(len(nodelocation_list)))
vesselhandle_list = experimentlib.find_vessels_on_nodes(identity, nodelocation_list)
print("Number of active vessels: " + str(len(vesselhandle_list)))
for vesselhandle in vesselhandle_list:
try:
nodeid, vesselname = experimentlib.get_nodeid_and_vesselname(vesselhandle)
nodelocation = experimentlib.get_node_location(nodeid)
print("====================================================================")
print("Log from " + nodelocation + " " + vesselname)
logcontents = experimentlib.get_vessel_log(vesselhandle, identity)
print(logcontents)
except experimentlib.SeattleExperimentError, e:
print("Failure on vessel " + vesselhandle + ". Error was: " + str(e))
def main():
identity = experimentlib.create_identity_from_key_files(PUBLICKEY_FILENAME, PRIVATEKEY_FILENAME)
nodelocation_list = experimentlib.lookup_node_locations_by_identity(identity)
print("Number of advertising nodes: " + str(len(nodelocation_list)))
vesselhandle_list = experimentlib.find_vessels_on_nodes(identity, nodelocation_list)
print("Number of active vessels: " + str(len(vesselhandle_list)))
for vesselhandle in vesselhandle_list:
try:
nodeid, vesselname = experimentlib.get_nodeid_and_vesselname(vesselhandle)
nodelocation = experimentlib.get_node_location(nodeid)
print("====================================================================")
print("Log from " + nodelocation + " " + vesselname)
logcontents = experimentlib.get_vessel_log(vesselhandle, identity)
print(logcontents)
except experimentlib.SeattleExperimentError, e:
print("Failure on vessel " + vesselhandle + ". Error was: " + str(e))
run.start()
# Wait a bit, overlord needs its time if you don't want it to throw up everywhere
time.sleep(60)
# Get the geoipserver so that we can map the ips of our nodes to actual locations
locations = {}
geoipserver = xmlrpclib.ServerProxy(GEO_IP_SERVER)
for vessel in overlord.get_vessels():
nodeid, vesselname = vessel.split(":")
location = experimentlib.get_node_location(nodeid).split(":")[0]
locations[location] = [vessel]
if debug:
try:
log = experimentlib.get_vessel_log(vessel, config['identity'])
print "Log for %s: %s" (location, log)
except:
print "Unnexpected Error: %s" % sys.exc_info()[0]
try:
loc = geoipserver.record_by_addr(location)
locations[location].append("%s - %s" % (loc['city'], loc['country_name']))
except:
locations[location].append("%s (No Location Data)" % location)
conn = None
# Connect to our local database, which contains the user agent strings and their associated OSs.
try:
conn = sqlite3.connect('viewpoints.db')
except:
print("Error: Database connection failure. Cannot launch viewpoints")
def run(*args):
"""
<Purpose>
Starts the deployment and monitoring of a service on a number of vessels.
Handles all acquisition of, uploading to, starting, and release of vessels.
Contains the main loop of this program, and is thus the final function to
call in all client programs. Requires init() to have been called prior to
running.
<Arguments>
*args
<Exceptions>
None
<Side Effects>
Persistently writes to a log file.
<Returns>
None
"""
# Write logfile header
config['logfile'] = open(config['logfilename'], 'w')
config['logfile'].write('################################################\n')
config['logfile'].write('## Overlord Deployment and Monitoring Log ##\n')
config['logfile'].write('################################################\n\n')
config['logfile'].write('GENI user: '******'identity']['username'] + '\n')
config['logfile'].write('Vessels to monitor: ' + str(config['vesselcount']) + '\n')
config['logfile'].write('Time of script start: ' + str(time.time()) + '\n\n')
config['logfile'].flush()
# Release any preallocated vessels
vesselhandle_list = explib.seattlegeni_get_acquired_vessels(config['identity'])
release_vessels(vesselhandle_list, 'Releasing ' + str(len(vesselhandle_list)) + ' preallocated vessels...')
# Acquire an initial sample of vessels
config['logfile'].write(str(time.time()) + ': Fetching initial batch of ' + str(config['vesselcount']) + ' vessels:\n')
config['logfile'].flush()
vesselhandle_list = []
while not vesselhandle_list:
vesselhandle_list = acquire_vessels(config['vesselcount'])
# Upload program to vessels
vesselhandle_list = upload_to_vessels(vesselhandle_list, config['program_filename'])
# Run program on vessels
vesselhandle_list, failed_list = run_on_vessels(vesselhandle_list,
config['program_filename'],
*args)
# Release any failed vessels
if failed_list:
config['logfile'].write(str(time.time()) + ': Running ' + config['program_filename'] + ' failed on ' + str(len(failed_list)) + ' vessels\n')
# Get details about failed vessel(s) and log them
for vh in failed_list:
try:
vessel_log = explib.get_vessel_log(vh, config['identity'])
except:
vessel_log = '[ERROR: vessel log fetch failed]'
nodeid, vesselname = explib.get_nodeid_and_vesselname(vh)
nodelocation = explib.get_node_location(nodeid)
# Log the vessel's log contents
config['logfile'].write('Log contents of failed vessel at ' + nodelocation + ': ' + vessel_log + '\n')
config['logfile'].flush()
# Release the failed vessels
release_vessels(failed_list, 'Releasing failed vessel(s)...')
# Initialize counter variable for loop iterations
loop_iterations = 0
PREPPED = True
print "PREPPED!"
print "Vessel Handles: %s" % vesselhandle_list
# Main loop
while KEEP_RUNNING == True:
print "Starting Loop!"
# Check for vessels not in started state
stopped_vessel_list = []
for vh in vesselhandle_list:
try:
vessel_status = explib.get_vessel_status(vh, config['identity'])
log = explib.get_vessel_log(vh, config['identity'])
print "Loop Log: %s" % log
except:
# Node lookup failed, so remove vessel from vesselhandle_list
# TODO: proper way to handle failed advertisements?
stopped_vessel_list.append(vh)
else:
if vessel_status != explib.VESSEL_STATUS_STARTED:
stopped_vessel_list.append(vh)
# Release and replace any stopped vessels
if stopped_vessel_list:
# Release any stopped vessels
release_vessels(stopped_vessel_list, 'Releasing ' + str(len(stopped_vessel_list)) + ' stopped vessel(s)...')
# Remove released vessels from vesselhandle_list
vesselhandle_list = list_difference(vesselhandle_list, stopped_vessel_list)
# Ensure that enough vessels are running
if len(vesselhandle_list) < config['vesselcount']:
# If there aren't enough active vessels, acquire some
config['logfile'].write(str(time.time()) + ': Only ' + str(len(vesselhandle_list)) + ' vessel(s) out of target ' + str(config['vesselcount']) + ' detected\n')
config['logfile'].flush()
fresh_vessels = acquire_vessels(config['vesselcount'] - len(vesselhandle_list))
# Upload and run program to/on fresh vessels
fresh_vessels = upload_to_vessels(fresh_vessels, config['program_filename'])
success_list, failed_list = run_on_vessels(fresh_vessels,
config['program_filename'],
*args)
# Release any failed vessels
if failed_list:
config['logfile'].write(str(time.time()) + ': Running ' + config['program_filename'] + ' failed on ' + str(len(failed_list)) + ' vessels\n')
# Get details about failed vessel(s) and log them
for vh in failed_list:
try:
vessel_log = explib.get_vessel_log(vh, config['identity'])
except:
vessel_log = '[ERROR: vessel log fetch failed]'
nodeid, vesselname = explib.get_nodeid_and_vesselname(vh)
nodelocation = explib.get_node_location(nodeid)
# Log the vessel's log contents
config['logfile'].write('Log contents of failed vessel at ' + nodelocation + ': ' + vessel_log + '\n')
config['logfile'].flush()
# Release the failed vessels
release_vessels(failed_list, 'Releasing failed vessel(s)...')
# Remove released vessels from fresh_vessels list
fresh_vessels = list_difference(fresh_vessels, failed_list)
# Add fresh_vessels to vesselhandle_list
vesselhandle_list.extend(fresh_vessels)
# Sleep for parameterized amount of time
time.sleep(VESSEL_POLLING_TIME)
# Log a liveness message every certain number of iterations
loop_iterations += 1
if loop_iterations % LOG_AFTER_THIS_MANY_LOOPS == 0:
config['logfile'].write(str(time.time()) + ': Still alive...\n')
config['logfile'].flush()
# Renew vessels according to constant period
if loop_iterations * VESSEL_POLLING_TIME > VESSEL_RENEWAL_PERIOD:
explib.seattlegeni_renew_vessels(config['identity'], vesselhandle_list)
loop_iterations = 0
