def main():
identity = experimentlib.create_identity_from_key_files(PUBLICKEY_FILENAME, PRIVATEKEY_FILENAME)
nodelocation_list = experimentlib.lookup_node_locations_by_identity(identity)
print("Number of advertising nodes: " + str(len(nodelocation_list)))
vesselhandle_list = experimentlib.find_vessels_on_nodes(identity, nodelocation_list)
print("Number of active vessels: " + str(len(vesselhandle_list)))
# Note that we could use experimentlib.run_parallelized() to parallelize
# this, but for simplicity we do each sequentially.
for vesselhandle in vesselhandle_list:
try:
nodeid, vesselname = experimentlib.get_nodeid_and_vesselname(vesselhandle)
nodelocation = experimentlib.get_node_location(nodeid)
# Note: you may want to reset_vessel().
experimentlib.upload_file_to_vessel(vesselhandle, identity, PROGRAM_FILE)
print("Uploaded " + PROGRAM_FILE + " to " + nodelocation + " vessel " + vesselname)
experimentlib.start_vessel(vesselhandle, identity, PROGRAM_FILE, ARGUMENTS_TO_START_PROGRAM_WITH)
print("Program started on " + nodelocation + " vessel " + vesselname)
except experimentlib.SeattleExperimentError, e:
print("Failure on vessel " + vesselhandle + ". Error was: " + str(e))
def main():
identity = experimentlib.create_identity_from_key_files(PUBLICKEY_FILENAME, PRIVATEKEY_FILENAME)
nodelocation_list = experimentlib.lookup_node_locations_by_identity(identity)
print("Number of advertising nodes: " + str(len(nodelocation_list)))
vesselhandle_list = experimentlib.find_vessels_on_nodes(identity, nodelocation_list)
print("Number of active vessels: " + str(len(vesselhandle_list)))
# Note that we could use experimentlib.run_parallelized() to parallelize the
# uploading of files and speed things up, but for simplicity we do each
# upload sequentially.
for vesselhandle in vesselhandle_list:
try:
nodeid, vesselname = experimentlib.get_nodeid_and_vesselname(vesselhandle)
nodelocation = experimentlib.get_node_location(nodeid)
for local_filename in FILES_TO_UPLOAD:
experimentlib.upload_file_to_vessel(vesselhandle, identity, local_filename)
print("Uploaded " + local_filename + " to " + nodelocation + " vessel " + vesselname)
except experimentlib.SeattleExperimentError, e:
print("Failure on vessel " + vesselhandle + ". Error was: " + str(e))
def get_node_location_or_unknown(vesselhandle):
"""
A helper function that will return the location of the node a vessel is on or
the string "Unknown" if the location of a vessel cannot be determined.
"""
nodeid, vesselname = experimentlib.get_nodeid_and_vesselname(vesselhandle)
try:
return experimentlib.get_node_location(nodeid)
except experimentlib.NodeLocationNotAdvertisedError, e:
print(str(e))
return "Unknown"
def get_node_location_or_unknown(vesselhandle):
"""
A helper function that will return the location of the node a vessel is on or
the string "Unknown" if the location of a vessel cannot be determined.
"""
nodeid, vesselname = experimentlib.get_nodeid_and_vesselname(vesselhandle)
try:
return experimentlib.get_node_location(nodeid)
except experimentlib.NodeLocationNotAdvertisedError, e:
print(str(e))
return "Unknown"
def main():
identity = experimentlib.create_identity_from_key_files(
PUBLICKEY_FILENAME, PRIVATEKEY_FILENAME)
nodelocation_list = experimentlib.lookup_node_locations_by_identity(
identity)
print("Number of advertising nodes: " + str(len(nodelocation_list)))
vesselhandle_list = experimentlib.find_vessels_on_nodes(
identity, nodelocation_list)
print("Number of active vessels: " + str(len(vesselhandle_list)))
for vesselhandle in vesselhandle_list:
try:
# Generate the vessel directory name with format: host_port_vesselname
nodeid, vesselname = experimentlib.get_nodeid_and_vesselname(
vesselhandle)
nodelocation = experimentlib.get_node_location(nodeid)
host, port = experimentlib.get_host_and_port(nodelocation)
dirname = host + "_" + str(port) + "_" + vesselname
# Create the vessel directory if it doesn't already exist.
vessel_directory = os.path.join(DOWNLOAD_DIRECTORY, dirname)
if not os.path.exists(vessel_directory):
os.mkdir(vessel_directory)
# Get a list of files on the vessel.
filelist = experimentlib.get_vessel_file_list(
vesselhandle, identity)
print("Files on " + nodelocation + " " + vesselname + ": " +
str(filelist))
for remote_filename in filelist:
local_filename = os.path.join(vessel_directory,
remote_filename)
experimentlib.download_file_from_vessel(
vesselhandle, identity, remote_filename, local_filename)
print("Downloaded " + local_filename)
except experimentlib.SeattleExperimentError, e:
print("Failure on vessel " + vesselhandle + ". Error was: " +
str(e))
def upload_to_vessels(vesselhandle_list, filename):
"""
<Purpose>
Uploads a file to a set of vessels. A batch wrapper around the Experiment
Library function upload_file_to_vessel, with logging support.
<Arguments>
vesselhandle_list
A list of vesselhandles of vessels to which the file is to be uploaded.
filename
The filename of the file to be uploaded.
<Exceptions>
None
<Side Effects>
None
<Returns>
A list of vessels to which the upload succeeded.
"""
# Log the fact that uploads are occurring
config['logfile'].write(str(time.time()) + ': Uploading ' + filename + ' to '+ str(len(vesselhandle_list)) + ' vessel(s)...\n')
config['logfile'].flush()
# Initially set return list equal to argument vesselhandle list
success_list = vesselhandle_list
failed_list = []
# For each vesselhandle, attempt an upload
for vh in vesselhandle_list:
try:
print "UPLOADING:%s - %s" % (vh, filename)
explib.upload_file_to_vessel(vh, config['identity'], filename)
except explib.NodeCommunicationError, e:
# If upload failed, remove from vesselhandle_list...
success_list.remove(vh)
# ...add to failed_list...
failed_list.append(vh)
# ...and lookup the nodelocation so it can be logged
nodeid, vesselname = explib.get_nodeid_and_vesselname(vh)
nodelocation = explib.get_node_location(nodeid)
config['logfile'].write('Failure on vessel ' + nodelocation + '\n')
config['logfile'].write('Error was: ' + str(e) + '\n')
config['logfile'].flush()
def vessel_location(self, vessel_handler):
"""
<Purpose>
Determine the node location of the given vessel handle
<Arguments>
vessel_handler:
The vesselhandle of the vessel whose location information
is needed
<Exceptions>
None
<Side Effects>
None
<Returns>
A nodelocation
"""
node_id = explib.get_nodeid_and_vesselname(vessel_handler)[0]
return explib.get_node_location(node_id)
def vessel_location(self, vessel_handler):
"""
<Purpose>
Determine the node location of the given vessel handle
<Arguments>
vessel_handler:
The vesselhandle of the vessel whose location information
is needed
<Exceptions>
None
<Side Effects>
None
<Returns>
A nodelocation
"""
node_id = explib.get_nodeid_and_vesselname(vessel_handler)[0]
return explib.get_node_location(node_id)
def main():
identity = experimentlib.create_identity_from_key_files(
PUBLICKEY_FILENAME, PRIVATEKEY_FILENAME)
nodelocation_list = experimentlib.lookup_node_locations_by_identity(
identity)
print("Number of advertising nodes: " + str(len(nodelocation_list)))
vesselhandle_list = experimentlib.find_vessels_on_nodes(
identity, nodelocation_list)
print("Number of active vessels: " + str(len(vesselhandle_list)))
# Note that we could use experimentlib.run_parallelized() to parallelize
# this, but for simplicity we do each sequentially.
for vesselhandle in vesselhandle_list:
try:
nodeid, vesselname = experimentlib.get_nodeid_and_vesselname(
vesselhandle)
nodelocation = experimentlib.get_node_location(nodeid)
# Note: you may want to reset_vessel().
experimentlib.upload_file_to_vessel(vesselhandle, identity,
PROGRAM_FILE)
print("Uploaded " + PROGRAM_FILE + " to " + nodelocation +
" vessel " + vesselname)
experimentlib.start_vessel(vesselhandle, identity, PROGRAM_FILE,
ARGUMENTS_TO_START_PROGRAM_WITH)
print("Program started on " + nodelocation + " vessel " +
vesselname)
except experimentlib.SeattleExperimentError, e:
print("Failure on vessel " + vesselhandle + ". Error was: " +
str(e))
def main():
identity = experimentlib.create_identity_from_key_files(PUBLICKEY_FILENAME, PRIVATEKEY_FILENAME)
nodelocation_list = experimentlib.lookup_node_locations_by_identity(identity)
print("Number of advertising nodes: " + str(len(nodelocation_list)))
vesselhandle_list = experimentlib.find_vessels_on_nodes(identity, nodelocation_list)
print("Number of active vessels: " + str(len(vesselhandle_list)))
for vesselhandle in vesselhandle_list:
try:
# Generate the vessel directory name with format: host_port_vesselname
nodeid, vesselname = experimentlib.get_nodeid_and_vesselname(vesselhandle)
nodelocation = experimentlib.get_node_location(nodeid)
host, port = experimentlib.get_host_and_port(nodelocation)
dirname = host + "_" + str(port) + "_" + vesselname
# Create the vessel directory if it doesn't already exist.
vessel_directory = os.path.join(DOWNLOAD_DIRECTORY, dirname)
if not os.path.exists(vessel_directory):
os.mkdir(vessel_directory)
# Get a list of files on the vessel.
filelist = experimentlib.get_vessel_file_list(vesselhandle, identity)
print("Files on " + nodelocation + " " + vesselname + ": " + str(filelist))
for remote_filename in filelist:
local_filename = os.path.join(vessel_directory, remote_filename)
experimentlib.download_file_from_vessel(vesselhandle, identity, remote_filename, local_filename)
print("Downloaded " + local_filename)
except experimentlib.SeattleExperimentError, e:
print("Failure on vessel " + vesselhandle + ". Error was: " + str(e))
def main():
identity = experimentlib.create_identity_from_key_files(PUBLICKEY_FILENAME, PRIVATEKEY_FILENAME)
nodelocation_list = experimentlib.lookup_node_locations_by_identity(identity)
print("Number of advertising nodes: " + str(len(nodelocation_list)))
vesselhandle_list = experimentlib.find_vessels_on_nodes(identity, nodelocation_list)
print("Number of active vessels: " + str(len(vesselhandle_list)))
for vesselhandle in vesselhandle_list:
try:
nodeid, vesselname = experimentlib.get_nodeid_and_vesselname(vesselhandle)
nodelocation = experimentlib.get_node_location(nodeid)
print("====================================================================")
print("Log from " + nodelocation + " " + vesselname)
logcontents = experimentlib.get_vessel_log(vesselhandle, identity)
print(logcontents)
except experimentlib.SeattleExperimentError, e:
print("Failure on vessel " + vesselhandle + ". Error was: " + str(e))
def main():
identity = experimentlib.create_identity_from_key_files(PUBLICKEY_FILENAME, PRIVATEKEY_FILENAME)
nodelocation_list = experimentlib.lookup_node_locations_by_identity(identity)
print("Number of advertising nodes: " + str(len(nodelocation_list)))
vesselhandle_list = experimentlib.find_vessels_on_nodes(identity, nodelocation_list)
print("Number of active vessels: " + str(len(vesselhandle_list)))
for vesselhandle in vesselhandle_list:
try:
nodeid, vesselname = experimentlib.get_nodeid_and_vesselname(vesselhandle)
nodelocation = experimentlib.get_node_location(nodeid)
print("====================================================================")
print("Log from " + nodelocation + " " + vesselname)
logcontents = experimentlib.get_vessel_log(vesselhandle, identity)
print(logcontents)
except experimentlib.SeattleExperimentError, e:
print("Failure on vessel " + vesselhandle + ". Error was: " + str(e))
def run(*args):
"""
<Purpose>
Starts the deployment and monitoring of a service on a number of vessels.
Handles all acquisition of, uploading to, starting, and release of vessels.
Contains the main loop of this program, and is thus the final function to
call in all client programs. Requires init() to have been called prior to
running.
<Arguments>
*args
<Exceptions>
None
<Side Effects>
Persistently writes to a log file.
<Returns>
None
"""
# Write logfile header
config['logfile'] = open(config['logfilename'], 'w')
config['logfile'].write('################################################\n')
config['logfile'].write('## Overlord Deployment and Monitoring Log ##\n')
config['logfile'].write('################################################\n\n')
config['logfile'].write('GENI user: '******'identity']['username'] + '\n')
config['logfile'].write('Vessels to monitor: ' + str(config['vesselcount']) + '\n')
config['logfile'].write('Time of script start: ' + str(time.time()) + '\n\n')
config['logfile'].flush()
# Release any preallocated vessels
vesselhandle_list = explib.seattlegeni_get_acquired_vessels(config['identity'])
release_vessels(vesselhandle_list, 'Releasing ' + str(len(vesselhandle_list)) + ' preallocated vessels...')
# Acquire an initial sample of vessels
config['logfile'].write(str(time.time()) + ': Fetching initial batch of ' + str(config['vesselcount']) + ' vessels:\n')
config['logfile'].flush()
vesselhandle_list = []
while not vesselhandle_list:
vesselhandle_list = acquire_vessels(config['vesselcount'])
# Upload program to vessels
vesselhandle_list = upload_to_vessels(vesselhandle_list, config['program_filename'])
# Run program on vessels
vesselhandle_list, failed_list = run_on_vessels(vesselhandle_list,
config['program_filename'],
*args)
# Release any failed vessels
if failed_list:
config['logfile'].write(str(time.time()) + ': Running ' + config['program_filename'] + ' failed on ' + str(len(failed_list)) + ' vessels\n')
# Get details about failed vessel(s) and log them
for vh in failed_list:
try:
vessel_log = explib.get_vessel_log(vh, config['identity'])
except:
vessel_log = '[ERROR: vessel log fetch failed]'
nodeid, vesselname = explib.get_nodeid_and_vesselname(vh)
nodelocation = explib.get_node_location(nodeid)
# Log the vessel's log contents
config['logfile'].write('Log contents of failed vessel at ' + nodelocation + ': ' + vessel_log + '\n')
config['logfile'].flush()
# Release the failed vessels
release_vessels(failed_list, 'Releasing failed vessel(s)...')
# Initialize counter variable for loop iterations
loop_iterations = 0
PREPPED = True
print "PREPPED!"
print "Vessel Handles: %s" % vesselhandle_list
# Main loop
while KEEP_RUNNING == True:
print "Starting Loop!"
# Check for vessels not in started state
stopped_vessel_list = []
for vh in vesselhandle_list:
try:
vessel_status = explib.get_vessel_status(vh, config['identity'])
log = explib.get_vessel_log(vh, config['identity'])
print "Loop Log: %s" % log
except:
# Node lookup failed, so remove vessel from vesselhandle_list
# TODO: proper way to handle failed advertisements?
stopped_vessel_list.append(vh)
else:
if vessel_status != explib.VESSEL_STATUS_STARTED:
stopped_vessel_list.append(vh)
# Release and replace any stopped vessels
if stopped_vessel_list:
# Release any stopped vessels
release_vessels(stopped_vessel_list, 'Releasing ' + str(len(stopped_vessel_list)) + ' stopped vessel(s)...')
# Remove released vessels from vesselhandle_list
vesselhandle_list = list_difference(vesselhandle_list, stopped_vessel_list)
# Ensure that enough vessels are running
if len(vesselhandle_list) < config['vesselcount']:
# If there aren't enough active vessels, acquire some
config['logfile'].write(str(time.time()) + ': Only ' + str(len(vesselhandle_list)) + ' vessel(s) out of target ' + str(config['vesselcount']) + ' detected\n')
config['logfile'].flush()
fresh_vessels = acquire_vessels(config['vesselcount'] - len(vesselhandle_list))
# Upload and run program to/on fresh vessels
fresh_vessels = upload_to_vessels(fresh_vessels, config['program_filename'])
success_list, failed_list = run_on_vessels(fresh_vessels,
config['program_filename'],
*args)
# Release any failed vessels
if failed_list:
config['logfile'].write(str(time.time()) + ': Running ' + config['program_filename'] + ' failed on ' + str(len(failed_list)) + ' vessels\n')
# Get details about failed vessel(s) and log them
for vh in failed_list:
try:
vessel_log = explib.get_vessel_log(vh, config['identity'])
except:
vessel_log = '[ERROR: vessel log fetch failed]'
nodeid, vesselname = explib.get_nodeid_and_vesselname(vh)
nodelocation = explib.get_node_location(nodeid)
# Log the vessel's log contents
config['logfile'].write('Log contents of failed vessel at ' + nodelocation + ': ' + vessel_log + '\n')
config['logfile'].flush()
# Release the failed vessels
release_vessels(failed_list, 'Releasing failed vessel(s)...')
# Remove released vessels from fresh_vessels list
fresh_vessels = list_difference(fresh_vessels, failed_list)
# Add fresh_vessels to vesselhandle_list
vesselhandle_list.extend(fresh_vessels)
# Sleep for parameterized amount of time
time.sleep(VESSEL_POLLING_TIME)
# Log a liveness message every certain number of iterations
loop_iterations += 1
if loop_iterations % LOG_AFTER_THIS_MANY_LOOPS == 0:
config['logfile'].write(str(time.time()) + ': Still alive...\n')
config['logfile'].flush()
# Renew vessels according to constant period
if loop_iterations * VESSEL_POLLING_TIME > VESSEL_RENEWAL_PERIOD:
explib.seattlegeni_renew_vessels(config['identity'], vesselhandle_list)
loop_iterations = 0
neighborip_fo = open("neighboriplist.txt", "w")
for ipport in nodelocation_list:
ip = ipport.split(":")[0]
NODEIPLIST.append(ip)
neighborip_fo.write(ip + "\n")
neighborip_fo.close()
# Using experimentlib.run_parallelized() to upload and run programs
uploadips_success, uploadips_fail = experimentlib.run_parallelized(CBFUNC_CONTEXT['vesselhandle_list'], experimentlib.upload_file_to_vessel, CBFUNC_CONTEXT['identity'], "neighboriplist.txt")
uploadprog_success, uploadprog_fail = experimentlib.run_parallelized(CBFUNC_CONTEXT['vesselhandle_list'], experimentlib.upload_file_to_vessel, CBFUNC_CONTEXT['identity'], "pingneighbors.py")
start_success, start_fail = experimentlib.run_parallelized(CBFUNC_CONTEXT['vesselhandle_list'], experimentlib.start_vessel, CBFUNC_CONTEXT['identity'], "pingneighbors.py", [str(CBFUNC_CONTEXT['geniport'])])
# Handle failed nodes
for failed in listops_uniq(uploadips_fail + uploadprog_fail + start_fail):
nodeid, vesselname = experimentlib.get_nodeid_and_vesselname(failed[0])
for vessel in vessel_details:
if vessel["nodeid"] == nodeid:
NODEIPLIST.remove(vessel["nodelocation"][:-5])
print "Failure on vessel " + failed[0] + ". Error was: " + failed[1]
print "Files uploaded to nodes and programs running"
# Now that vessel setup is complete, handle output
geoip_init_client("http://geoip.cs.washington.edu:12679")
if appmap_config['output'] == 'localhost':
# Set up http listener to display results
CBFUNC_CONTEXT['listenerid'] = httpserver_registercallback(("localhost", appmap_config['local_port']), display_results)
print "Browse to 'http://localhost:" + str(appmap_config['local_port']) + "' to view results."
else:
print "Sleeping for 25 seconds to let latencies percolate across vessels."