def load_identities(directory): """ <Purpose> Loads all the public keys from a given directory and returns a dictionary of identities for those public keys. <Arguments> directory: The directory to search in for public keys. <Exceptions> If directory isn't a valid directory or one of the publickey files is improperly formated, an exception will be raised. <Side Effects> None. <Returns> A dictionary with public key names (the file name minus the ending .publickey) mapping to identities for the public keys. """ identitydict = {} filelist = os.listdir(directory) for entry in filelist: if entry.endswith(".publickey"): identity = experimentlib.create_identity_from_key_files(directory + "/" + entry) keyname = entry[:-len(".publickey")] identitydict[keyname] = identity oldlocationdict[keyname] = [] return identitydict
def load_identities(directory): """ <Purpose> Loads all the public keys from a given directory and returns a dictionary of identities for those public keys. <Arguments> directory: The directory to search in for public keys. <Exceptions> If directory isn't a valid directory or one of the publickey files is improperly formated, an exception will be raised. <Side Effects> None. <Returns> A dictionary with public key names (the file name minus the ending .publickey) mapping to identities for the public keys. """ identitydict = {} filelist = os.listdir(directory) for entry in filelist: if entry.endswith(".publickey"): identity = experimentlib.create_identity_from_key_files( directory + "/" + entry) keyname = entry[:-len(".publickey")] identitydict[keyname] = identity oldlocationdict[keyname] = [] return identitydict
def _beta_keys(): """ <Purpose> To generate a dictionary of beta keys for checking to see if a node is a production node or a beta node based off of what user keys its vessels have. <Argument> None. <Exceptions> Possible IO errors if keys have not been downloaded to their correct locations. <Side Effects> None. <Return> Returns a dictionary of beta keys in the form of {'key_name':publickey_dict}. """ beta_keys_dict = {} beta_key_names = [ 'acceptdonation', 'canonical', 'movingtoonepercent_manyevents', 'onepercentmanyevents' ] for name in beta_key_names: beta_keys_dict[name] = experimentlib.create_identity_from_key_files( BETA_KEY_PATH + name + '.publickey').get('publickey_dict') return beta_keys_dict
def main(): identity = experimentlib.create_identity_from_key_files(PUBLICKEY_FILENAME, PRIVATEKEY_FILENAME) nodelocation_list = experimentlib.lookup_node_locations_by_identity(identity) print("Number of advertising nodes: " + str(len(nodelocation_list))) vesselhandle_list = experimentlib.find_vessels_on_nodes(identity, nodelocation_list) print("Number of active vessels: " + str(len(vesselhandle_list))) # Note that we could use experimentlib.run_parallelized() to parallelize # this, but for simplicity we do each sequentially. for vesselhandle in vesselhandle_list: try: nodeid, vesselname = experimentlib.get_nodeid_and_vesselname(vesselhandle) nodelocation = experimentlib.get_node_location(nodeid) # Note: you may want to reset_vessel(). experimentlib.upload_file_to_vessel(vesselhandle, identity, PROGRAM_FILE) print("Uploaded " + PROGRAM_FILE + " to " + nodelocation + " vessel " + vesselname) experimentlib.start_vessel(vesselhandle, identity, PROGRAM_FILE, ARGUMENTS_TO_START_PROGRAM_WITH) print("Program started on " + nodelocation + " vessel " + vesselname) except experimentlib.SeattleExperimentError, e: print("Failure on vessel " + vesselhandle + ". Error was: " + str(e))
def main(): monitorhandle = None while True: identity = experimentlib.create_identity_from_key_files(PUBLICKEY_FILENAME) # Get a list of nodes advertising under the public key. Some of these # nodes may be unreachable or may have gone offline since they advertised. # This won't try to communicate with the actual nodes. nodelocation_list = experimentlib.lookup_node_locations_by_identity(identity) # Contact each node and find out which vessels are usable by this identity/key. vesselhandle_list = experimentlib.find_vessels_on_nodes(identity, nodelocation_list) print("Lookup and vessel discovery shows " + str(len(vesselhandle_list)) + " active vessels.") # Either register a monitor if this is the first time through the loop or # just add the vesselhandles to the existing monitor. Adding the vesselhandles # will not remove the old one and duplicates will be ignored. if monitorhandle is None: monitorhandle = vesselstatusmonitor.register_vessel_status_monitor(identity, vesselhandle_list, vessel_status_callback, waittime=MONITOR_SLEEP_SECONDS) print("Vessel status monitor registered.") else: vesselstatusmonitor.add_to_vessel_status_monitor(monitorhandle, vesselhandle_list) time.sleep(LOOKUP_SLEEP_SECONDS)
def main(): identity = experimentlib.create_identity_from_key_files( SEATTLECLEARINGHOUSE_PUBLICKEY_FILENAME) nodelocation_list = experimentlib.lookup_node_locations_by_identity( identity) print("Number of advertising nodes: " + str(len(nodelocation_list))) if MAX_NODES_TO_LOOK_AT is not None: print("DEBUG: only looking at " + str(MAX_NODES_TO_LOOK_AT) + " nodes.") nodelocation_list = nodelocation_list[:MAX_NODES_TO_LOOK_AT] # Talk to each nodemanager to find out vessel information. browse_successlist, failurelist = \ experimentlib.run_parallelized(nodelocation_list, browse_node_for_available_vessels) # Create a dictionary whose keys are the nodeids and values are lists of # vesseldicts of the available vessels on that node. available_vesseldicts_by_node = {} for (nodeid, available_vesseldicts) in browse_successlist: if available_vesseldicts: available_vesseldicts_by_node[nodeid] = available_vesseldicts print("Number of nodes that SeattleGENI vessels are available on: " + str(len(available_vesseldicts_by_node.keys())))
def main(): identity = experimentlib.create_identity_from_key_files(PUBLICKEY_FILENAME, PRIVATEKEY_FILENAME) nodelocation_list = experimentlib.lookup_node_locations_by_identity(identity) print("Number of advertising nodes: " + str(len(nodelocation_list))) vesselhandle_list = experimentlib.find_vessels_on_nodes(identity, nodelocation_list) print("Number of active vessels: " + str(len(vesselhandle_list))) # Note that we could use experimentlib.run_parallelized() to parallelize the # uploading of files and speed things up, but for simplicity we do each # upload sequentially. for vesselhandle in vesselhandle_list: try: nodeid, vesselname = experimentlib.get_nodeid_and_vesselname(vesselhandle) nodelocation = experimentlib.get_node_location(nodeid) for local_filename in FILES_TO_UPLOAD: experimentlib.upload_file_to_vessel(vesselhandle, identity, local_filename) print("Uploaded " + local_filename + " to " + nodelocation + " vessel " + vesselname) except experimentlib.SeattleExperimentError, e: print("Failure on vessel " + vesselhandle + ". Error was: " + str(e))
def _beta_keys(): """ <Purpose> To generate a dictionary of beta keys for checking to see if a node is a production node or a beta node based off of what user keys its vessels have. <Argument> None. <Exceptions> Possible IO errors if keys have not been downloaded to their correct locations. <Side Effects> None. <Return> Returns a dictionary of beta keys in the form of {'key_name':publickey_dict}. """ beta_keys_dict = {} beta_key_names = ['acceptdonation', 'canonical', 'movingtoonepercent_manyevents', 'onepercentmanyevents'] for name in beta_key_names: beta_keys_dict[name] = experimentlib.create_identity_from_key_files(BETA_KEY_PATH + name + '.publickey').get('publickey_dict') return beta_keys_dict
def main(): identity = experimentlib.create_identity_from_key_files(PUBLICKEY_FILENAME) nodelocation_list = experimentlib.lookup_node_locations_by_identity(identity) print("Number of nodes advertising their location under this key: " + str(len(nodelocation_list))) for nodelocation in nodelocation_list: print(nodelocation)
def main(): identity = experimentlib.create_identity_from_key_files(PUBLICKEY_FILENAME) nodelocation_list = experimentlib.lookup_node_locations_by_identity( identity) print("Number of nodes advertising their location under this key: " + str(len(nodelocation_list))) for nodelocation in nodelocation_list: print(nodelocation)
def main(): identity = experimentlib.create_identity_from_key_files( PUBLICKEY_FILENAME, PRIVATEKEY_FILENAME) nodelocation_list = experimentlib.lookup_node_locations_by_identity( identity) print("Number of advertising nodes: " + str(len(nodelocation_list))) vesselhandle_list = experimentlib.find_vessels_on_nodes( identity, nodelocation_list) print("Number of active vessels: " + str(len(vesselhandle_list))) for vesselhandle in vesselhandle_list: try: # Generate the vessel directory name with format: host_port_vesselname nodeid, vesselname = experimentlib.get_nodeid_and_vesselname( vesselhandle) nodelocation = experimentlib.get_node_location(nodeid) host, port = experimentlib.get_host_and_port(nodelocation) dirname = host + "_" + str(port) + "_" + vesselname # Create the vessel directory if it doesn't already exist. vessel_directory = os.path.join(DOWNLOAD_DIRECTORY, dirname) if not os.path.exists(vessel_directory): os.mkdir(vessel_directory) # Get a list of files on the vessel. filelist = experimentlib.get_vessel_file_list( vesselhandle, identity) print("Files on " + nodelocation + " " + vesselname + ": " + str(filelist)) for remote_filename in filelist: local_filename = os.path.join(vessel_directory, remote_filename) experimentlib.download_file_from_vessel( vesselhandle, identity, remote_filename, local_filename) print("Downloaded " + local_filename) except experimentlib.SeattleExperimentError, e: print("Failure on vessel " + vesselhandle + ". Error was: " + str(e))
def main(): identity = experimentlib.create_identity_from_key_files( PUBLICKEY_FILENAME, PRIVATEKEY_FILENAME) nodelocation_list = experimentlib.lookup_node_locations_by_identity( identity) print("Number of advertising nodes: " + str(len(nodelocation_list))) vesselhandle_list = experimentlib.find_vessels_on_nodes( identity, nodelocation_list) print("Number of active vessels: " + str(len(vesselhandle_list))) # Note that we could use experimentlib.run_parallelized() to parallelize # this, but for simplicity we do each sequentially. for vesselhandle in vesselhandle_list: try: nodeid, vesselname = experimentlib.get_nodeid_and_vesselname( vesselhandle) nodelocation = experimentlib.get_node_location(nodeid) # Note: you may want to reset_vessel(). experimentlib.upload_file_to_vessel(vesselhandle, identity, PROGRAM_FILE) print("Uploaded " + PROGRAM_FILE + " to " + nodelocation + " vessel " + vesselname) experimentlib.start_vessel(vesselhandle, identity, PROGRAM_FILE, ARGUMENTS_TO_START_PROGRAM_WITH) print("Program started on " + nodelocation + " vessel " + vesselname) except experimentlib.SeattleExperimentError, e: print("Failure on vessel " + vesselhandle + ". Error was: " + str(e))
def main(): identity = experimentlib.create_identity_from_key_files(PUBLICKEY_FILENAME, PRIVATEKEY_FILENAME) nodelocation_list = experimentlib.lookup_node_locations_by_identity(identity) print("Number of advertising nodes: " + str(len(nodelocation_list))) browse_successlist, browse_failurelist = experimentlib.run_parallelized(nodelocation_list, experimentlib.browse_node, identity) vesseldict_list = [] for (nodeid, vesseldicts_of_node) in browse_successlist: vesseldict_list += vesseldicts_of_node print("Good vessels: " + str(len(vesseldict_list))) set_keys_successlist, set_keys_failure_list = experimentlib.run_parallelized(vesseldict_list, set_keys, identity) print("Vessels with proper user keys: " + str(len(set_keys_successlist))) print("Vessels that failed user key setting: " + str(len(set_keys_failure_list))) print("Done.")
def main(appmap_config): # Set SSL security level and warn user if running without SSL experimentlib.SEATTLECLEARINGHOUSE_ALLOW_SSL_INSECURE = appmap_config['allow_ssl_insecure'] if appmap_config['allow_ssl_insecure']: print "Running without secure SSL connection to SeattleGENI" # Get GENI user identity object and port CBFUNC_CONTEXT['identity'] = experimentlib.create_identity_from_key_files(appmap_config['geni_username'] + ".publickey", appmap_config['geni_username'] + ".privatekey") # If --insecure switch not used, attempt a secure SSL connection to # SeattleGENI. If SeattleClearinghouseError exception thrown, print warning. try: CBFUNC_CONTEXT['geniport'] = experimentlib.seattlegeni_user_port(CBFUNC_CONTEXT['identity']) except Exception, e: if repr(e).startswith("SeattleClearinghouseError"): print """Error: Cannot make secure SSL connection to SeattleGENI Please make sure that: * M2Crypto is installed * the current directory contains a valid PEM file containing CA certificates To run appmap.py without SSL authentication, use the `--insecure` switch. Check documentation for more details.""" sys.exit()
def main(): identity = experimentlib.create_identity_from_key_files(PUBLICKEY_FILENAME, PRIVATEKEY_FILENAME) nodelocation_list = experimentlib.lookup_node_locations_by_identity(identity) print("Number of advertising nodes: " + str(len(nodelocation_list))) vesselhandle_list = experimentlib.find_vessels_on_nodes(identity, nodelocation_list) print("Number of active vessels: " + str(len(vesselhandle_list))) for vesselhandle in vesselhandle_list: try: # Generate the vessel directory name with format: host_port_vesselname nodeid, vesselname = experimentlib.get_nodeid_and_vesselname(vesselhandle) nodelocation = experimentlib.get_node_location(nodeid) host, port = experimentlib.get_host_and_port(nodelocation) dirname = host + "_" + str(port) + "_" + vesselname # Create the vessel directory if it doesn't already exist. vessel_directory = os.path.join(DOWNLOAD_DIRECTORY, dirname) if not os.path.exists(vessel_directory): os.mkdir(vessel_directory) # Get a list of files on the vessel. filelist = experimentlib.get_vessel_file_list(vesselhandle, identity) print("Files on " + nodelocation + " " + vesselname + ": " + str(filelist)) for remote_filename in filelist: local_filename = os.path.join(vessel_directory, remote_filename) experimentlib.download_file_from_vessel(vesselhandle, identity, remote_filename, local_filename) print("Downloaded " + local_filename) except experimentlib.SeattleExperimentError, e: print("Failure on vessel " + vesselhandle + ". Error was: " + str(e))
def main(): identity = experimentlib.create_identity_from_key_files(SEATTLEGENI_PUBLICKEY_FILENAME) nodelocation_list = experimentlib.lookup_node_locations_by_identity(identity) print("Number of advertising nodes: " + str(len(nodelocation_list))) if MAX_NODES_TO_LOOK_AT is not None: print("DEBUG: only looking at " + str(MAX_NODES_TO_LOOK_AT) + " nodes.") nodelocation_list = nodelocation_list[:MAX_NODES_TO_LOOK_AT] # Talk to each nodemanager to find out vessel information. browse_successlist, failurelist = \ experimentlib.run_parallelized(nodelocation_list, browse_node_for_available_vessels) # Create a dictionary whose keys are the nodeids and values are lists of # vesseldicts of the available vessels on that node. available_vesseldicts_by_node = {} for (nodeid, available_vesseldicts) in browse_successlist: if available_vesseldicts: available_vesseldicts_by_node[nodeid] = available_vesseldicts print("Number of nodes that SeattleGENI vessels are available on: " + str(len(available_vesseldicts_by_node.keys())))
def main(): identity = experimentlib.create_identity_from_key_files(PUBLICKEY_FILENAME, PRIVATEKEY_FILENAME) nodelocation_list = experimentlib.lookup_node_locations_by_identity(identity) print("Number of advertising nodes: " + str(len(nodelocation_list))) vesselhandle_list = experimentlib.find_vessels_on_nodes(identity, nodelocation_list) print("Number of active vessels: " + str(len(vesselhandle_list))) for vesselhandle in vesselhandle_list: try: nodeid, vesselname = experimentlib.get_nodeid_and_vesselname(vesselhandle) nodelocation = experimentlib.get_node_location(nodeid) print("====================================================================") print("Log from " + nodelocation + " " + vesselname) logcontents = experimentlib.get_vessel_log(vesselhandle, identity) print(logcontents) except experimentlib.SeattleExperimentError, e: print("Failure on vessel " + vesselhandle + ". Error was: " + str(e))
def main(): monitorhandle = None while True: identity = experimentlib.create_identity_from_key_files( PUBLICKEY_FILENAME) # Get a list of nodes advertising under the public key. Some of these # nodes may be unreachable or may have gone offline since they advertised. # This won't try to communicate with the actual nodes. nodelocation_list = experimentlib.lookup_node_locations_by_identity( identity) # Contact each node and find out which vessels are usable by this identity/key. vesselhandle_list = experimentlib.find_vessels_on_nodes( identity, nodelocation_list) print("Lookup and vessel discovery shows " + str(len(vesselhandle_list)) + " active vessels.") # Either register a monitor if this is the first time through the loop or # just add the vesselhandles to the existing monitor. Adding the vesselhandles # will not remove the old one and duplicates will be ignored. if monitorhandle is None: monitorhandle = vesselstatusmonitor.register_vessel_status_monitor( identity, vesselhandle_list, vessel_status_callback, waittime=MONITOR_SLEEP_SECONDS) print("Vessel status monitor registered.") else: vesselstatusmonitor.add_to_vessel_status_monitor( monitorhandle, vesselhandle_list) time.sleep(LOOKUP_SLEEP_SECONDS)
def main(): identity = experimentlib.create_identity_from_key_files(PUBLICKEY_FILENAME, PRIVATEKEY_FILENAME) # Get a list of nodes advertising under the public key. Some of these # nodes may be unreachable or may have gone offline since they advertised. # This won't try to communicate with the actual nodes. nodelocation_list = experimentlib.lookup_node_locations_by_identity(identity) print("nodelocation_list:" + str(nodelocation_list)) # Talk to each advertising node to find out which vessels we have on each. # We get back a list of dictionaries, where each dictionary describes a # vessel. There may be multiple vessels from any single node in this list. active_vesselhandle_list = experimentlib.find_vessels_on_nodes(identity, nodelocation_list) print("active_vesselhandle_list:" + str(active_vesselhandle_list)) # Now we want to find out which vessels we've acquired through seattlegeni # that are not in our list. We may, for example, want to release those # vessels because we consider them unusable. try: expected_vesselhandle_list = experimentlib.seattlegeni_get_acquired_vessels(identity) print("expected_vesselhandle_list:" + str(expected_vesselhandle_list)) # If we already have enough usable vessels, we're done. if len(active_vesselhandle_list) >= MIN_VESSELS_TO_KEEP: print( "There are already " + str(len(active_vesselhandle_list)) + " active vessels " + "and our MIN_VESSELS_TO_KEEP is " + str(MIN_VESSELS_TO_KEEP) ) return # We assume all of our vessels come from seattlegeni, so if any vessels we # should have access to according to seattlegeni aren't accessible or # otherwise usable, then release them. vesselhandles_to_release = [] for vesselhandle in expected_vesselhandle_list: if vesselhandle not in active_vesselhandle_list: vesselhandles_to_release.append(vesselhandle) if vesselhandles_to_release: print( str(len(vesselhandles_to_release)) + " vessels were inaccessible, so will try to release them:" + str(vesselhandles_to_release) ) try: experimentlib.seattlegeni_release_vessels(identity, vesselhandles_to_release) except experimentlib.SeattleClearinghouseError, e: print("Failed to release vessels: " + str(e)) else: print("Vessels successfully released.") # Determine the maximum number of vessels we can acquire through seattlegeni. max_vessels_allowed = experimentlib.seattlegeni_max_vessels_allowed(identity) print("max_vessels_allowed: " + str(max_vessels_allowed)) # Determine the number of vessels we already have acquired through seattlegeni. num_currently_acquired = len(experimentlib.seattlegeni_get_acquired_vessels(identity)) print("currently_acquired_count: " + str(num_currently_acquired)) # Let's try to get as close to MIN_VESSELS_TO_KEEP without requesting more # than is allowed by this account. num_vessels_to_request = min(max_vessels_allowed, MIN_VESSELS_TO_KEEP) - num_currently_acquired if num_vessels_to_request <= 0: print("This account doesn't have enough vessel credits to request more vessels.") return else: print("Will try to acquire " + str(num_vessels_to_request) + " vessels.") vessel_type = experimentlib.SEATTLECLEARINGHOUSE_VESSEL_TYPE_WAN acquired_vessels = experimentlib.seattlegeni_acquire_vessels(identity, vessel_type, num_vessels_to_request) print("Acquired " + str(num_vessels_to_request) + " vessels: " + str(acquired_vessels))
def init(geni_username, vesselcount, vesseltype, program_filename): """ <Purpose> Initializes the deployment of an arbitrary service. Populates a global configuration dictionary and returns a dict of data that could be required by the run() function. init() must be called before the run() function. Note on the return dict: The return value of init() is meant to contain data that is needed by calls to explib.start_vessel() within the run() function. At the time of creation of this library, the only such data that is required by deployable services is the user's GENI port. To add support for services requiring more arguments, simply add the necessary data to the dictionary returned by this function. <Arguments> geni_username SeattleGENI username. Used to locate and handle public and private key files. vesselcount The number of vessels on which to deploy. vesseltype The type of vessel to acquire, based on the SEATTLEGENI_VESSEL_TYPE_* constants within experimentlib.py program_filename The filename of the program to deploy and monitor on vessels. <Exceptions> ValueError Raised if argument vesseltype doesn't match one of the experimentlib SEATTLEGENI_VESSEL_TYPE_* constants, if argument program file does not exist, or if argument number of vessels on which to deploy exceeds the user's number of vessel credits. <Side Effects> Initializes certain global variables. <Returns> A dictionary containing data that clients might need for running program on vessels. """ # Fill config dict with argument data # Validate vesseltype, based on constants in explib if vesseltype not in [explib.SEATTLEGENI_VESSEL_TYPE_WAN, explib.SEATTLEGENI_VESSEL_TYPE_LAN, explib.SEATTLEGENI_VESSEL_TYPE_NAT, explib.SEATTLEGENI_VESSEL_TYPE_RAND]: raise ValueError('Invalid vessel type specified. Argument vessel type must be one of the SEATTLEGENI_VESSEL_TYPE_* constants defined in experimentlib.py') config['vesseltype'] = vesseltype # Verify that program file exists if not os.path.isfile(program_filename): raise ValueError('Specified program file ' + program_filename + ' does not exist') config['program_filename'] = program_filename # Setup explib identity object and GENI details config['identity'] = explib.create_identity_from_key_files( geni_username + '.publickey', geni_username + '.privatekey') print config['identity'] config['geni_port'] = explib.seattlegeni_user_port(config['identity']) # Validate number of vessels on which to deploy num_vslcredits = explib.seattlegeni_max_vessels_allowed(config['identity']) if vesselcount > num_vslcredits: raise ValueError('Invalid number of vessels specified. The number of deployed vessels must be less than or equal to the user\'s number of vessel credits.') config['vesselcount'] = vesselcount # Create and populate the return dict ret_dict = { 'geni_port': config['geni_port'] } return ret_dict
def poll_data(): """ <Purpose> This is used to collect and store node/vessel data. <Arguments> None. <Exceptions> NodeLocationListEmptyError This error will be raised if we are unable to lookup any node locations because there won't be any nodes to look up any information on. ExperimentLibError If any experiment library errors occur they will be logged in 'fail_list.dat.gz' at the end of poll_data() <Side Effects> Creates a new folder at the users path (specified above) with a timestamp. Inside that folder will exist 2 gzip files one called 'nodes_list.gz' and one called 'fail_list.gz', these are compressed python pickles of the objects used in this script. <Returns> None. """ poll_identity = experimentlib.create_identity_from_key_files(PATH_TO_EXP_LIB + "stat_lib/perc20.publickey") # Creates a list of IPs that are associated with this identity. nodelocation_list = experimentlib.lookup_node_locations_by_identity(poll_identity) # The format of the timestamp is of the form: # YYYYMMDD_HHMM, with single digits zero padded (5 -> 05). curr_time = datetime.datetime.now() # Converts datetime object to a formated string. folder_name = datetime.datetime.strftime(curr_time, "%Y%m%d_%H%M") month_folder = datetime.datetime.strftime(curr_time, "%Y_%m") # If a folder does not exist for the month of the poll, it is created. if not os.path.isdir(PATH_TO_EXP_LIB + "seattle_node_data/" + month_folder): os.mkdir(PATH_TO_EXP_LIB + "seattle_node_data/" + month_folder) # Create the folder for this poll data to be stored. data_storage_path = PATH_TO_EXP_LIB + "seattle_node_data/" + month_folder + "/" + folder_name os.mkdir(data_storage_path) # Checks to make sure we have enough node locations to continue with the # poll, if not a NodeLocationListEmpty Error is raised. if(len(nodelocation_list) < 0): raise NodeLocationListEmptyError("Node location list is empty, " + "poll_data stopped at: " + str(curr_time)) # Creates initial gzip data files to be saved (appends if needed). nodes_file = gzip.open(data_storage_path + "/node_list.dat.gz",'ar') fail_file = gzip.open(data_storage_path + "/fail_list.dat.gz",'ar') # The initial data structures to be pickled. total_node_dicts = {} fail_dict = {} for node_ip in nodelocation_list: single_node_dict = {} single_node_dict['timestamp'] = curr_time time_before_browsenode = time.time() # If browse_node fails it will throw an ExperimentLibError which means we # are not able to communicate with the node so it is added to the list # of failed nodes. try: list_of_vessel_dicts = experimentlib.browse_node(node_ip) except experimentlib.SeattleExperimentError, err: time_atfail_browsenode = time.time() elapsed_time_fail = time_atfail_browsenode - time_before_browsenode fail_dict[str(node_ip)] = [err, elapsed_time_fail] time_before_browsenode = 0 time_after_browsenode = 0 continue time_after_browsenode = time.time() elapsed_time = time_after_browsenode - time_before_browsenode single_node_dict['responsetime'] = elapsed_time single_node_dict['nodelocation'] = node_ip single_node_dict['version'] = list_of_vessel_dicts[0].get('version') unique_id = list_of_vessel_dicts[0].get('nodeid') single_node_dict['nodeid'] = unique_id single_node_dict['vessels'] = list_of_vessel_dicts total_node_dicts[str(unique_id)] = single_node_dict
def __init__(self, geni_username, vessel_count, vessel_type, program_filename, log_filename=None): """ <Purpose> Initializes an instance of Overlord for the deployment of an arbitrary service. Populates the instance's configuration dictionary, which can be accessed from the object if data is needed for the run() function <Arguments> geni_username SeattleGENI username. Used to locate and handle public and private key files. vesselcount The number of vessels on which to deploy. vesseltype The type of vessel to acquire, based on the SEATTLECLEARINGHOUSE_VESSEL_TYPE_* constants within experimentlib.py program_filename The filename of the program to deploy and monitor on vessels. log_filename The file the user wants overlord to log to. If None, logs directly to console <Exceptions> ValueError Raised if argument vesseltype doesn't match one of the experimentlib SEATTLECLEARINGHOUSE_VESSEL_TYPE_* constants, if argument program file does not exist, or if argument number of vessels on which to deploy exceeds the user's number of vessel credits. <Side Effects> Initializes certain global variables. Removes 'stop' file from directory if it exists Sets the functions that makes up run() to the default methods listed at the end of the code. <Returns> None """ # If a stop file still exists, delete it for the user if os.path.isfile("./stop"): os.remove(os.path.expanduser("./stop")) # List of valid vessel types. vessel_types = [ explib.SEATTLECLEARINGHOUSE_VESSEL_TYPE_WAN, explib.SEATTLECLEARINGHOUSE_VESSEL_TYPE_LAN, explib.SEATTLECLEARINGHOUSE_VESSEL_TYPE_NAT, explib.SEATTLECLEARINGHOUSE_VESSEL_TYPE_RAND ] if vessel_type not in vessel_types: raise ValueError( "Invalid vessel type specified. Argument 'vessel_type' must be one of " + "the SEATTLECLEARINGHOUSE_VESSEL_TYPE_* constants defined in 'experimentlib.py'" ) self.config['vessel_type'] = vessel_type # If a program file isn't passed, assume user will handle the issue if program_filename: # Verify that program file exists if not os.path.isfile(program_filename): raise ValueError("Specified program file '" + program_filename + "' does not exist") self.config['program_filename'] = program_filename # Setup explib identity object and GENI details self.config['identity'] = explib.create_identity_from_key_files( geni_username + '.publickey', geni_username + '.privatekey') self.config['geni_port'] = explib.seattlegeni_user_port( self.config['identity']) # Ensure that the user has enough credits to acquire the specified number of vessels. num_vessel_credits = explib.seattlegeni_max_vessels_allowed( self.config['identity']) if vessel_count > num_vessel_credits: raise ValueError( "Invalid number of vessels specified. The number of deployed vessels must " + "be less than or equal to the user's number of vessel credits." ) self.config['vessel_count'] = vessel_count # Set up the logger according to passed arguments if log_filename: # Add the file logger. fileLog = logging.FileHandler(log_filename, 'w') fileLog.setFormatter( logging.Formatter('%(asctime)s %(levelname)-8s %(message)s', '%Y-%m-%d %H:%M:%S')) self.logger.addHandler(fileLog) else: # Add the console logger. consoleLog = logging.StreamHandler() consoleLog.setFormatter( logging.Formatter('%(asctime)s %(levelname)-8s %(message)s', '%H:%M:%S')) self.logger.addHandler(consoleLog) # Setting the functions of 'run' to default self.init_overlord_func = default_init_overlord self.acquire_vessels_func = default_acquire_vessels self.init_vessels_func = default_initiate_vessels self.remove_vessels_func = default_remove_vessels self.maintenance_func = default_maintenance
def __init__(self, geni_username, vessel_count, vessel_type, program_filename, log_filename=None): """ <Purpose> Initializes an instance of Overlord for the deployment of an arbitrary service. Populates the instance's configuration dictionary, which can be accessed from the object if data is needed for the run() function <Arguments> geni_username SeattleGENI username. Used to locate and handle public and private key files. vesselcount The number of vessels on which to deploy. vesseltype The type of vessel to acquire, based on the SEATTLECLEARINGHOUSE_VESSEL_TYPE_* constants within experimentlib.py program_filename The filename of the program to deploy and monitor on vessels. log_filename The file the user wants overlord to log to. If None, logs directly to console <Exceptions> ValueError Raised if argument vesseltype doesn't match one of the experimentlib SEATTLECLEARINGHOUSE_VESSEL_TYPE_* constants, if argument program file does not exist, or if argument number of vessels on which to deploy exceeds the user's number of vessel credits. <Side Effects> Initializes certain global variables. Removes 'stop' file from directory if it exists Sets the functions that makes up run() to the default methods listed at the end of the code. <Returns> None """ # If a stop file still exists, delete it for the user if os.path.isfile("./stop"): os.remove(os.path.expanduser("./stop")) # List of valid vessel types. vessel_types = [ explib.SEATTLECLEARINGHOUSE_VESSEL_TYPE_WAN, explib.SEATTLECLEARINGHOUSE_VESSEL_TYPE_LAN, explib.SEATTLECLEARINGHOUSE_VESSEL_TYPE_NAT, explib.SEATTLECLEARINGHOUSE_VESSEL_TYPE_RAND, ] if vessel_type not in vessel_types: raise ValueError( "Invalid vessel type specified. Argument 'vessel_type' must be one of " + "the SEATTLECLEARINGHOUSE_VESSEL_TYPE_* constants defined in 'experimentlib.py'" ) self.config["vessel_type"] = vessel_type # If a program file isn't passed, assume user will handle the issue if program_filename: # Verify that program file exists if not os.path.isfile(program_filename): raise ValueError("Specified program file '" + program_filename + "' does not exist") self.config["program_filename"] = program_filename # Setup explib identity object and GENI details self.config["identity"] = explib.create_identity_from_key_files( geni_username + ".publickey", geni_username + ".privatekey" ) self.config["geni_port"] = explib.seattlegeni_user_port(self.config["identity"]) # Ensure that the user has enough credits to acquire the specified number of vessels. num_vessel_credits = explib.seattlegeni_max_vessels_allowed(self.config["identity"]) if vessel_count > num_vessel_credits: raise ValueError( "Invalid number of vessels specified. The number of deployed vessels must " + "be less than or equal to the user's number of vessel credits." ) self.config["vessel_count"] = vessel_count # Set up the logger according to passed arguments if log_filename: # Add the file logger. fileLog = logging.FileHandler(log_filename, "w") fileLog.setFormatter(logging.Formatter("%(asctime)s %(levelname)-8s %(message)s", "%Y-%m-%d %H:%M:%S")) self.logger.addHandler(fileLog) else: # Add the console logger. consoleLog = logging.StreamHandler() consoleLog.setFormatter(logging.Formatter("%(asctime)s %(levelname)-8s %(message)s", "%H:%M:%S")) self.logger.addHandler(consoleLog) # Setting the functions of 'run' to default self.init_overlord_func = default_init_overlord self.acquire_vessels_func = default_acquire_vessels self.init_vessels_func = default_initiate_vessels self.remove_vessels_func = default_remove_vessels self.maintenance_func = default_maintenance
def poll_data(): """ <Purpose> This is used to collect and store node/vessel data. <Arguments> None. <Exceptions> NodeLocationListEmptyError This error will be raised if we are unable to lookup any node locations because there won't be any nodes to look up any information on. ExperimentLibError If any experiment library errors occur they will be logged in 'fail_list.dat.gz' at the end of poll_data() <Side Effects> Creates a new folder at the users path (specified above) with a timestamp. Inside that folder will exist 2 gzip files one called 'nodes_list.gz' and one called 'fail_list.gz', these are compressed python pickles of the objects used in this script. <Returns> None. """ poll_identity = experimentlib.create_identity_from_key_files( PATH_TO_EXP_LIB + "stat_lib/perc20.publickey") # Creates a list of IPs that are associated with this identity. nodelocation_list = experimentlib.lookup_node_locations_by_identity( poll_identity) # The format of the timestamp is of the form: # YYYYMMDD_HHMM, with single digits zero padded (5 -> 05). curr_time = datetime.datetime.now() # Converts datetime object to a formated string. folder_name = datetime.datetime.strftime(curr_time, "%Y%m%d_%H%M") month_folder = datetime.datetime.strftime(curr_time, "%Y_%m") # If a folder does not exist for the month of the poll, it is created. if not os.path.isdir(PATH_TO_EXP_LIB + "seattle_node_data/" + month_folder): os.mkdir(PATH_TO_EXP_LIB + "seattle_node_data/" + month_folder) # Create the folder for this poll data to be stored. data_storage_path = PATH_TO_EXP_LIB + "seattle_node_data/" + month_folder + "/" + folder_name os.mkdir(data_storage_path) # Checks to make sure we have enough node locations to continue with the # poll, if not a NodeLocationListEmpty Error is raised. if (len(nodelocation_list) < 0): raise NodeLocationListEmptyError("Node location list is empty, " + "poll_data stopped at: " + str(curr_time)) # Creates initial gzip data files to be saved (appends if needed). nodes_file = gzip.open(data_storage_path + "/node_list.dat.gz", 'ar') fail_file = gzip.open(data_storage_path + "/fail_list.dat.gz", 'ar') # The initial data structures to be pickled. total_node_dicts = {} fail_dict = {} for node_ip in nodelocation_list: single_node_dict = {} single_node_dict['timestamp'] = curr_time time_before_browsenode = time.time() # If browse_node fails it will throw an ExperimentLibError which means we # are not able to communicate with the node so it is added to the list # of failed nodes. try: list_of_vessel_dicts = experimentlib.browse_node(node_ip) except experimentlib.SeattleExperimentError, err: time_atfail_browsenode = time.time() elapsed_time_fail = time_atfail_browsenode - time_before_browsenode fail_dict[str(node_ip)] = [err, elapsed_time_fail] time_before_browsenode = 0 time_after_browsenode = 0 continue time_after_browsenode = time.time() elapsed_time = time_after_browsenode - time_before_browsenode single_node_dict['responsetime'] = elapsed_time single_node_dict['nodelocation'] = node_ip single_node_dict['version'] = list_of_vessel_dicts[0].get('version') unique_id = list_of_vessel_dicts[0].get('nodeid') single_node_dict['nodeid'] = unique_id single_node_dict['vessels'] = list_of_vessel_dicts total_node_dicts[str(unique_id)] = single_node_dict
def main(): identity = experimentlib.create_identity_from_key_files( PUBLICKEY_FILENAME, PRIVATEKEY_FILENAME) # Get a list of nodes advertising under the public key. Some of these # nodes may be unreachable or may have gone offline since they advertised. # This won't try to communicate with the actual nodes. nodelocation_list = experimentlib.lookup_node_locations_by_identity( identity) print("nodelocation_list:" + str(nodelocation_list)) # Talk to each advertising node to find out which vessels we have on each. # We get back a list of dictionaries, where each dictionary describes a # vessel. There may be multiple vessels from any single node in this list. active_vesselhandle_list = experimentlib.find_vessels_on_nodes( identity, nodelocation_list) print("active_vesselhandle_list:" + str(active_vesselhandle_list)) # Now we want to find out which vessels we've acquired through seattlegeni # that are not in our list. We may, for example, want to release those # vessels because we consider them unusable. try: expected_vesselhandle_list = experimentlib.seattlegeni_get_acquired_vessels( identity) print("expected_vesselhandle_list:" + str(expected_vesselhandle_list)) # If we already have enough usable vessels, we're done. if len(active_vesselhandle_list) >= MIN_VESSELS_TO_KEEP: print("There are already " + str(len(active_vesselhandle_list)) + " active vessels " + "and our MIN_VESSELS_TO_KEEP is " + str(MIN_VESSELS_TO_KEEP)) return # We assume all of our vessels come from seattlegeni, so if any vessels we # should have access to according to seattlegeni aren't accessible or # otherwise usable, then release them. vesselhandles_to_release = [] for vesselhandle in expected_vesselhandle_list: if vesselhandle not in active_vesselhandle_list: vesselhandles_to_release.append(vesselhandle) if vesselhandles_to_release: print( str(len(vesselhandles_to_release)) + " vessels were inaccessible, so will try to release them:" + str(vesselhandles_to_release)) try: experimentlib.seattlegeni_release_vessels( identity, vesselhandles_to_release) except experimentlib.SeattleClearinghouseError, e: print("Failed to release vessels: " + str(e)) else: print("Vessels successfully released.") # Determine the maximum number of vessels we can acquire through seattlegeni. max_vessels_allowed = experimentlib.seattlegeni_max_vessels_allowed( identity) print("max_vessels_allowed: " + str(max_vessels_allowed)) # Determine the number of vessels we already have acquired through seattlegeni. num_currently_acquired = len( experimentlib.seattlegeni_get_acquired_vessels(identity)) print("currently_acquired_count: " + str(num_currently_acquired)) # Let's try to get as close to MIN_VESSELS_TO_KEEP without requesting more # than is allowed by this account. num_vessels_to_request = min( max_vessels_allowed, MIN_VESSELS_TO_KEEP) - num_currently_acquired if num_vessels_to_request <= 0: print( "This account doesn't have enough vessel credits to request more vessels." ) return else: print("Will try to acquire " + str(num_vessels_to_request) + " vessels.") vessel_type = experimentlib.SEATTLECLEARINGHOUSE_VESSEL_TYPE_WAN acquired_vessels = experimentlib.seattlegeni_acquire_vessels( identity, vessel_type, num_vessels_to_request) print("Acquired " + str(num_vessels_to_request) + " vessels: " + str(acquired_vessels))