Пример #1
0
def default_maintenance(overlord, renewal_delay, last_renewal, vessel_handlers):
    # Renew vessels periodically.
    time_elapsed = datetime.utcnow() - last_renewal

    if time_elapsed > renewal_delay:
        # Perform the vessel renewal.
        overlord.logger.info("Renewing " + str(len(vessel_handlers)) + " vessels")
        explib.seattlegeni_renew_vessels(overlord.config["identity"], vessel_handlers)

        # Reset the time.
        last_renewal = datetime.utcnow()

    return last_renewal
Пример #2
0
def default_maintenance(overlord, renewal_delay, last_renewal,
                        vessel_handlers):
    # Renew vessels periodically.
    time_elapsed = datetime.utcnow() - last_renewal

    if time_elapsed > renewal_delay:
        # Perform the vessel renewal.
        overlord.logger.info('Renewing ' + str(len(vessel_handlers)) +
                             ' vessels')
        explib.seattlegeni_renew_vessels(overlord.config['identity'],
                                         vessel_handlers)

        # Reset the time.
        last_renewal = datetime.utcnow()

    return last_renewal
Пример #3
0
    def run(self, *args):
        """
    <Purpose>
      Starts the deployment and monitoring of a service on a number of vessels.
      Handles all acquisition of, uploading to, starting, and release of vessels.
      Contains the main loop of this program, and is thus the final function to
      call in all client programs. 
    <Arguments>
      *args: 
        Optional arguments that well be passed when starting the uploaded
        program file, by default

    <Exceptions>
      Common SeattleGENI exceptions described in the module comments.

    <Side Effects>
      Unless overridden beforehand, persistently logs to overlord_instance.log and
      console
      Returns if a 'stop' file is found in directory of the program importing the API

    <Returns>
      None
    """
        # Log setup information for user reference.
        self.logger.info("Performing actions as GENI user '" +
                         self.config['identity']['username'] + "'")
        self.logger.info('Vessel status will be polled every ' +
                         str(self.VESSEL_POLLING_PERIOD) + ' seconds')
        self.logger.info('Vessels will be renewed every ' +
                         str(self.VESSEL_RENEWAL_PERIOD) + ' seconds')

        # Run the start-up function for overlord
        self.init_overlord_func(self)

        # Retrieve the current list of vessel handles owned by identity, regardless
        # of the results of overlord's start-up function
        vessel_handlers = explib.seattlegeni_get_acquired_vessels(
            self.config['identity'])

        # If there are still vessels allocated to the identity, renew them to keep
        # renewal periods consistent with newly acquired vessels later on
        if vessel_handlers:
            self.logger.info('Renewing ' + str(len(vessel_handlers)) +
                             ' vessels')
            explib.seattlegeni_renew_vessels(self.config['identity'],
                                             vessel_handlers)

        # Turn VESSEL_RENEWAL_PERIOD into a timedelta object.
        renewal_delay = timedelta(seconds=self.VESSEL_RENEWAL_PERIOD)

        # Assume that the vessels have been renewed.
        last_renewal = datetime.utcnow()

        # Main loop
        while True:

            # End operations if "stop" file is found in same directory as overlord.py
            if os.path.isfile("./stop"):
                self.logger.info("Stop file found; Discontinuing operations.")
                return

            # Call the acquire vessels function and set the list of the handles of the
            # newly acquired vessels
            fresh_handlers = self.acquire_vessels_func(self, vessel_handlers)

            # If the list is not empty, call the initiate vessel function, and an updated
            # list of currently owned vessel handles that includes the newly acquired ones
            # should be returned
            if fresh_handlers:
                vessel_handlers = self.init_vessels_func(
                    self, fresh_handlers, vessel_handlers, *args)

            # Weed out any vessels based on the remove vessels function
            vessel_handlers = self.remove_vessels_func(self, vessel_handlers)

            # Perform any maintenance on the vessels, including renewals if needed,
            # and return the relevant information needed for next maintenance.
            # Currently coded to fit with default behavior, but should somehow be made a
            # little more flexible for user's function overrides?
            last_renewal = self.maintenance_func(self, renewal_delay,
                                                 last_renewal, vessel_handlers)

            # If the list of running vessels does not match the requested vessel count,
            # repeat the loop until it does
            if not len(vessel_handlers) == self.config['vessel_count']:
                continue

            # Sleep between polling vessel statuses.
            time.sleep(self.VESSEL_POLLING_PERIOD)
Пример #4
0
            vessel_handlers = explib.seattlegeni_acquire_vessels(
                self.config['identity'], self.config['vessel_type'],
                vessel_count)

        except explib.SeattleClearinghouseError, e:
            self.logger.error('Error while acquiring vessels: ' + str(e))
            return []

        else:
            self.logger.debug('Successfully acquired ' + str(vessel_count) +
                              ' vessels')

        # Renew vessels to extend their expiration time.
        self.logger.debug('Renewing ' + str(len(vessel_handlers)) +
                          ' newly acquired vessels')
        explib.seattlegeni_renew_vessels(self.config['identity'],
                                         vessel_handlers)

        return vessel_handlers

    def upload_to_vessels(self, vessel_handlers, filename_list):
        """
    <Purpose>
      Uploads a list of file to a set of vessels. A batch wrapper around the Experiment
      Library function upload_file_to_vessel, with logging and parallelization support.

    <Arguments>
      vesselhandle_list
        A list of vesselhandles of vessels to which the file is to be uploaded.
      filename
        The filename of the file to be uploaded.
Пример #5
0
def run(*args):
  """
  <Purpose>
    Starts the deployment and monitoring of a service on a number of vessels.
    Handles all acquisition of, uploading to, starting, and release of vessels.
    Contains the main loop of this program, and is thus the final function to
    call in all client programs. Requires init() to have been called prior to
    running.
  
  <Arguments>
    *args

  <Exceptions>
    None

  <Side Effects>
    Persistently writes to a log file.
    
  <Returns>
    None
  """
  # Write logfile header
  config['logfile'] = open(config['logfilename'], 'w')
  config['logfile'].write('################################################\n')
  config['logfile'].write('##   Overlord Deployment and Monitoring Log   ##\n')
  config['logfile'].write('################################################\n\n')
  config['logfile'].write('GENI user:              '******'identity']['username'] + '\n')
  config['logfile'].write('Vessels to monitor:     ' + str(config['vesselcount']) + '\n')
  config['logfile'].write('Time of script start:   ' + str(time.time()) + '\n\n')
  config['logfile'].flush()

  
  # Release any preallocated vessels
  vesselhandle_list = explib.seattlegeni_get_acquired_vessels(config['identity'])
  release_vessels(vesselhandle_list, 'Releasing ' + str(len(vesselhandle_list)) + ' preallocated vessels...')

  
  # Acquire an initial sample of vessels
  config['logfile'].write(str(time.time()) + ': Fetching initial batch of ' + str(config['vesselcount']) + ' vessels:\n')
  config['logfile'].flush()
  vesselhandle_list = []
  while not vesselhandle_list:
    vesselhandle_list = acquire_vessels(config['vesselcount'])

  # Upload program to vessels
  vesselhandle_list = upload_to_vessels(vesselhandle_list, config['program_filename'])


  # Run program on vessels
  vesselhandle_list, failed_list = run_on_vessels(vesselhandle_list,
                                             config['program_filename'],
                                             *args)


  # Release any failed vessels
  if failed_list:
    config['logfile'].write(str(time.time()) + ': Running ' + config['program_filename'] + ' failed on ' + str(len(failed_list)) + ' vessels\n')

    # Get details about failed vessel(s) and log them
    for vh in failed_list:
      try:
        vessel_log = explib.get_vessel_log(vh, config['identity'])
      except:
        vessel_log = '[ERROR: vessel log fetch failed]'
        
      nodeid, vesselname = explib.get_nodeid_and_vesselname(vh)
      nodelocation = explib.get_node_location(nodeid)
      
      # Log the vessel's log contents
      config['logfile'].write('Log contents of failed vessel at ' + nodelocation + ': ' + vessel_log + '\n')
      config['logfile'].flush()
      
    # Release the failed vessels
    release_vessels(failed_list, 'Releasing failed vessel(s)...')



  # Initialize counter variable for loop iterations
  loop_iterations = 0
  PREPPED = True
  print "PREPPED!"
  print "Vessel Handles: %s" % vesselhandle_list

  # Main loop
  while KEEP_RUNNING == True:
    print "Starting Loop!"
    # Check for vessels not in started state
    stopped_vessel_list = []
    for vh in vesselhandle_list:
      try:
        vessel_status = explib.get_vessel_status(vh, config['identity'])
        log = explib.get_vessel_log(vh, config['identity'])
        print "Loop Log: %s" % log
      except:
        # Node lookup failed, so remove vessel from vesselhandle_list
        # TODO: proper way to handle failed advertisements?
        stopped_vessel_list.append(vh)
      else:
        if vessel_status != explib.VESSEL_STATUS_STARTED:
          stopped_vessel_list.append(vh)

    # Release and replace any stopped vessels
    if stopped_vessel_list:
      # Release any stopped vessels
      release_vessels(stopped_vessel_list, 'Releasing ' + str(len(stopped_vessel_list)) + ' stopped vessel(s)...')

      # Remove released vessels from vesselhandle_list
      vesselhandle_list = list_difference(vesselhandle_list, stopped_vessel_list)

    # Ensure that enough vessels are running
    if len(vesselhandle_list) < config['vesselcount']:
      # If there aren't enough active vessels, acquire some
      config['logfile'].write(str(time.time()) + ': Only ' + str(len(vesselhandle_list)) + ' vessel(s) out of target ' + str(config['vesselcount']) + ' detected\n')
      config['logfile'].flush()
      fresh_vessels = acquire_vessels(config['vesselcount'] - len(vesselhandle_list))

      # Upload and run program to/on fresh vessels
      fresh_vessels = upload_to_vessels(fresh_vessels, config['program_filename'])
      success_list, failed_list = run_on_vessels(fresh_vessels,
                                                 config['program_filename'],
                                                 *args)

      # Release any failed vessels
      if failed_list:
        config['logfile'].write(str(time.time()) + ': Running ' + config['program_filename'] + ' failed on ' + str(len(failed_list)) + ' vessels\n')

        # Get details about failed vessel(s) and log them
        for vh in failed_list:
          try:
            vessel_log = explib.get_vessel_log(vh, config['identity'])
          except:
            vessel_log = '[ERROR: vessel log fetch failed]'

          nodeid, vesselname = explib.get_nodeid_and_vesselname(vh)
          nodelocation = explib.get_node_location(nodeid)

          # Log the vessel's log contents
          config['logfile'].write('Log contents of failed vessel at ' + nodelocation + ': ' + vessel_log + '\n')
          config['logfile'].flush()

        # Release the failed vessels
        release_vessels(failed_list, 'Releasing failed vessel(s)...')
        
        # Remove released vessels from fresh_vessels list
        fresh_vessels = list_difference(fresh_vessels, failed_list)

      # Add fresh_vessels to vesselhandle_list
      vesselhandle_list.extend(fresh_vessels)


    # Sleep for parameterized amount of time
    time.sleep(VESSEL_POLLING_TIME)
    
    # Log a liveness message every certain number of iterations
    loop_iterations += 1
    if loop_iterations % LOG_AFTER_THIS_MANY_LOOPS == 0:
      config['logfile'].write(str(time.time()) + ': Still alive...\n')
      config['logfile'].flush()

    # Renew vessels according to constant period
    if loop_iterations * VESSEL_POLLING_TIME > VESSEL_RENEWAL_PERIOD:
      explib.seattlegeni_renew_vessels(config['identity'], vesselhandle_list)
      loop_iterations = 0
Пример #6
0
  # Attempt to acquire vessels. Log success or failure, accordingly.
  try:
    vesselhandle_list = explib.seattlegeni_acquire_vessels(config['identity'],
                                                           config['vesseltype'],
                                                           number)
  except explib.SeattleGENIError, e:
    config['logfile'].write('failure\n')
    config['logfile'].write('Error was: ' + str(e) + '\n')
    return []
  else:
    config['logfile'].write('success\n')
  finally:
    config['logfile'].flush()

  # Renew vessels to maximum expiration time
  explib.seattlegeni_renew_vessels(config['identity'], vesselhandle_list)
  
  return vesselhandle_list





def upload_to_vessels(vesselhandle_list, filename):
  """
  <Purpose>
    Uploads a file to a set of vessels. A batch wrapper around the Experiment
    Library function upload_file_to_vessel, with logging support.

  <Arguments>
    vesselhandle_list
Пример #7
0
    def run(self, *args):
        """
    <Purpose>
      Starts the deployment and monitoring of a service on a number of vessels.
      Handles all acquisition of, uploading to, starting, and release of vessels.
      Contains the main loop of this program, and is thus the final function to
      call in all client programs. 
    <Arguments>
      *args: 
        Optional arguments that well be passed when starting the uploaded
        program file, by default

    <Exceptions>
      Common SeattleGENI exceptions described in the module comments.

    <Side Effects>
      Unless overridden beforehand, persistently logs to overlord_instance.log and
      console
      Returns if a 'stop' file is found in directory of the program importing the API

    <Returns>
      None
    """
        # Log setup information for user reference.
        self.logger.info("Performing actions as GENI user '" + self.config["identity"]["username"] + "'")
        self.logger.info("Vessel status will be polled every " + str(self.VESSEL_POLLING_PERIOD) + " seconds")
        self.logger.info("Vessels will be renewed every " + str(self.VESSEL_RENEWAL_PERIOD) + " seconds")

        # Run the start-up function for overlord
        self.init_overlord_func(self)

        # Retrieve the current list of vessel handles owned by identity, regardless
        # of the results of overlord's start-up function
        vessel_handlers = explib.seattlegeni_get_acquired_vessels(self.config["identity"])

        # If there are still vessels allocated to the identity, renew them to keep
        # renewal periods consistent with newly acquired vessels later on
        if vessel_handlers:
            self.logger.info("Renewing " + str(len(vessel_handlers)) + " vessels")
            explib.seattlegeni_renew_vessels(self.config["identity"], vessel_handlers)

        # Turn VESSEL_RENEWAL_PERIOD into a timedelta object.
        renewal_delay = timedelta(seconds=self.VESSEL_RENEWAL_PERIOD)

        # Assume that the vessels have been renewed.
        last_renewal = datetime.utcnow()

        # Main loop
        while True:

            # End operations if "stop" file is found in same directory as overlord.py
            if os.path.isfile("./stop"):
                self.logger.info("Stop file found; Discontinuing operations.")
                return

            # Call the acquire vessels function and set the list of the handles of the
            # newly acquired vessels
            fresh_handlers = self.acquire_vessels_func(self, vessel_handlers)

            # If the list is not empty, call the initiate vessel function, and an updated
            # list of currently owned vessel handles that includes the newly acquired ones
            # should be returned
            if fresh_handlers:
                vessel_handlers = self.init_vessels_func(self, fresh_handlers, vessel_handlers, *args)

            # Weed out any vessels based on the remove vessels function
            vessel_handlers = self.remove_vessels_func(self, vessel_handlers)

            # Perform any maintenance on the vessels, including renewals if needed,
            # and return the relevant information needed for next maintenance.
            # Currently coded to fit with default behavior, but should somehow be made a
            # little more flexible for user's function overrides?
            last_renewal = self.maintenance_func(self, renewal_delay, last_renewal, vessel_handlers)

            # If the list of running vessels does not match the requested vessel count,
            # repeat the loop until it does
            if not len(vessel_handlers) == self.config["vessel_count"]:
                continue

            # Sleep between polling vessel statuses.
            time.sleep(self.VESSEL_POLLING_PERIOD)
Пример #8
0
        try:
            vessel_handlers = explib.seattlegeni_acquire_vessels(
                self.config["identity"], self.config["vessel_type"], vessel_count
            )

        except explib.SeattleClearinghouseError, e:
            self.logger.error("Error while acquiring vessels: " + str(e))
            return []

        else:
            self.logger.debug("Successfully acquired " + str(vessel_count) + " vessels")

        # Renew vessels to extend their expiration time.
        self.logger.debug("Renewing " + str(len(vessel_handlers)) + " newly acquired vessels")
        explib.seattlegeni_renew_vessels(self.config["identity"], vessel_handlers)

        return vessel_handlers

    def upload_to_vessels(self, vessel_handlers, filename_list):
        """
    <Purpose>
      Uploads a list of file to a set of vessels. A batch wrapper around the Experiment
      Library function upload_file_to_vessel, with logging and parallelization support.

    <Arguments>
      vesselhandle_list
        A list of vesselhandles of vessels to which the file is to be uploaded.
      filename
        The filename of the file to be uploaded.