def pid_timeout():
  """
  <Purpose>
    This function is intented to be called once and supposed to run on a 
    separate thread. Until the 'kill' flag is set, it will spin and see
    which pid's need to be killed.
    
    All process IDs are set via the set_pid_timeout method.

  <Arguments>
    None.

  <Exceptions>
    OSError: the process no longer exists, ignore
    ValueError: when removing host from running hosts this means that the
      host has already been terminated.
    Any other exception is unexpected

  <Side Effects>
    None.

  <Returns>
    None.
  """
  # keeps spinning and sleeping, checking which PIDs need to be killed
  thread_communications['running_process_ids'] = []
  # while the kill flag is false. Kill flag is modified right before
  # exit
  while not thread_communications['kill_flag']:
    # sleep and wakeup every couple seconds.
    time.sleep(5)
    # this list will keep track of the pids that we've killed
    killed_pids = []
    
    # check the running_process_ids and see if any of them have expired
    for each_process in thread_communications['running_process_ids']:
      # each process is a tuple that consists of (pid, expiretime, hostname, username)
      process_to_kill = each_process[0]
      expire_time = each_process[1]
      remote_host = each_process[2]
      user = each_process[3]
      # if the current time is past the set expire time then we need to try and kill it
      if expire_time <= time.time():
        # try to kill process
        try:
          # check if process is still running
          if os.path.exists('/proc/'+str(process_to_kill)):
            os.kill(process_to_kill, 9)
            killed_pids.append(each_process)
            # sleep a second, and then check that the process was killed. if 
            # not, try a 2nd and third time
            time.sleep(1)
            if os.path.exists('/proc/'+str(process_to_kill)):
              # try os.kill again, and if that doesn't work, use shellexec method
              os.kill(process_to_kill, 9)
              time.sleep(1)
              if os.path.exists('/proc/'+str(process_to_kill)):
                deploy_helper.shellexec2('kill -9 '+str(process_to_kill))
                time.sleep(1)
            if remote_host:
              deploy_logging.logerror("Forced kill of PID "+str(process_to_kill)+" due to timeout! The host"+\
                      " on this thread is "+remote_host)
            else:
              deploy_logging.logerror("Forced kill of PID "+str(process_to_kill)+" due to timeout!")
            # subtract from out running thread count and remove host
            subtract_host_left([(user, remote_host)])
          else:
            # the process is dead, just remove host from hosts_left just in case, and
            # remove from running pids as well, but dont sub the # of threads
            killed_pids.append(each_process)
            subtract_host_left([(user, remote_host)], False)
            
        except OSError, ose:
          # this means no pid found and process has most likely 
          # already terminated
          deploy_logging.logerror("Process"+str(process_to_kill)+"("+remote_host+") is already done.")
          subtract_host_left([(user, remote_host)], False)
          pass
        except Exception, e:
          deploy_logging.logerror("Unexpected error in pid_timeout thread "+\
            "while killing a child process: "+str(e))
예제 #2
0
def remote_get_log(user, remote_host):
  """
  <Purpose>
    Gets the remote logs (all tarred up) from remote_host and copies it to a 
    local directory via scp then untars it into deploy.logs/[remote_host]/.

  <Arguments>
    user:
      the user to log in as
    remote_host:
      the IP of the host to get the logs from
    
  <Exceptions>
    scp fails/times out.

  <Side Effects>
    None.

  <Returns>
    No returns.
  """

  try:
    # set up dir that we'll move the remote .tar into
    if not os.path.isdir('./deploy.logs/'+remote_host):
      os.mkdir('./deploy.logs/'+remote_host)
    
    # download the tar file from remote host
    out, err, returncode = remote_download_file(remote_host+'.tgz', 
        './deploy.logs/'+remote_host+'/'+remote_host+'.tgz', user, remote_host)

    deploy_logging.log('Downloading logs', 'Logs downloaded from '+remote_host)
    # now try to untar the files

    # build up a command list to execute
    command_list = []

    # tar is picky about where it'll unzip to (CWD), so we'll just Cd there
    command_list.append('cd ./deploy.logs/'+remote_host+'/')

    # now untar. if deploy_main.verbosity >=1 then we'll be verbose
    if deploy_main.verbosity >=1:
      command_list.append('tar -xvvf '+remote_host+'.tgz')
    else:
      command_list.append('tar -xf '+remote_host+'.tgz')

    # not make command string by joining the list elements with '; '  
    command_string = '; '.join(command_list)

    # execute string
    out, err, retvalue = deploy_helper.shellexec2(command_string)

    deploy_logging.log('Downloading logs', 'Logs from '+remote_host+' are ready')

    # we no longer need the tar file, just hogging up space
    os.remove('./deploy.logs/'+remote_host+'/'+remote_host+'.tgz')

  except Exception, e:
    if deploy_main.verbosity == 2:
      # Only log if we error and need to narrow this down. otherwise,
      # it gets really spammy.    
      deploy_logging.logerror(remote_host+": Some kind of err in remote_get_log. ("+\
          remote_host+") , error:"+str(e)+")")
예제 #3
0
def pid_timeout():
    """
  <Purpose>
    This function is intented to be called once and supposed to run on a 
    separate thread. Until the 'kill' flag is set, it will spin and see
    which pid's need to be killed.
    
    All process IDs are set via the set_pid_timeout method.

  <Arguments>
    None.

  <Exceptions>
    OSError: the process no longer exists, ignore
    ValueError: when removing host from running hosts this means that the
      host has already been terminated.
    Any other exception is unexpected

  <Side Effects>
    None.

  <Returns>
    None.
  """
    # keeps spinning and sleeping, checking which PIDs need to be killed
    thread_communications['running_process_ids'] = []
    # while the kill flag is false. Kill flag is modified right before
    # exit
    while not thread_communications['kill_flag']:
        # sleep and wakeup every couple seconds.
        time.sleep(5)
        # this list will keep track of the pids that we've killed
        killed_pids = []

        # check the running_process_ids and see if any of them have expired
        for each_process in thread_communications['running_process_ids']:
            # each process is a tuple that consists of (pid, expiretime, hostname, username)
            process_to_kill = each_process[0]
            expire_time = each_process[1]
            remote_host = each_process[2]
            user = each_process[3]
            # if the current time is past the set expire time then we need to try and kill it
            if expire_time <= time.time():
                # try to kill process
                try:
                    # check if process is still running
                    if os.path.exists('/proc/' + str(process_to_kill)):
                        os.kill(process_to_kill, 9)
                        killed_pids.append(each_process)
                        # sleep a second, and then check that the process was killed. if
                        # not, try a 2nd and third time
                        time.sleep(1)
                        if os.path.exists('/proc/' + str(process_to_kill)):
                            # try os.kill again, and if that doesn't work, use shellexec method
                            os.kill(process_to_kill, 9)
                            time.sleep(1)
                            if os.path.exists('/proc/' + str(process_to_kill)):
                                deploy_helper.shellexec2('kill -9 ' +
                                                         str(process_to_kill))
                                time.sleep(1)
                        if remote_host:
                            deploy_logging.logerror("Forced kill of PID "+str(process_to_kill)+" due to timeout! The host"+\
                                    " on this thread is "+remote_host)
                        else:
                            deploy_logging.logerror("Forced kill of PID " +
                                                    str(process_to_kill) +
                                                    " due to timeout!")
                        # subtract from out running thread count and remove host
                        subtract_host_left([(user, remote_host)])
                    else:
                        # the process is dead, just remove host from hosts_left just in case, and
                        # remove from running pids as well, but dont sub the # of threads
                        killed_pids.append(each_process)
                        subtract_host_left([(user, remote_host)], False)

                except OSError, ose:
                    # this means no pid found and process has most likely
                    # already terminated
                    deploy_logging.logerror("Process" + str(process_to_kill) +
                                            "(" + remote_host +
                                            ") is already done.")
                    subtract_host_left([(user, remote_host)], False)
                    pass
                except Exception, e:
                    deploy_logging.logerror("Unexpected error in pid_timeout thread "+\
                      "while killing a child process: "+str(e))