def pid_timeout():
"""
<Purpose>
This function is intented to be called once and supposed to run on a
separate thread. Until the 'kill' flag is set, it will spin and see
which pid's need to be killed.
All process IDs are set via the set_pid_timeout method.
<Arguments>
None.
<Exceptions>
OSError: the process no longer exists, ignore
ValueError: when removing host from running hosts this means that the
host has already been terminated.
Any other exception is unexpected
<Side Effects>
None.
<Returns>
None.
"""
# keeps spinning and sleeping, checking which PIDs need to be killed
thread_communications['running_process_ids'] = []
# while the kill flag is false. Kill flag is modified right before
# exit
while not thread_communications['kill_flag']:
# sleep and wakeup every couple seconds.
time.sleep(5)
# this list will keep track of the pids that we've killed
killed_pids = []
# check the running_process_ids and see if any of them have expired
for each_process in thread_communications['running_process_ids']:
# each process is a tuple that consists of (pid, expiretime, hostname, username)
process_to_kill = each_process[0]
expire_time = each_process[1]
remote_host = each_process[2]
user = each_process[3]
# if the current time is past the set expire time then we need to try and kill it
if expire_time <= time.time():
# try to kill process
try:
# check if process is still running
if os.path.exists('/proc/'+str(process_to_kill)):
os.kill(process_to_kill, 9)
killed_pids.append(each_process)
# sleep a second, and then check that the process was killed. if
# not, try a 2nd and third time
time.sleep(1)
if os.path.exists('/proc/'+str(process_to_kill)):
# try os.kill again, and if that doesn't work, use shellexec method
os.kill(process_to_kill, 9)
time.sleep(1)
if os.path.exists('/proc/'+str(process_to_kill)):
deploy_helper.shellexec2('kill -9 '+str(process_to_kill))
time.sleep(1)
if remote_host:
deploy_logging.logerror("Forced kill of PID "+str(process_to_kill)+" due to timeout! The host"+\
" on this thread is "+remote_host)
else:
deploy_logging.logerror("Forced kill of PID "+str(process_to_kill)+" due to timeout!")
# subtract from out running thread count and remove host
subtract_host_left([(user, remote_host)])
else:
# the process is dead, just remove host from hosts_left just in case, and
# remove from running pids as well, but dont sub the # of threads
killed_pids.append(each_process)
subtract_host_left([(user, remote_host)], False)
except OSError, ose:
# this means no pid found and process has most likely
# already terminated
deploy_logging.logerror("Process"+str(process_to_kill)+"("+remote_host+") is already done.")
subtract_host_left([(user, remote_host)], False)
pass
except Exception, e:
deploy_logging.logerror("Unexpected error in pid_timeout thread "+\
"while killing a child process: "+str(e))
def remote_get_log(user, remote_host):
"""
<Purpose>
Gets the remote logs (all tarred up) from remote_host and copies it to a
local directory via scp then untars it into deploy.logs/[remote_host]/.
<Arguments>
user:
the user to log in as
remote_host:
the IP of the host to get the logs from
<Exceptions>
scp fails/times out.
<Side Effects>
None.
<Returns>
No returns.
"""
try:
# set up dir that we'll move the remote .tar into
if not os.path.isdir('./deploy.logs/'+remote_host):
os.mkdir('./deploy.logs/'+remote_host)
# download the tar file from remote host
out, err, returncode = remote_download_file(remote_host+'.tgz',
'./deploy.logs/'+remote_host+'/'+remote_host+'.tgz', user, remote_host)
deploy_logging.log('Downloading logs', 'Logs downloaded from '+remote_host)
# now try to untar the files
# build up a command list to execute
command_list = []
# tar is picky about where it'll unzip to (CWD), so we'll just Cd there
command_list.append('cd ./deploy.logs/'+remote_host+'/')
# now untar. if deploy_main.verbosity >=1 then we'll be verbose
if deploy_main.verbosity >=1:
command_list.append('tar -xvvf '+remote_host+'.tgz')
else:
command_list.append('tar -xf '+remote_host+'.tgz')
# not make command string by joining the list elements with '; '
command_string = '; '.join(command_list)
# execute string
out, err, retvalue = deploy_helper.shellexec2(command_string)
deploy_logging.log('Downloading logs', 'Logs from '+remote_host+' are ready')
# we no longer need the tar file, just hogging up space
os.remove('./deploy.logs/'+remote_host+'/'+remote_host+'.tgz')
except Exception, e:
if deploy_main.verbosity == 2:
# Only log if we error and need to narrow this down. otherwise,
# it gets really spammy.
deploy_logging.logerror(remote_host+": Some kind of err in remote_get_log. ("+\
remote_host+") , error:"+str(e)+")")
def pid_timeout():
"""
<Purpose>
This function is intented to be called once and supposed to run on a
separate thread. Until the 'kill' flag is set, it will spin and see
which pid's need to be killed.
All process IDs are set via the set_pid_timeout method.
<Arguments>
None.
<Exceptions>
OSError: the process no longer exists, ignore
ValueError: when removing host from running hosts this means that the
host has already been terminated.
Any other exception is unexpected
<Side Effects>
None.
<Returns>
None.
"""
# keeps spinning and sleeping, checking which PIDs need to be killed
thread_communications['running_process_ids'] = []
# while the kill flag is false. Kill flag is modified right before
# exit
while not thread_communications['kill_flag']:
# sleep and wakeup every couple seconds.
time.sleep(5)
# this list will keep track of the pids that we've killed
killed_pids = []
# check the running_process_ids and see if any of them have expired
for each_process in thread_communications['running_process_ids']:
# each process is a tuple that consists of (pid, expiretime, hostname, username)
process_to_kill = each_process[0]
expire_time = each_process[1]
remote_host = each_process[2]
user = each_process[3]
# if the current time is past the set expire time then we need to try and kill it
if expire_time <= time.time():
# try to kill process
try:
# check if process is still running
if os.path.exists('/proc/' + str(process_to_kill)):
os.kill(process_to_kill, 9)
killed_pids.append(each_process)
# sleep a second, and then check that the process was killed. if
# not, try a 2nd and third time
time.sleep(1)
if os.path.exists('/proc/' + str(process_to_kill)):
# try os.kill again, and if that doesn't work, use shellexec method
os.kill(process_to_kill, 9)
time.sleep(1)
if os.path.exists('/proc/' + str(process_to_kill)):
deploy_helper.shellexec2('kill -9 ' +
str(process_to_kill))
time.sleep(1)
if remote_host:
deploy_logging.logerror("Forced kill of PID "+str(process_to_kill)+" due to timeout! The host"+\
" on this thread is "+remote_host)
else:
deploy_logging.logerror("Forced kill of PID " +
str(process_to_kill) +
" due to timeout!")
# subtract from out running thread count and remove host
subtract_host_left([(user, remote_host)])
else:
# the process is dead, just remove host from hosts_left just in case, and
# remove from running pids as well, but dont sub the # of threads
killed_pids.append(each_process)
subtract_host_left([(user, remote_host)], False)
except OSError, ose:
# this means no pid found and process has most likely
# already terminated
deploy_logging.logerror("Process" + str(process_to_kill) +
"(" + remote_host +
") is already done.")
subtract_host_left([(user, remote_host)], False)
pass
except Exception, e:
deploy_logging.logerror("Unexpected error in pid_timeout thread "+\
"while killing a child process: "+str(e))