def kick_downloader(speed):
""" write speed and signal dowloader to update """
with open(DAEMON_PIPE, 'w') as filehandle:
filehandle.write(speed + "\n")
# if running, signal to update config
if megatools_status():
log.info("sending signal to megatools")
utils.kill("megatools", signal.SIGHUP)
def clear(self): ''' Pre-destructor code to: break circular references between the watchdog and the thread kill child processes ''' self.shuttingDown = True #get the PIDs of the processes associated with this thread pids = self.getPIDs() #and kill them all utils.kill([pids], self.server.waitMutex) self.server.watchdog.removeThread(self.threadId) #inform watchdog that this thread is shutting down self.scriptRunner.clear() self.server = None
def daemon_stop():
""" Stop the scheduling daemon """
if not daemon_status():
print("> Scheduler not running.")
return
# try stopping it
with open(DAEMON_PIDFILE, 'r') as filehandle:
error = utils.kill(filehandle.read(), signo=signal.SIGHUP,
cat="pid") != 0
if error:
print("Couldn't stop scheduler.")
def run(self):
# The main execution function for watchdog
while not self.readyToExit:
time.sleep(self.interval.seconds)
# check to see if any children have died, and reap them if they have. Then notify the parents.
self.blackStork()
# need to acquire a lock here for the threads list since hatching and dying threads can modify this list
self.threadsMutex.acquire()
if self.threads == {}: # then all threads have been removed and it's time to shut down
self.clear()
else:
processes = utils.checkProcesses(self.threads, self.server.waitMutex)
try:
for threadId in self.threads.keys():
# if this thread is processing a job, determine whether or not it's been running too long
if not self.threads[threadId]["ticket"] is "ready":
# first add this interval's sample of the CPU usage
self.threads[threadId]["cpu"] += processes[threadId][0]
# if the job has been running longer than self.jobTimeout seconds
if self.jobTimeout < (datetime.datetime.now() - self.threads[threadId]["timestamp"]):
# if there haven't been too many extensions granted, and the average CPU usage has been at least self.minCPU
if (
self.threads[threadId]["extensions granted"] < self.maxExtensions
and self.threads[threadId]["cpu"]
/ (self.jobTimeout.seconds // self.interval.seconds)
> self.minCPU
):
self.threads[threadId][
"extensions granted"
] += 1 # note that another extension has been granted
self.threads[threadId]["timestamp"] += (
self.jobTimeout / 2
) # pretends that the job started 1/2 of jobTimeout later
else:
"""
This job's time has run out. Kill the processes and deathNotify singleProcess.
Reset the timestamp and the extensions granted.
this will not become an infinite time extension because singleProcess will give up
on a job after too many failures.
"""
self.log(
"Watchdog is killing job "
+ self.threads[threadId]["ticket"]
+ " in thread "
+ threadId
+ " for taking too long."
)
self.threads[threadId]["extensions granted"] = 0
self.threads[threadId]["timestamp"] = datetime.datetime.now()
utils.kill(self.threads[threadId]["processes"], self.server.waitMutex)
# restarting a thread takes so long that it makes sense to refresh information about the threads
processes = utils.checkProcesses(self.threads, self.server.waitMutex)
# in case removeThread() was called while watchdog was running
except KeyError:
pass
# release the mutex so that runScript can deal with its dead child
self.threadsMutex.release()
