def _caretaker(self):
self.log.info("Caretaker waiting on lock...")
self.ct_lock.acquire()
self.log.info("Caretaker Running...")
dbI=self.db(self.databaseHost,self.databasePort,self.databaseInstance)
incomplete = dbI.getincompletetasks()
for task_id in incomplete:
jobid = dbI.getjobid(task_id)
jobrecord = dbI.getjob(jobid)
#TODO get task record not job record
job = Job().load(jobrecord)
for task in job.state.tasks:
if task.state.id == task_id and task.state.timeout and task.state.start_time:
if task.state.timeout < (time.time() - task.state.start_time):
self.log.info("Setting task {0} from job {1} as FAILED".format(task.state.id,job.state.id))
task.state.status = 'FAILED'
task.state.end_time = time.time()
job.state.status = 'FAILED'
job.state.end_time = time.time()
job.save()
dbI.updatejob(job)
self.ct_lock.release()
