def wait_for_workers_to_finish(self, task_ids):
WRK_LOGGER.debug("wait_for_workers_to_finish for "+self.worker+" max_worker_wait_time: "+str(self.max_worker_wait_time))
task_completed = False
self.wait_start_time=datetime.utcnow()
###this is an infinite loop until we find a completed task
###we need to add some wait time to check on the status of child tasks
while task_completed == False:
time.sleep(5)
completed_tasks = self.check_tasks_completed(task_ids)
WRK_LOGGER.debug("wait_for_workers_to_finish for "+self.worker+" completed_tasks :"+str(completed_tasks))
if len(completed_tasks) > 0:
task_completed = True
for task in completed_tasks:
self.worker_ids.remove(task['task_id'])
WRK_LOGGER.debug("child task completed, update EntityTasks "+self.worker+" completed_tasks :"+str(task['task_id']))
ets = DBSession.query(EntityTasks).\
filter(EntityTasks.worker_id==to_unicode(task['task_id'])).all()
for et in ets:
et.worker_id=None
et.finished=True
et.end_time=datetime.utcnow()
DBSession.merge(et)
transaction.commit()
WRK_LOGGER.debug("child tasks completed, updated EntityTasks "+self.worker)
else :
# if True:
# continue
wait_time_sec=(datetime.utcnow()-self.wait_start_time).seconds
WRK_LOGGER.debug("No completed child tasks for "+self.worker+". waiting for "+str(wait_time_sec))
if wait_time_sec > self.max_worker_wait_time:
task_service = self.svc_central.get_service(self.task_service_id)
past_time = self.start_time-timedelta(minutes=1)
for task_id in task_ids:
task_obj = task_service.get_running_task_obj(task_id)
if task_obj:
(hung, completed, pending) = task_obj.get_running_status()
WRK_LOGGER.debug("HUNG STATUS for "+self.worker+":"+str(hung)+":"+str(task_id)+\
":"+str(completed)+":"+str(pending))
if hung:
task_completed = True
self.worker_ids.remove(task_id)
WRK_LOGGER.debug("Hung task. Cleanup EntityTask for "+self.worker+". task id : "+str(task_id))
DBSession.query(EntityTasks).filter(EntityTasks.worker==self.worker).\
filter(EntityTasks.entity_id.in_(completed)).\
update(dict(worker_id=None,finished=True, end_time=datetime.utcnow()))
DBSession.query(EntityTasks).filter(EntityTasks.worker==self.worker).\
filter(EntityTasks.entity_id.in_(pending)).\
update(dict(worker_id=None,finished=True, start_time=past_time))
transaction.commit()
WRK_LOGGER.debug("Hung task. Cleaned up EntityTask for "+self.worker+". task id : "+str(task_id))
def wait_for_workers_to_finish(self, task_ids):
WRK_LOGGER.debug("wait_for_workers_to_finish for " + self.worker +
" max_worker_wait_time: " +
str(self.max_worker_wait_time))
task_completed = False
self.wait_start_time = datetime.utcnow()
###this is an infinite loop until we find a completed task
###we need to add some wait time to check on the status of child tasks
while task_completed == False:
time.sleep(5)
completed_tasks = self.check_tasks_completed(task_ids)
WRK_LOGGER.debug("wait_for_workers_to_finish for " + self.worker +
" completed_tasks :" + str(completed_tasks))
if len(completed_tasks) > 0:
task_completed = True
for task in completed_tasks:
self.worker_ids.remove(task['task_id'])
WRK_LOGGER.debug(
"child task completed, update EntityTasks " +
self.worker + " completed_tasks :" +
str(task['task_id']))
ets = DBSession.query(EntityTasks).\
filter(EntityTasks.worker_id==to_unicode(task['task_id'])).all()
for et in ets:
et.worker_id = None
et.finished = True
et.end_time = datetime.utcnow()
DBSession.merge(et)
transaction.commit()
WRK_LOGGER.debug(
"child tasks completed, updated EntityTasks " +
self.worker)
else:
# if True:
# continue
wait_time_sec = (datetime.utcnow() -
self.wait_start_time).seconds
WRK_LOGGER.debug("No completed child tasks for " +
self.worker + ". waiting for " +
str(wait_time_sec))
if wait_time_sec > self.max_worker_wait_time:
task_service = self.svc_central.get_service(
self.task_service_id)
past_time = self.start_time - timedelta(minutes=1)
for task_id in task_ids:
task_obj = task_service.get_running_task_obj(task_id)
if task_obj:
(hung, completed,
pending) = task_obj.get_running_status()
WRK_LOGGER.debug("HUNG STATUS for "+self.worker+":"+str(hung)+":"+str(task_id)+\
":"+str(completed)+":"+str(pending))
if hung:
task_completed = True
self.worker_ids.remove(task_id)
WRK_LOGGER.debug(
"Hung task. Cleanup EntityTask for " +
self.worker + ". task id : " +
str(task_id))
DBSession.query(EntityTasks).filter(EntityTasks.worker==self.worker).\
filter(EntityTasks.entity_id.in_(completed)).\
update(dict(worker_id=None,finished=True, end_time=datetime.utcnow()))
DBSession.query(EntityTasks).filter(EntityTasks.worker==self.worker).\
filter(EntityTasks.entity_id.in_(pending)).\
update(dict(worker_id=None,finished=True, start_time=past_time))
transaction.commit()
WRK_LOGGER.debug(
"Hung task. Cleaned up EntityTask for " +
self.worker + ". task id : " +
str(task_id))
WRK_LOGGER.debug("in make_entity_task_entries task_id : " +
str(task_id) + " :entity_ids :" + str(entity_ids))
for ent_id in entity_ids:
try:
###update the entity_tasks table
ent_task = EntityTasks(self.worker, to_unicode(task_id),
ent_id, False, datetime.utcnow())
ent_tasks.append(ent_task)
except Exception, e:
traceback.print_exc()
# transaction.commit()
self.update_execution_context()
for ent_task in ent_tasks:
DBSession.merge(ent_task)
transaction.commit()
WRK_LOGGER.debug("in make_entity_task_entries committed task_id : " +
str(task_id) + " :entity_ids :" + str(entity_ids))
def wait_for_workers_to_finish(self, task_ids):
WRK_LOGGER.debug("wait_for_workers_to_finish for " + self.worker +
" max_worker_wait_time: " +
str(self.max_worker_wait_time))
task_completed = False
self.wait_start_time = datetime.utcnow()
###this is an infinite loop until we find a completed task
###we need to add some wait time to check on the status of child tasks
while task_completed == False:
# estimated_time=datetime.utcnow()+timedelta(minutes=int(tg.config.get("completion_time")))
ent_tasks = []
WRK_LOGGER.debug("in make_entity_task_entries task_id : "+str(task_id)+ " :entity_ids :"+str(entity_ids))
for ent_id in entity_ids:
try:
###update the entity_tasks table
ent_task=EntityTasks(self.worker,to_unicode(task_id),ent_id,False,datetime.utcnow())
ent_tasks.append(ent_task)
except Exception, e:
traceback.print_exc()
# transaction.commit()
self.update_execution_context()
for ent_task in ent_tasks:
DBSession.merge(ent_task)
transaction.commit()
WRK_LOGGER.debug("in make_entity_task_entries committed task_id : "+str(task_id)+ " :entity_ids :"+str(entity_ids))
def wait_for_workers_to_finish(self, task_ids):
WRK_LOGGER.debug("wait_for_workers_to_finish for "+self.worker+" max_worker_wait_time: "+str(self.max_worker_wait_time))
task_completed = False
self.wait_start_time=datetime.utcnow()
###this is an infinite loop until we find a completed task
###we need to add some wait time to check on the status of child tasks
while task_completed == False:
time.sleep(5)
completed_tasks = self.check_tasks_completed(task_ids)
