class WorkManager():
#pass
def __init__(self, auth):
from stackone.viewModel.TaskCreator import TaskCreator
self.max_workers = 1
self.worker = u'DefaultWorker'
self.worker_ids = []
self.available_workers = self.max_workers
self.tc = TaskCreator()
self.auth = auth
self.start_time = datetime.now()
self.execution_context = {}
s = DBSession.query(ServiceItem).filter(ServiceItem.name == to_unicode('Task Manager Service')).one()
self.task_service_id = s.id
self.svc_central = base_config.stackone_service_central
try:
self.max_worker_wait_time = int(tg.config.get('max_worker_wait_time'))
except Exception as e:
print 'Exception: ',e
self.max_worker_wait_time = 300
#pass
def get_work(self):
raise Exception('get_work needs to be implemented.')
#pass
def get_task(self):
pass
#pass
def workers_in_progress(self):
return self.worker_ids
#pass
def make_entity_task_entries(self, task_id, entity_ids):
ent_tasks = []
#add 0902
WRK_LOGGER.debug('in make_entity_task_entries task_id : ' + str(task_id) + ' :entity_ids :' + str(entity_ids))
for ent_id in entity_ids:
try:
ent_task = EntityTasks(self.worker, to_unicode(task_id), ent_id, False, datetime.now())
ent_tasks.append(ent_task)
except Exception as e:
traceback.print_exc()
self.update_execution_context()
for ent_task in ent_tasks:
DBSession.merge(ent_task)
transaction.commit()
#add 0902
WRK_LOGGER.debug('in make_entity_task_entries committed task_id : ' + str(task_id) + ' :entity_ids :' + str(entity_ids))
#pass
def wait_for_workers_to_finish(self, task_ids):
WRK_LOGGER.debug('wait_for_workers_to_finish for ' + self.worker + ' max_worker_wait_time: ' + str(self.max_worker_wait_time))
task_completed = False
self.wait_start_time = datetime.now()
while task_completed == False:
time.sleep(5)
completed_tasks = self.check_tasks_completed(task_ids)
WRK_LOGGER.debug('wait_for_workers_to_finish for ' + self.worker + ' completed_tasks :' + str(completed_tasks))
if len(completed_tasks) > 0:
task_completed = True
for task in completed_tasks:
self.worker_ids.remove(task['task_id'])
WRK_LOGGER.debug('child task completed, update EntityTasks ' + self.worker + ' completed_tasks :' + str(task['task_id']))
ets = DBSession.query(EntityTasks).filter(EntityTasks.worker_id == to_unicode(task['task_id'])).all()
for et in ets:
et.worker_id = None
et.finished = True
et.end_time = datetime.now()
DBSession.merge(et)
transaction.commit()
WRK_LOGGER.debug('child tasks completed, updated EntityTasks ' + self.worker)
wait_time_sec = (datetime.now() - self.wait_start_time).seconds
WRK_LOGGER.debug('No completed child tasks for ' + self.worker + '. waiting for ' + str(wait_time_sec))
if wait_time_sec > self.max_worker_wait_time:
task_service = self.svc_central.get_service(self.task_service_id)
past_time = self.start_time - timedelta(minutes=1)
for task_id in task_ids:
task_obj = task_service.get_running_task_obj(task_id)
if task_obj:
(hung,completed,pending) = task_obj.get_running_status()
WRK_LOGGER.debug('HUNG STATUS for ' + self.worker + ':' + str(hung) + ':' + str(task_id) + ':' + str(completed) + ':' + str(pending))
if hung:
task_completed = True
self.worker_ids.remove(task_id)
WRK_LOGGER.debug('Hung task. Cleanup EntityTask for ' + self.worker + '. task id : ' + str(task_id))
DBSession.query(EntityTasks).filter(EntityTasks.worker == self.worker).filter(EntityTasks.entity_id.in_(completed)).update(dict(worker_id=None, finished=True, end_time=datetime.now()))
DBSession.query(EntityTasks).filter(EntityTasks.worker == self.worker).filter(EntityTasks.entity_id.in_(pending)).update(dict(worker_id=None, finished=True, start_time=past_time))
transaction.commit()
WRK_LOGGER.debug('Hung task. Cleaned up EntityTask for ' + self.worker + '. task id : ' + str(task_id))
#pass
def check_tasks_completed(self, task_ids):
transaction.begin()
tasks = DBSession.query(Task).filter(Task.task_id.in_(task_ids)).options(eagerload('result')).all()
completed_tasks = []
for task in tasks:
if task.is_finished():
completed_tasks.append(dict(task_id=task.task_id, status=task.result[0L].status))
transaction.commit()
return completed_tasks
#pass
def do_work(self):
WRK_LOGGER.debug('GETTING WORK for ' + self.worker)
new_work,entity_ids = self.get_work()
wip = self.workers_in_progress()
while new_work or wip:
if new_work and len(wip) < self.max_workers:
# add 0902
WRK_LOGGER.debug('Submitting new WORK for ' + self.worker)
work_id = self.tc.submit_task(new_work)
self.worker_ids.append(work_id)
WRK_LOGGER.debug('Submitting new WORK for ' + self.worker + ' : child task id : ' + str(work_id))
self.make_entity_task_entries(work_id, entity_ids)
else:
if len(wip) > 0:
WRK_LOGGER.debug('WAIT for ' + self.worker)
self.wait_for_workers_to_finish(wip)
LOGGER.debug('WAIT OVER for ' + self.worker)
WRK_LOGGER.debug('WAIT OVER for ' + self.worker)
WRK_LOGGER.debug('GETTING WORK new for ' + self.worker)
new_work,entity_ids = self.get_work()
wip = self.workers_in_progress()
#pass
def update_execution_context(self):
tid = TaskUtil.get_task_context()
WRK_LOGGER.debug('in update_execution_context Parent task : ' + str(tid) + ' : child tasks :' + str(self.worker_ids))
task = Task.get_task(tid)
if task is not None:
self.execution_context['start_time'] = self.start_time
self.execution_context['worker_ids'] = self.worker_ids
task.context['execution_context'] = self.execution_context
DBSession.add(task)
WRK_LOGGER.debug('in update_execution_context updated Parent task : ' + str(tid))
#pass
def resume_work(self, context):
execution_context = context['execution_context']
WRK_LOGGER.debug('RESUMING WORKER for :' + self.worker)
if execution_context:
self.start_time = execution_context.get('start_time', datetime.now())
self.worker_ids = execution_context.get('worker_ids', [])
self.sp_list = execution_context.get('sp_list', [])
ets = DBSession.query(EntityTasks).filter(EntityTasks.worker == self.worker).filter(not_(EntityTasks.worker_id.in_(self.worker_ids))).all()
#add 0902
if len.ents() > 0:
xtra_work_ids = [et.worker_id for et in ets]
WRK_LOGGER.error('GOT ENT Tasks different from execution_context :' + self.worker + ': CONTEXT WORKERS : ' + str(self.worker_ids) + ': XTRA WORKERS :' + str(xtra_work_ids))
r = DBSession.query(EntityTasks.entity_id).filter(EntityTasks.worker_id.in_(xtra_work_ids)).filter(EntityTasks.worker(self.worker.update=='values')).dict(worker_id =None,finished=True, end_time= datetime.now())
transaction.commit()
WRK_LOGGER.debug('Cleaned Up entity_tasks . worker:rows : ' + self.worker + ':' + str(r))
WRK_LOGGER.debug('RESUMING WORKER for :' + self.worker + ':' + str(self.start_time) + ':' + str(self.worker_ids))
self.do_work()
return None
