def dequeue_task(self):
sq = sess.query(TaskQueue).order_by(TaskQueue.id).filter(TaskQueue.has_processed == 0)
task = sq.first()
res = None
if task:
sq2 = sess.query(MRTask).filter(MRTask.id == task.mr_task_id)
mr_task = sq2.first()
atask = ActiveTask(mr_task)
atask.table_stage_list = task.table_stage_list
atask.begin_time = CommonUtil.getCurrentDatetime()
res = (atask, task.id)
else:
res = (None,None)
sess.commit()
sess.flush()
return res
def _get_task_queue(self):
self.logger.info("Begin to execute _get_task_queue")
task_queue_list = []
sq = sess.query(TaskQueue).filter(TaskQueue.has_processed == 0)
for s in sq:
task_queue_list.append(s)
self.logger.info("Success to execute _get_task_queue")
return task_queue_list
def _get_mr_tasks(self):
self.logger.info("Begin to execute _get_mr_tasks")
mr_task_list = [] # MRTask list
sq = sess.query(MRTask).filter(MRTask.type < 2)
for s in sq:
mr_task_list.append(s)
self.logger.info("Success to execute _get_mr_tasks")
return mr_task_list
def set_task_processed(self, queued_id):
queued_task = sess.query(TaskQueue).get(queued_id)
if queued_task:
queued_task.has_processed = 1
queued_task.end_time = CommonUtil.getCurrentDatetime().replace(tzinfo=None)
sess.commit()
sess.flush()
else:
raise ValueError("Can not find the queued_id(" + queued_id + ") from task queue")
def _get_lastest_stages(self):
self.logger.info("Begin to execute _get_latest_stages")
stages = []
# Get not-processed stages
sq = sess.query(Stage).order_by(
Stage.id).filter(Stage.process_status == 0)
for s in sq:
stages.append(s)
# Get normal ordered stages
self.logger.info("Success to execute _get_latest_stages")
# return stages[::-1]
return stages
def _update_stage_to_processed(self):
self.logger.info("Begin to _update_stage_to_processed")
# Get not-processed stages from the bottom of the table
sq = sess.query(Stage).order_by(desc(
Stage.id)).filter(Stage.process_status == 0)
processed_stages = set()
for stage in self.stages:
processed_stages.add(stage.id)
for s in sq:
if s.id in processed_stages:
self.logger.debug("Updating stage(stage_id=%s)" % s.id)
s.process_status = 1
sess.commit()
sess.flush()
def enqueue_time_task(self):
self.logger.info("Begin to enque_time_task")
sq_task = sess.query(MRTask).filter(MRTask.type > 1)
for s in sq_task:
#sq2_queue = sess.query(TaskQueue).filter(TaskQueue.mr_task_id == s.id)
#qtask = sq2_queue.first()
# time task not in task queue, we should enqueue the task
if s.time_to_process():
tq = TaskQueue(mr_task_id=s.id)
tq.create_time = CommonUtil.getCurrentDatetime().replace(
tzinfo=None)
#s.update_time == tq.create_time
tq.has_processed = 0
sess.add(tq)
sess.commit()
sess.flush()
from common.db.db_config import session as sess
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy import Column
from sqlalchemy.dialects.mysql import INTEGER, DATETIME, TEXT, TINYINT, VARCHAR
from sqlalchemy.schema import ForeignKey
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy import desc
Base = declarative_base()
# tb_task_queue table
class Animal(Base):
__tablename__ = "animals"
id = Column('id', INTEGER(11), primary_key=True)
name = Column('name', VARCHAR(50))
sq = sess.query(Animal).order_by(desc(Animal.id))
for s in sq:
print(s)
print(s.id)
def delete_queued_task(self, queued_id):
self.logger.info("delete_queued_task: delete task from queue(queued_id = %s)" % queued_id)
tq = sess.query(TaskQueue).get(queued_id)
sess.delete(tq)
sess.commit()
sess.flush()
