def dequeue_task(self): sq = sess.query(TaskQueue).order_by(TaskQueue.id).filter(TaskQueue.has_processed == 0) task = sq.first() res = None if task: sq2 = sess.query(MRTask).filter(MRTask.id == task.mr_task_id) mr_task = sq2.first() atask = ActiveTask(mr_task) atask.table_stage_list = task.table_stage_list atask.begin_time = CommonUtil.getCurrentDatetime() res = (atask, task.id) else: res = (None,None) sess.commit() sess.flush() return res
def _get_task_queue(self): self.logger.info("Begin to execute _get_task_queue") task_queue_list = [] sq = sess.query(TaskQueue).filter(TaskQueue.has_processed == 0) for s in sq: task_queue_list.append(s) self.logger.info("Success to execute _get_task_queue") return task_queue_list
def _get_mr_tasks(self): self.logger.info("Begin to execute _get_mr_tasks") mr_task_list = [] # MRTask list sq = sess.query(MRTask).filter(MRTask.type < 2) for s in sq: mr_task_list.append(s) self.logger.info("Success to execute _get_mr_tasks") return mr_task_list
def set_task_processed(self, queued_id): queued_task = sess.query(TaskQueue).get(queued_id) if queued_task: queued_task.has_processed = 1 queued_task.end_time = CommonUtil.getCurrentDatetime().replace(tzinfo=None) sess.commit() sess.flush() else: raise ValueError("Can not find the queued_id(" + queued_id + ") from task queue")
def _get_lastest_stages(self): self.logger.info("Begin to execute _get_latest_stages") stages = [] # Get not-processed stages sq = sess.query(Stage).order_by( Stage.id).filter(Stage.process_status == 0) for s in sq: stages.append(s) # Get normal ordered stages self.logger.info("Success to execute _get_latest_stages") # return stages[::-1] return stages
def _update_stage_to_processed(self): self.logger.info("Begin to _update_stage_to_processed") # Get not-processed stages from the bottom of the table sq = sess.query(Stage).order_by(desc( Stage.id)).filter(Stage.process_status == 0) processed_stages = set() for stage in self.stages: processed_stages.add(stage.id) for s in sq: if s.id in processed_stages: self.logger.debug("Updating stage(stage_id=%s)" % s.id) s.process_status = 1 sess.commit() sess.flush()
def enqueue_time_task(self): self.logger.info("Begin to enque_time_task") sq_task = sess.query(MRTask).filter(MRTask.type > 1) for s in sq_task: #sq2_queue = sess.query(TaskQueue).filter(TaskQueue.mr_task_id == s.id) #qtask = sq2_queue.first() # time task not in task queue, we should enqueue the task if s.time_to_process(): tq = TaskQueue(mr_task_id=s.id) tq.create_time = CommonUtil.getCurrentDatetime().replace( tzinfo=None) #s.update_time == tq.create_time tq.has_processed = 0 sess.add(tq) sess.commit() sess.flush()
from common.db.db_config import session as sess from sqlalchemy.ext.declarative import declarative_base from sqlalchemy import Column from sqlalchemy.dialects.mysql import INTEGER, DATETIME, TEXT, TINYINT, VARCHAR from sqlalchemy.schema import ForeignKey from sqlalchemy.ext.declarative import declarative_base from sqlalchemy import desc Base = declarative_base() # tb_task_queue table class Animal(Base): __tablename__ = "animals" id = Column('id', INTEGER(11), primary_key=True) name = Column('name', VARCHAR(50)) sq = sess.query(Animal).order_by(desc(Animal.id)) for s in sq: print(s) print(s.id)
def delete_queued_task(self, queued_id): self.logger.info("delete_queued_task: delete task from queue(queued_id = %s)" % queued_id) tq = sess.query(TaskQueue).get(queued_id) sess.delete(tq) sess.commit() sess.flush()