def spawnImmediateTask(self, jobid, taskid_complex=None, params=None): ''' Run task immediately using basic configuration of job @param jobid:int @param taskid_complex: string @param params: string ''' job = Job.getByJobID(jobid) if not job: raise ValueError("Job(id:%d) is not existed or not enabled." % (jobid)) if params : job.set_command(job._substituteReservedWord(params)) if taskid_complex: task = Task(job.get_jobid(), uuid.uuid1().hex, datetime.now(), job.getCommandToExcute(), job.get_retry(), job, job.get_depend(), taskid_complex) else: task = Task(job.get_jobid(), uuid.uuid1().hex, datetime.now(), job.getCommandToExcute(), job.get_retry(), job, job.get_depend()) t = TaskRunner(0, task, self._status, self, params) t.daemon = True task.save() self._status.add_waiting_task(task, t) t.start() return task
def removeJob(self, jobid): ''' Remove one job from server @param jobid: int jobid We remove task from waiting queue but do not care of the running tasks ''' self._lock.acquire() try: job = Job.getByJobID(jobid) if not job: raise ValueError("Job(id:%d) is not exsited or not enabled." % (jobid)) self._status.remove_job(job) if jobid in self._depository: self._depository.pop(jobid) for job_list in self.hour_index.itervalues(): for job in job_list: if job.get_jobid() == jobid: job_list.remove(job) self.count -= 1 self._lock.release() except Exception, e: self._lock.release() raise e;
def runChildTask(self): ''' Run child tasks of complex job ''' jobid = self.task.get_jobid() conn = get_conn() sql = "select jobid_inclusion from CW_JOB_INCLUSION where jobid=%d" % (jobid) sql2 = "select status from CW_TASK where taskid='%s'" count, rst = conn.select(sql) if count: child_task = {} # KEY:TASK VALUE:TASKSTATUS for jobid_inclusion, in rst: job = Job.getByJobID(jobid_inclusion) if job: task = self.jobmanager.spawnImmediateTask(jobid=jobid_inclusion, taskid_complex=self.task.get_taskid(), params=self.params) child_task[task] = Status.RUNNING while True: done = True normal = True for task, task_status in child_task.iteritems(): if task_status != Status.SUCCESS: conn = get_conn() count2, rst2 = conn.select(sql2 % (task.get_taskid())) if count2 == 1: # Task is not written to database t_status = rst2[0][0] self.logger.debug ("check db " + task.get_taskid() + " " + str(t_status)) child_task[task] = t_status if t_status != Status.SUCCESS: done = False if t_status == Status.FAIL or t_status == Status.CANCEL: child_task[task] = t_status normal = False else: done = False if done == True: # All children success return Status.SUCCESS elif not normal: # Some Task fails or cancels,then we cancel all other running for task, task_status in child_task.iteritems(): self.logger.debug ("check dict " + task.get_taskid() + " " + str(task_status)) if task_status in (Status.RUNNING, Status.BLOCKING): self.server_status.cancelTask(task) self.logger.debug ("task %s is canceled by complex task." % task.get_taskid()) if self.stopped: return Status.CANCEL else : return Status.FAIL else: if self.stopped: return Status.CANCEL else : return Status.FAIL time.sleep(_server_conf[_eviron]['COMPLEX_JOB_CHECKER_SECONDS'])
def __init__(self, jobid, taskid, exc_time, command, retry, job=None, depend=None, taskid_complex=None, status=Status.WAITING): self._jobid = jobid self._taskid = taskid self._exc_time = exc_time self._command = command self._retry = retry self._depend = depend self._status = status self._taskid_complex = taskid_complex self._endtime = None if job: self._job = job else: j = Job.getByJobID(jobid) self._job = j self._depend = j.get_depend()
def spawnTask(self, jobid, start_time, end_time): ''' Spawn thread based on one Job's timer @param jobid:int @param start_time:datetime ,start time of time section @param end_time:datetime ,end time of time section ''' tasks = [] self._lock.acquire() try: job = Job.getByJobID(jobid) if not job: raise ValueError("Job(id:%d) is not exsited or not enabled." % (jobid)) # Clear second and be care of start minute determine_time = start_time if start_time.second != 0: determine_time = start_time + timedelta(minutes=1) determine_time = datetime.combine(determine_time.date(), time(hour=determine_time.hour, minute=determine_time.minute)) while determine_time < end_time: if TimeMatcher.matchTimePattern(job.get_time_pattern(), determine_time): task = Task(job.get_jobid(), uuid.uuid1().hex, determine_time, job.getCommandToExcute(), job.get_retry(), job, job.get_depend()) t = TaskRunner((determine_time - datetime.now()).seconds + 1, task, self._status, self) t.daemon = True t.start() task.save() tasks.append(task) self._status.add_waiting_task(task, t) determine_time = determine_time + timedelta(minutes=1) self._lock.release() except Exception, e: self._lock.release() raise e;
def addJob(self, jobid): ''' Add one job to Server and then server start to schedule it. @param jobid:int ''' jobid = int(jobid) if jobid in self._depository: raise ValueError("Job(id:%d) has been loaded." % (jobid)) else: self._lock.acquire() try: job = Job.getByJobID(jobid) if not job: raise ValueError("Job(id:%d) is not exsited or not enabled." % (jobid)) self.loadJob(job) self.indexJob(job) self._lock.release() except Exception, e: self._lock.release() raise e; self._logger.info("Job(id:%d) is added." % (jobid)) self._server.spawnTaskThread(jobid)