def spawnImmediateTask(self, jobid, taskid_complex=None, params=None):
'''
Run task immediately using basic configuration of job
@param jobid:int
@param taskid_complex: string
@param params: string
'''
job = Job.getByJobID(jobid)
if not job:
raise ValueError("Job(id:%d) is not existed or not enabled." % (jobid))
if params :
job.set_command(job._substituteReservedWord(params))
if taskid_complex:
task = Task(job.get_jobid(), uuid.uuid1().hex, datetime.now(),
job.getCommandToExcute(), job.get_retry(), job, job.get_depend(), taskid_complex)
else:
task = Task(job.get_jobid(), uuid.uuid1().hex, datetime.now(),
job.getCommandToExcute(), job.get_retry(), job, job.get_depend())
t = TaskRunner(0, task, self._status, self, params)
t.daemon = True
task.save()
self._status.add_waiting_task(task, t)
t.start()
return task
def removeJob(self, jobid):
'''
Remove one job from server
@param jobid: int jobid
We remove task from waiting queue but do not care of the running tasks
'''
self._lock.acquire()
try:
job = Job.getByJobID(jobid)
if not job:
raise ValueError("Job(id:%d) is not exsited or not enabled." % (jobid))
self._status.remove_job(job)
if jobid in self._depository:
self._depository.pop(jobid)
for job_list in self.hour_index.itervalues():
for job in job_list:
if job.get_jobid() == jobid:
job_list.remove(job)
self.count -= 1
self._lock.release()
except Exception, e:
self._lock.release()
raise e;
def runChildTask(self):
'''
Run child tasks of complex job
'''
jobid = self.task.get_jobid()
conn = get_conn()
sql = "select jobid_inclusion from CW_JOB_INCLUSION where jobid=%d" % (jobid)
sql2 = "select status from CW_TASK where taskid='%s'"
count, rst = conn.select(sql)
if count:
child_task = {} # KEY:TASK VALUE:TASKSTATUS
for jobid_inclusion, in rst:
job = Job.getByJobID(jobid_inclusion)
if job:
task = self.jobmanager.spawnImmediateTask(jobid=jobid_inclusion, taskid_complex=self.task.get_taskid(), params=self.params)
child_task[task] = Status.RUNNING
while True:
done = True
normal = True
for task, task_status in child_task.iteritems():
if task_status != Status.SUCCESS:
conn = get_conn()
count2, rst2 = conn.select(sql2 % (task.get_taskid()))
if count2 == 1: # Task is not written to database
t_status = rst2[0][0]
self.logger.debug ("check db " + task.get_taskid() + " " + str(t_status))
child_task[task] = t_status
if t_status != Status.SUCCESS:
done = False
if t_status == Status.FAIL or t_status == Status.CANCEL:
child_task[task] = t_status
normal = False
else:
done = False
if done == True: # All children success
return Status.SUCCESS
elif not normal: # Some Task fails or cancels,then we cancel all other running
for task, task_status in child_task.iteritems():
self.logger.debug ("check dict " + task.get_taskid() + " " + str(task_status))
if task_status in (Status.RUNNING, Status.BLOCKING):
self.server_status.cancelTask(task)
self.logger.debug ("task %s is canceled by complex task." % task.get_taskid())
if self.stopped:
return Status.CANCEL
else :
return Status.FAIL
else:
if self.stopped:
return Status.CANCEL
else :
return Status.FAIL
time.sleep(_server_conf[_eviron]['COMPLEX_JOB_CHECKER_SECONDS'])
def __init__(self, jobid, taskid, exc_time, command, retry, job=None, depend=None, taskid_complex=None, status=Status.WAITING):
self._jobid = jobid
self._taskid = taskid
self._exc_time = exc_time
self._command = command
self._retry = retry
self._depend = depend
self._status = status
self._taskid_complex = taskid_complex
self._endtime = None
if job:
self._job = job
else:
j = Job.getByJobID(jobid)
self._job = j
self._depend = j.get_depend()
def spawnTask(self, jobid, start_time, end_time):
'''
Spawn thread based on one Job's timer
@param jobid:int
@param start_time:datetime ,start time of time section
@param end_time:datetime ,end time of time section
'''
tasks = []
self._lock.acquire()
try:
job = Job.getByJobID(jobid)
if not job:
raise ValueError("Job(id:%d) is not exsited or not enabled." % (jobid))
# Clear second and be care of start minute
determine_time = start_time
if start_time.second != 0:
determine_time = start_time + timedelta(minutes=1)
determine_time = datetime.combine(determine_time.date(), time(hour=determine_time.hour, minute=determine_time.minute))
while determine_time < end_time:
if TimeMatcher.matchTimePattern(job.get_time_pattern(), determine_time):
task = Task(job.get_jobid(), uuid.uuid1().hex, determine_time,
job.getCommandToExcute(), job.get_retry(), job, job.get_depend())
t = TaskRunner((determine_time - datetime.now()).seconds + 1,
task, self._status, self)
t.daemon = True
t.start()
task.save()
tasks.append(task)
self._status.add_waiting_task(task, t)
determine_time = determine_time + timedelta(minutes=1)
self._lock.release()
except Exception, e:
self._lock.release()
raise e;
def addJob(self, jobid):
'''
Add one job to Server and then server start to schedule it.
@param jobid:int
'''
jobid = int(jobid)
if jobid in self._depository:
raise ValueError("Job(id:%d) has been loaded." % (jobid))
else:
self._lock.acquire()
try:
job = Job.getByJobID(jobid)
if not job:
raise ValueError("Job(id:%d) is not exsited or not enabled." % (jobid))
self.loadJob(job)
self.indexJob(job)
self._lock.release()
except Exception, e:
self._lock.release()
raise e;
self._logger.info("Job(id:%d) is added." % (jobid))
self._server.spawnTaskThread(jobid)
