def child_unpause(child): if child.status != scheduledb.PAUSED: print "Can't unpause task %d because it isn't paused"%(child.task.id) os.kill(child.process.pid, signal.SIGCONT) child.status = scheduledb.RUNNING scheduledb.update_task(child.task,'status',child.status)
def child_pause(child): if child.status != scheduledb.RUNNING: print "Can't pause task %d because it isn't running"%(child.task.id) os.kill(child.process.pid, signal.SIGSTOP) child.status = scheduledb.PAUSED scheduledb.update_task(child.task,'status',child.status)
def check_db(): running_tasks = scheduledb.get_tasks(where='status = %d'%(scheduledb.RUNNING)) # check if tasks that were "running" are dead for task in running_tasks[:]: if not psutil.pid_exists(task.pid): print "Error: task %d was RUNNING but pid not found! Assumed dead." task.status = scheduledb.ERROR_CRASH running_tasks.remove(task) scheduledb.update_task(task,'status',task.status) tasks = scheduledb.get_tasks(where='status >= %d and status <= %d'%(scheduledb.WAITING_FOR_INPUT,scheduledb.PAUSED)) # check if a task was waiting on time or another task #print "Checking database, found %d stalled tasks"%(len(tasks)) for task in tasks: task.changed = False if (task.status == scheduledb.WAITING_FOR_INPUT or (task.status == scheduledb.WAITING_FOR_START and task.start_after <= datetime.now())): if ( len(task.prerequisites) > 0 and scheduledb.count(where='('+' or '.join(['id = %d'%i for i in task.prerequisites])+')'+' and status != %d'%(scheduledb.DONE)) > 0): task.status = scheduledb.WAITING_FOR_INPUT task.changed = True else: task.status = scheduledb.WAITING_FOR_CPU task.changed = True # check if there is CPU available for a new task curcpu = 0 curio = 0 curmem = 0 for task in running_tasks: prof = profiles[task.profile_tag] curcpu += prof.cpu_usage curio += prof.io_usage curmem += prof.mem_usage #print "Current cpu=%.2f%% io=%.2fMBps mem=%.2fMB"%(curcpu*100.0,curio,curmem) tasks_to_start = [] if ( len(running_tasks) < MAX_CHILDREN): if (curcpu < CPU_TARGET): for task in tasks: prof = profiles[task.profile_tag] if ( task.status == scheduledb.WAITING_FOR_CPU and prof.cpu_usage > 0 and prof.cpu_usage + curcpu <= CPU_MAX and prof.io_usage + curio <= IO_MAX and prof.mem_usage + curmem <= MEM_MAX ): tasks_to_start.append(task) curcpu += prof.cpu_usage curmem += prof.mem_usage curio += prof.io_usage task.status = scheduledb.RUNNING if (curmem < MEM_TARGET): for task in tasks: prof = profiles[task.profile_tag] if ( task.status == scheduledb.WAITING_FOR_CPU and prof.mem_usage > 0 and prof.cpu_usage + curcpu <= CPU_MAX and prof.io_usage + curio <= IO_MAX and prof.mem_usage + curmem <= MEM_MAX ): tasks_to_start.append(task) curcpu += prof.cpu_usage curmem += prof.mem_usage curio += prof.io_usage task.status = scheduledb.RUNNING if (curio < IO_TARGET): for task in tasks: prof = profiles[task.profile_tag] if ( task.status == scheduledb.WAITING_FOR_CPU and prof.io_usage > 0 and prof.cpu_usage + curcpu <= CPU_MAX and prof.io_usage + curio <= IO_MAX and prof.mem_usage + curmem <= MEM_MAX ): tasks_to_start.append(task) curcpu += prof.cpu_usage curmem += prof.mem_usage curio += prof.io_usage task.status = scheduledb.RUNNING # start the tasks we should start for task in tasks_to_start: child_start_task(task) # update the database for task in tasks: if task.changed: scheduledb.update_task(task,'status',task.status)