def idle_clients(self):
log.msg("Looking for idle clients in pool")
return (
self.clients[id]
for id in self.clients
if self.clients[id].is_idle()
)
def all_tasks_complete(self):
complete = [] == [ 1 for task in self.tasks if task.status != 'complete' ]
log.msg("%s" % (complete,))
if complete:
self.status = 'complete'
log.msg("All tasks complete for job %d" % (self.id))
return complete
def set_task_status(self, frame, status):
log.msg("Job %d: Setting task status for frame %d to '%s'"
% (self.id, frame, status))
for task in self.tasks:
if task.startframe != frame: continue
task.status = status
def check_timed_out_tasks(self):
log.msg("checking for timed out tasks")
for job in self.active:
for task in job.tasks:
if not (task.status == 'rendering'
and task.start_time < time - job.timeout): break
job.status = 'pending'
log.warn("job %d frame %d has timed out and been re-queued")
def do_set_task_status(self, client, args):
self.check_arg_count(args, 3)
jobid, taskid, status = args
jobid = int(jobid)
taskid = int(taskid)
job = self.jobs.get_job(jobid)
job.set_task_status(taskid, status)
log.msg("%s" % ([jobid, taskid, status],))
client.status = 'idle'
def active_job(self):
log.msg("getting active job")
if not self.active_job_id or \
self.jobs[self.active_job_id].status == 'complete' or \
self.jobs[self.active_job_id].all_tasks_complete():
try:
self.active_job_id = self.pending().next().id
except StopIteration:
return None # no jobs
return self.jobs[self.active_job_id]
def assign_next_task(self, client):
log.msg("get next step for job %d" % (self.id))
start = end = None
incomplete = (
task
for task in self.tasks
if task.status in ['pending', 'error']
)
try:
next_task = incomplete.next()
next_task.status = 'rendering'
next_task.client = client
next_task.start_time = time.time()
return next_task
except StopIteration:
return None
def handle_line(self, client_id, line):
client = self.get_client(client_id)
line = line.strip()
tokens = line.split(' ')
log.msg(tokens)
try:
if len(tokens) == 0:
raise RuntimeError("Empty line")
cmd = tokens[0]
cmd = re.sub(r'\W', '_', cmd)
try:
func = getattr(self, 'do_%s' % (cmd,))
func(client, tokens[1:])
except AttributeError:
raise
except Exception, e:
client.send_line("Error: %s" % (e,))
def reset_tasks(self, job_id=None, slave=None):
all_complete = True
for job in self.all():
if (job_id not in [job.id, None]
and job.status != 'complete'
and job.status != 'paused'): continue
for task in job.tasks:
if task.assigned_to != slave:
continue
if task.status not in ['completed', 'pending']:
task.status = 'pending'
task.assigned_to = None
if task.status != 'complete':
all_complete = False
if not all_complete:
log.msg("setting job %d status to pending" % (job.id,))
job.status = 'pending'
if job_id: break
def dispatch_idle_clients(self):
log.msg("Check for clients that need work")
idle = self.idle_clients()
active_job = self.jobs.active_job()
if not active_job: return
log.msg("Active job %s" % (active_job.to_hash()))
log.msg("Active tasks %s" % ( [ t.to_hash() for t in active_job.tasks ], ))
for client in idle:
task = active_job.assign_next_task(client)
client.render_task(active_job, task)
def send_line(self, s):
log.msg(">>> %d | %s" % (self.id, s))
self._protocol.sendLine(s)
def add_client(self, client):
self.clients[client.id] = client
client.send_line("# Welcome client %d" % (client.id))
log.msg("Adding client %d" % (client.id))
log.msg("%d clients currently in pool" % len(self.clients))
