def _handle_req(self, path, args): if path[0] == 'status': data = job_tracking.get(args.get('subjob_cookie') or None) if not data: self.do_response(500, 'text/plain', 'bad subjob_cookie!\n') return timeout = min(float(args.get('timeout', 0)), 128) status = DotDict(idle=data.lock.acquire(False)) deadline = time.time() + timeout while not status.idle and time.time() < deadline: time.sleep(0.1) status.idle = data.lock.acquire(False) if status.idle: if data.last_error: status.last_error = data.last_error data.last_error = None else: status.last_time = data.last_time data.lock.release() elif path == ['status', 'full']: status.status_stacks, status.current = status_stacks_export() self.do_response(200, "text/json", status) return elif path == ['list_workspaces']: ws = {k: v.path for k, v in self.ctrl.list_workspaces().items()} self.do_response(200, "text/json", ws) elif path == ['config']: self.do_response(200, "text/json", self.ctrl.config) elif path == ['update_methods']: self.do_response(200, "text/json", self.ctrl.update_methods()) elif path == ['methods']: """ return a json with everything the Method object knows about the methods """ self.do_response(200, "text/json", self.ctrl.get_methods()) elif path[0] == 'method_info': method = path[1] self.do_response(200, "text/json", self.ctrl.method_info(method)) elif path[0] == 'workspace_info': self.do_response(200, 'text/json', self.ctrl.get_workspace_details()) elif path[0] == 'abort': tokill = list(children) print('Force abort', tokill) for child in tokill: os.killpg(child, signal.SIGKILL) self.do_response(200, 'text/json', {'killed': len(tokill)}) elif path == ['submit']: if self.ctrl.broken: self.do_response( 500, "text/json", { 'broken': self.ctrl.broken, 'error': 'Broken methods: ' + ', '.join( sorted( m.split('.')[-1][2:] for m in self.ctrl.broken)) }) elif 'xml' in args: self.do_response(500, 'text/plain', 'JSON > XML!\n') elif 'json' in args: if DEBUG_WRITE_JSON: with open('DEBUG_WRITE.json', 'wb') as fh: fh.write(args['json']) setup = json_decode(args['json']) data = job_tracking.get(setup.get('subjob_cookie') or None) if not data: self.do_response(500, 'text/plain', 'bad subjob_cookie!\n') return if len(job_tracking) - 1 > 5: # max five levels print('Too deep subjob nesting!') self.do_response(500, 'text/plain', 'Too deep subjob nesting') return if data.lock.acquire(False): respond_after = True try: if self.DEBUG: print('@daemon.py: Got the lock!', file=sys.stderr) jobidv, job_res = self.ctrl.initialise_jobs(setup) job_res['done'] = False if jobidv: error = [] tlock = TLock() link2job = { j['link']: j for j in job_res['jobs'].values() } def run(jobidv, tlock): for jobid in jobidv: passed_cookie = None # This is not a race - all higher locks are locked too. while passed_cookie in job_tracking: passed_cookie = gen_cookie() job_tracking[passed_cookie] = DotDict( lock=JLock(), last_error=None, last_time=0) try: self.ctrl.run_job( jobid, subjob_cookie=passed_cookie, parent_pid=setup.get( 'parent_pid', 0)) # update database since a new jobid was just created job = self.ctrl.add_single_jobid(jobid) with tlock: link2job[jobid]['make'] = 'DONE' link2job[jobid][ 'total_time'] = job.total except JobError as e: error.append( [e.jobid, e.method, e.status]) with tlock: link2job[jobid]['make'] = 'FAIL' return finally: del job_tracking[passed_cookie] # everything was built ok, update symlink try: wn = self.ctrl.target_workdir dn = self.ctrl.workspaces[wn].path ln = os.path.join(dn, wn + "-LATEST_") try: os.unlink(ln) except OSError: pass os.symlink(jobid, ln) os.rename(ln, os.path.join(dn, wn + "-LATEST")) except OSError: traceback.print_exc() t = Thread(target=run, name="job runner", args=( jobidv, tlock, )) t.daemon = True t.start() t.join(2) # give job two seconds to complete with tlock: for j in link2job.values(): if j['make'] in ( True, 'FAIL', ): respond_after = False job_res_json = json_encode(job_res) break if not respond_after: # not all jobs are done yet, give partial response self.do_response(200, "text/json", job_res_json) t.join() # wait until actually complete del tlock del t # verify that all jobs got built. total_time = 0 for j in link2job.values(): jobid = j['link'] if j['make'] == True: # Well, crap. error.append([ jobid, "unknown", { "INTERNAL": "Not built" } ]) print("INTERNAL ERROR IN JOB BUILDING!", file=sys.stderr) total_time += j.get('total_time', 0) data.last_error = error data.last_time = total_time except Exception as e: if respond_after: self.do_response(500, "text/json", {'error': str(e)}) raise finally: data.lock.release() if respond_after: job_res['done'] = True self.do_response(200, "text/json", job_res) if self.DEBUG: print("@daemon.py: Process releases lock!", file=sys.stderr ) # note: has already done http response else: self.do_response(200, 'text/plain', 'Busy doing work for you...\n') else: self.do_response(500, 'text/plain', 'Missing json input!\n') else: self.do_response(500, 'text/plain', 'Unknown path\n') return
def _url_json(self, *path, **kw): return json_decode(self._url_get(*path, **kw))