def accept_remote_notify(conn, addr): if VERBOSE >= 2: print('accept_remote_notify({})'.format(addr)) cc_key = str(schedaemon_util.recv_buffer(conn)) try: cc_bin = cc_key_map[cc_key] except KeyError: print('[{}] Unrecognized cc_key: {}'.format(addr, cc_key)) return info = 'ccerbd-compile@{}: {}'.format(addr, cc_key) print(info) while True: try: job = schedaemon.ScheduledJob(REMOTE_COMPILE_PRIORITY, info) except schedaemon.ExTimeout: return if not job.acquire(): return reconn_token = os.urandom(RECONN_TOKEN_BYTES) with pending_reconnects_lock: assert reconn_token not in pending_reconnects pending_reconnects[reconn_token] = PendingReconnect(job, cc_bin, addr) schedaemon_util.send_buffer(conn, reconn_token) schedaemon_util.spawn_thread(thread__pending_compile_timeout, (reconn_token,)) conn.settimeout(None) if not schedaemon_util.recv_poke(conn): return
def __init__(self, priority=0, info=''): assert len(info) < 1024 self.priority = priority self.info = info self.conn = None try: conn = socket.create_connection(ADDR, NET_TIMEOUT) schedaemon_util.send_struct(conn, PRIORITY_STRUCT, self.priority) schedaemon_util.send_buffer(conn, self.info) except socket.error: raise ExTimeout() self.conn = conn return
def remote_compile_client(conn, source_file_name, compile_args, source_data): conn.settimeout(None) schedaemon_util.send_buffer(conn, source_file_name) compile_args_str = '\0'.join(compile_args) schedaemon_util.send_buffer(conn, compile_args_str) schedaemon_util.send_buffer(conn, source_data) try: returncode = schedaemon_util.recv_struct(conn, '<i') outdata = schedaemon_util.recv_buffer(conn) errdata = schedaemon_util.recv_buffer(conn) sys.stderr.write(errdata) sys.stdout.write(outdata) num_files = schedaemon_util.recv_struct(conn, '<Q') output_files = [] for i in range(num_files): file_name = str(schedaemon_util.recv_buffer(conn)) file_data = schedaemon_util.recv_buffer(conn) output_files.append((file_name, file_data)) continue ccerbd.write_files('', output_files) return returncode except schedaemon_util.ExSocketClosed: print('ERROR: `ccerb` socket closed early.', file=sys.stderr) return 1
def accept_local(conn, addr): if VERBOSE >= 2: print('accept_local({})'.format(addr)) cc_key = str(schedaemon_util.recv_buffer(conn)) try: cc_bin = cc_key_map[cc_key] except KeyError: print('[{}] Unrecognized cc_key: {}'.format(addr, cc_key)) return #### (remote_gai, reconn_token) = acquire_remote(cc_key) if not remote_gai: print('[{}] Failed to find remotes for: {}'.format(addr, cc_key)) return remote_gai_str = marshal.dumps(remote_gai) schedaemon_util.send_buffer(conn, remote_gai_str) schedaemon_util.send_buffer(conn, reconn_token) return
def acquire_remote(cc_key): addr_list = [REMOTE_ADDR] gai_list = [] for (host, port) in addr_list: gai_list += socket.getaddrinfo(host, port, 0, socket.SOCK_STREAM) for gai in gai_list: print('gai:', gai) (family, socktype, proto, _, sockaddr) = gai remote_conn = None try: remote_conn = socket.socket(family, socktype, proto) remote_conn.settimeout(REMOTE_NET_TIMEOUT) remote_conn.connect(sockaddr) except (socket.error, socket.timeout) as e: if remote_conn: schedaemon_util.kill_socket(remote_conn) print('gai failed:', e) continue break else: print('out of gai') return (None, None) try: schedaemon_util.send_buffer(remote_conn, '') # no token schedaemon_util.send_buffer(remote_conn, cc_key) remote_conn.settimeout(None) reconn_token = str(schedaemon_util.recv_buffer(remote_conn)) return (gai, reconn_token) except schedaemon_util.ExSocketClosed: print('no token') return (None, None) finally: schedaemon_util.kill_socket(remote_conn)
def remote_compile_server(conn, cc_bin): conn.settimeout(None) try: source_file_name = str(schedaemon_util.recv_buffer(conn)) compile_args_str = str(schedaemon_util.recv_buffer(conn)) compile_args = compile_args_str.split('\0') source_data = schedaemon_util.recv_buffer(conn) except schedaemon_util.ExSocketClosed: return input_files = [(source_file_name, source_data)] args = [cc_bin] + compile_args + [source_file_name] (returncode, outdata, errdata, output_files) = run_in_temp_dir(input_files, args) schedaemon_util.send_struct(conn, '<i', returncode) schedaemon_util.send_buffer(conn, outdata) schedaemon_util.send_buffer(conn, errdata) schedaemon_util.send_struct(conn, '<Q', len(output_files)) for (name, data) in output_files: schedaemon_util.send_buffer(conn, name) schedaemon_util.send_buffer(conn, data) return
def main(args): assert len(args) >= 1 cc_bin = args[0] cc_args = args[1:] (preproc_args, compile_args, source_file_name) = process_args(cc_args) #print('\npreproc_args:', preproc_args) #print('\ncompile_args:', compile_args) #### try: info = 'ccerb-preproc: {}'.format(source_file_name) with schedaemon.ScheduledJob(PREPROC_PRIORITY, info): cc_key = ccerbd.get_cc_key(cc_bin) preproc_args = [cc_bin] + preproc_args + [source_file_name] p = subprocess.Popen(preproc_args, stdout=subprocess.PIPE, stderr=subprocess.PIPE) (outdata, errdata) = p.communicate() if p.returncode != 0: sys.stderr.write(errdata) sys.stdout.write(outdata) exit(p.returncode) preproc_data = outdata except schedaemon.ExTimeout: print('ERROR: Missing `schedaemon`.', file=sys.stderr) return 1 # Will oversubscribe heavily without schedaemon. #print(preproc_data) #exit(0) #### while True: try: local_conn = socket.create_connection(ccerbd.LOCAL_ADDR, LOCAL_TIMEOUT) except socket.timeout: print('ERROR: Missing `ccerbd`.', file=sys.stderr) return 1 # Exit, since even shimming out is strictly slower than normal. try: schedaemon_util.send_buffer(local_conn, cc_key) local_conn.settimeout(None) remote_gai_str = str(schedaemon_util.recv_buffer(local_conn)) reconn_token = schedaemon_util.recv_buffer(local_conn) except (socket.error, schedaemon_util.ExSocketClosed) as e: if VERBOSE >= 1: print('Warning: Failed to request remote compile addr. ({})'.format(e), file=sys.stderr) raise ExShimOut('no remote compile addr') finally: schedaemon_util.kill_socket(local_conn) #### remote_gai = marshal.loads(remote_gai_str) (family, socktype, proto, _, sockaddr) = remote_gai #### try: remote_conn = socket.socket(family, socktype, proto) remote_conn.connect(sockaddr) except (socket.error, socket.timeout) as e: print('Failed to connect to remote addr ({}): {}'.format(remote_gai, e)) continue try: schedaemon_util.send_buffer(remote_conn, reconn_token) returncode = remote_compile_client(remote_conn, source_file_name, compile_args, preproc_data) except (socket.error, socket.timeout, schedaemon_util.ExSocketClosed) as e: print('Remote compile failed ({}): {}'.format(remote_gai, e)) continue finally: schedaemon_util.kill_socket(remote_conn) break #### return returncode