Example #1
0
def remote_compile_client(conn, source_file_name, compile_args, source_data):
    conn.settimeout(None)
    schedaemon_util.send_buffer(conn, source_file_name)
    compile_args_str = '\0'.join(compile_args)
    schedaemon_util.send_buffer(conn, compile_args_str)
    schedaemon_util.send_buffer(conn, source_data)

    try:
        returncode = schedaemon_util.recv_struct(conn, '<i')
        outdata = schedaemon_util.recv_buffer(conn)
        errdata = schedaemon_util.recv_buffer(conn)

        sys.stderr.write(errdata)
        sys.stdout.write(outdata)

        num_files = schedaemon_util.recv_struct(conn, '<Q')
        output_files = []
        for i in range(num_files):
            file_name = str(schedaemon_util.recv_buffer(conn))
            file_data = schedaemon_util.recv_buffer(conn)
            output_files.append((file_name, file_data))
            continue

        ccerbd.write_files('', output_files)
        return returncode
    except schedaemon_util.ExSocketClosed:
        print('ERROR: `ccerb` socket closed early.', file=sys.stderr)
        return 1
Example #2
0
def accept_remote(conn, addr):
    if VERBOSE >= 2:
        print('accept_remote({})'.format(addr))

    conn.settimeout(REMOTE_NET_TIMEOUT)
    reconn_token = str(schedaemon_util.recv_buffer(conn))
    if not reconn_token:
        return accept_remote_notify(conn, addr)

    ####

    try:
        cur = pop_pending_reconnect(reconn_token)
    except KeyError:
        return

    cc_bin = cur.cc_bin
    job = cur.job

    ####

    remote_compile_server(conn, cc_bin)
    job.cancel()

    return
Example #3
0
def accept_remote_notify(conn, addr):
    if VERBOSE >= 2:
        print('accept_remote_notify({})'.format(addr))

    cc_key = str(schedaemon_util.recv_buffer(conn))
    try:
        cc_bin = cc_key_map[cc_key]
    except KeyError:
        print('[{}] Unrecognized cc_key: {}'.format(addr, cc_key))
        return

    info = 'ccerbd-compile@{}: {}'.format(addr, cc_key)
    print(info)

    while True:
        try:
            job = schedaemon.ScheduledJob(REMOTE_COMPILE_PRIORITY, info)
        except schedaemon.ExTimeout:
            return

        if not job.acquire():
            return

        reconn_token = os.urandom(RECONN_TOKEN_BYTES)
        with pending_reconnects_lock:
            assert reconn_token not in pending_reconnects
            pending_reconnects[reconn_token] = PendingReconnect(job, cc_bin, addr)

        schedaemon_util.send_buffer(conn, reconn_token)

        schedaemon_util.spawn_thread(thread__pending_compile_timeout, (reconn_token,))

        conn.settimeout(None)
        if not schedaemon_util.recv_poke(conn):
            return
Example #4
0
def remote_compile_server(conn, cc_bin):
    conn.settimeout(None)

    try:
        source_file_name = str(schedaemon_util.recv_buffer(conn))
        compile_args_str = str(schedaemon_util.recv_buffer(conn))
        compile_args = compile_args_str.split('\0')
        source_data = schedaemon_util.recv_buffer(conn)
    except schedaemon_util.ExSocketClosed:
        return

    input_files = [(source_file_name, source_data)]
    args = [cc_bin] + compile_args + [source_file_name]
    (returncode, outdata, errdata, output_files) = run_in_temp_dir(input_files, args)

    schedaemon_util.send_struct(conn, '<i', returncode)
    schedaemon_util.send_buffer(conn, outdata)
    schedaemon_util.send_buffer(conn, errdata)
    schedaemon_util.send_struct(conn, '<Q', len(output_files))
    for (name, data) in output_files:
        schedaemon_util.send_buffer(conn, name)
        schedaemon_util.send_buffer(conn, data)

    return
Example #5
0
def sched_accept(conn, addr):
    if VERBOSE >= 2:
        print('new conn: ', addr)

    conn.settimeout(NET_TIMEOUT)
    try:
        priority = schedaemon_util.recv_struct(conn, PRIORITY_STRUCT)
        info = str(schedaemon_util.recv_buffer(conn))

    except (socket.timeout, socket.error, schedaemon_util.ExSocketClosed):
        return

    ####

    job = Job(priority, info)
    if VERBOSE >= 1:
        print('<new {}>'.format(job))
    g_pq.put_right(job)

    ####

    conn.setblocking(True)

    try:
        job.begin_event.wait()
        if VERBOSE >= 1:
            start = time.time()
            print('<+{}>'.format(job))

        conn.sendall( bytearray([1]) )
        conn.recv(1) # Blocks until recv or socket dies.
    except socket.error:
        pass

    if VERBOSE >= 1:
        time_diff = time.time() - start
        print('<-{}@{}ms>'.format(job, time_diff))

    job.end_event.set()
    g_pq.erase(job)
    return
Example #6
0
def accept_local(conn, addr):
    if VERBOSE >= 2:
        print('accept_local({})'.format(addr))

    cc_key = str(schedaemon_util.recv_buffer(conn))
    try:
        cc_bin = cc_key_map[cc_key]
    except KeyError:
        print('[{}] Unrecognized cc_key: {}'.format(addr, cc_key))
        return

    ####

    (remote_gai, reconn_token) = acquire_remote(cc_key)
    if not remote_gai:
        print('[{}] Failed to find remotes for: {}'.format(addr, cc_key))
        return

    remote_gai_str = marshal.dumps(remote_gai)
    schedaemon_util.send_buffer(conn, remote_gai_str)
    schedaemon_util.send_buffer(conn, reconn_token)
    return
Example #7
0
def acquire_remote(cc_key):
    addr_list = [REMOTE_ADDR]

    gai_list = []
    for (host, port) in addr_list:
        gai_list += socket.getaddrinfo(host, port, 0, socket.SOCK_STREAM)

    for gai in gai_list:
        print('gai:', gai)
        (family, socktype, proto, _, sockaddr) = gai
        remote_conn = None
        try:
            remote_conn = socket.socket(family, socktype, proto)
            remote_conn.settimeout(REMOTE_NET_TIMEOUT)
            remote_conn.connect(sockaddr)
        except (socket.error, socket.timeout) as e:
            if remote_conn:
                schedaemon_util.kill_socket(remote_conn)
            print('gai failed:', e)
            continue
        break
    else:
        print('out of gai')
        return (None, None)

    try:
        schedaemon_util.send_buffer(remote_conn, '') # no token
        schedaemon_util.send_buffer(remote_conn, cc_key)

        remote_conn.settimeout(None)
        reconn_token = str(schedaemon_util.recv_buffer(remote_conn))
        return (gai, reconn_token)
    except schedaemon_util.ExSocketClosed:
        print('no token')
        return (None, None)
    finally:
        schedaemon_util.kill_socket(remote_conn)
Example #8
0
def main(args):
    assert len(args) >= 1

    cc_bin = args[0]
    cc_args = args[1:]

    (preproc_args, compile_args, source_file_name) = process_args(cc_args)
    #print('\npreproc_args:', preproc_args)
    #print('\ncompile_args:', compile_args)

    ####

    try:
        info = 'ccerb-preproc: {}'.format(source_file_name)
        with schedaemon.ScheduledJob(PREPROC_PRIORITY, info):
            cc_key = ccerbd.get_cc_key(cc_bin)
            preproc_args = [cc_bin] + preproc_args + [source_file_name]
            p = subprocess.Popen(preproc_args, stdout=subprocess.PIPE, stderr=subprocess.PIPE)

            (outdata, errdata) = p.communicate()
            if p.returncode != 0:
                sys.stderr.write(errdata)
                sys.stdout.write(outdata)
                exit(p.returncode)

            preproc_data = outdata
    except schedaemon.ExTimeout:
        print('ERROR: Missing `schedaemon`.', file=sys.stderr)
        return 1 # Will oversubscribe heavily without schedaemon.

    #print(preproc_data)
    #exit(0)

    ####

    while True:
        try:
            local_conn = socket.create_connection(ccerbd.LOCAL_ADDR, LOCAL_TIMEOUT)
        except socket.timeout:
            print('ERROR: Missing `ccerbd`.', file=sys.stderr)
            return 1 # Exit, since even shimming out is strictly slower than normal.

        try:
            schedaemon_util.send_buffer(local_conn, cc_key)

            local_conn.settimeout(None)
            remote_gai_str = str(schedaemon_util.recv_buffer(local_conn))
            reconn_token = schedaemon_util.recv_buffer(local_conn)
        except  (socket.error, schedaemon_util.ExSocketClosed) as e:
            if VERBOSE >= 1:
                print('Warning: Failed to request remote compile addr. ({})'.format(e),
                      file=sys.stderr)
            raise ExShimOut('no remote compile addr')
        finally:
            schedaemon_util.kill_socket(local_conn)

        ####

        remote_gai = marshal.loads(remote_gai_str)
        (family, socktype, proto, _, sockaddr) = remote_gai

        ####

        try:
            remote_conn = socket.socket(family, socktype, proto)
            remote_conn.connect(sockaddr)
        except (socket.error, socket.timeout) as e:
            print('Failed to connect to remote addr ({}): {}'.format(remote_gai, e))
            continue

        try:
            schedaemon_util.send_buffer(remote_conn, reconn_token)
            returncode = remote_compile_client(remote_conn, source_file_name,
                                               compile_args, preproc_data)
        except (socket.error, socket.timeout, schedaemon_util.ExSocketClosed) as e:
            print('Remote compile failed ({}): {}'.format(remote_gai, e))
            continue
        finally:
            schedaemon_util.kill_socket(remote_conn)

        break

    ####

    return returncode