def setup(self) -> None: """Setup acceptors.""" self._start() execution_mode = ( 'threadless (local)' if self.flags.local_executor else 'threadless (remote)' ) if self.flags.threadless else 'threaded' logger.info( 'Started %d acceptors in %s mode' % ( self.flags.num_acceptors, execution_mode, ), ) # Send file descriptor to all acceptor processes. for index in range(self.flags.num_acceptors): self.fd_queues[index].send(len(self.listeners.pool)) for listener in self.listeners.pool: fd = listener.fileno() assert fd is not None send_handle( self.fd_queues[index], fd, self.acceptors[index].pid, ) self.fd_queues[index].close()
def start_work(self, conn: socket.socket, addr: Tuple[str, int]) -> None: if self.flags.threadless and \ self.threadless_client_queue and \ self.threadless_process: self.threadless_client_queue.send(addr) send_handle( self.threadless_client_queue, conn.fileno(), self.threadless_process.pid ) conn.close() else: work = self.work_klass( TcpClientConnection(conn, addr), flags=self.flags, event_queue=self.event_queue ) work_thread = threading.Thread(target=work.run) work_thread.daemon = True work.publish_event( event_name=eventNames.WORK_STARTED, event_payload={'fileno': conn.fileno(), 'addr': addr}, publisher_id=self.__class__.__name__ ) work_thread.start()
def target(): conn = listener.accept() with chdir(_fd_sharing_base_path_fd): listener.close() pid = conn.recv() send_handle(conn, fd, pid) conn.close()
def server(backlog = 5): server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) server_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) server_socket.bind((HOST, PORT)) server_socket.listen(backlog) processes = [ClientProcess() for x in xrange(10)] try: while 1: conn, addr = server_socket.accept() clients = 0 for p in processes: if p.is_alive(): clients += 1 print "new client " + str(addr) task_send = False for p in processes: if p.add_task(): reduction.send_handle(p.parent, conn.fileno(), p.pid) client += 1 task_send = True break if not task_send: p = ClientProcess() processes.append(p) clients += 1 p.add_task(clients) reduction.send_handle(p.parent, (conn.fileno(), clients), p.pid) finally: # Ctrl-C for proc in processes: proc.terminate() clients -= 1 server_socket.close()
def entry(s, address): if 0 == len(server.workers): w = Worker(server) w.http_request(http.Request(socket=s)) else: child = server.workers[random.randint(0, len(server.workers) - 1)] reduction.send_handle(child.pipe_parent, s.fileno(), child.ch.pid)
def server(): work_serv = Listener('\0singe', authkey=b'peekaboo') worker = work_serv.accept() print('Got a worker') worker_pid = worker.recv() print(worker_pid) client = os.open("/", os.O_RDONLY) send_handle(worker, client, worker_pid)
def open_uhid_process(queue_in, conn_out): while True: try: pid = queue_in.get() except KeyboardInterrupt: return 0 else: fd = os.open('/dev/uhid', os.O_RDWR) reduction.send_handle(conn_out, fd, pid)
def new_connection(self, s: JsonSocket, player_id: str) -> bool: if player_id in self._player_ids: reduction.send_handle( self._games[self._player_ids[player_id]].connection, s.fileno(), self._games[self._player_ids[player_id]].session.pid, ) self._games[self._player_ids[player_id]].connection.send(player_id) return True return False
def server(address, in_p, out_p, worker_pid): in_p.close() s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, True) s.bind(address) s.listen(1) while True: client, addr = s.accept() print("SERVER: Got connection from", addr) send_handle(out_p, client.fileno(), worker_pid) client.close() # 当发送完这个连接之后,服务端需要立刻关闭这个连接,但这个连接还是会在worker工作
def server(address, in_p, out_p, worker_pid): in_p.close() s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, True) s.bind(address) s.listen(1) while True: client, addr = s.accept() print('SERVER: Got connection from', addr) send_handle(out_p, client.fileno(), worker_pid) client.close()
def server(addr, in_p, out_p, worker_pid): in_p.close() s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, True) s.bind(addr) s.listen(1) while True: client, addr = s.accept() print("server: got connection from", addr) send_handle(out_p, client.fileno(), worker_pid) client.close()
def server(address, in_p, out_p, worker_pid): in_p.close() # This is how socket.socket() is generally used to return new socket fd s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, True) s.bind(address) s.listen(1) while True: client, address = s.accept() print(f"Server got connection from {address}") send_handle(out_p, client.fileno(), worker_pid) client.close()
def server(work_address, port): work_serv = Listener(work_address, authkey=b'peekaboo') worker = work_serv.accept() worker_pid = worker.recv() s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, True) s.bind(('', port)) s.listen(1) while True: client, addr = s.accept() print('SERVER: Got connection from ', addr) send_handle(worker, client.fileno(), worker_pid) client.close()
def setup(self): self.listen() self.start_workers() assert self.socket is not None # if self.flags.num_workers == 1: # self.socket.close() # return for idx in range(self.flags.num_workers): send_handle(self.work_queues[idx], self.socket.fileno(), self.acceptors[idx].pid) self.work_queues[idx].close() self.socket.close()
def main(): parent, child = multiprocessing.Pipe() ch = multiprocessing.Process(target=worker, args=(child,)) ch.start() s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) s.connect(('www.google.com', 80)) while not ch.pid: time.sleep(.25) reduction.send_handle(parent, s.fileno(), ch.pid) s.close() time.sleep(1) result = parent.recv() print result ch.join()
def _send(self, processed_tasks: _ProcessedTasks): """Send the processed task back to the main process""" if processed_tasks.is_failed(): # one of the tasks failed completed_tasks = CompletedTasks.failed(self.worker_id, processed_tasks) self.res_pipe.send(completed_tasks) return serialized_batch = write_batch(processed_tasks.mem_chunk, processed_tasks.data_batch) completed_tasks = CompletedTasks.done(self.worker_id, processed_tasks, serialized_batch) self.res_pipe.send(completed_tasks) # send shared memory handle for underlaying shared memory chunk # if it hasn't been sent ever before mem_chunk_id = serialized_batch.mem_chunk_id if mem_chunk_id not in self.handle_sent: self.handle_sent.add(mem_chunk_id) reduction.send_handle(self.sock, processed_tasks.mem_chunk.shm_chunk.handle, os.getppid())
def setup(self) -> None: """Listen on port, setup workers and pass server socket to workers.""" self.listen() if self.flags.enable_events: logger.info('Core Event enabled') self.start_event_dispatcher() self.start_workers() # Send server socket to all acceptor processes. assert self.socket is not None for index in range(self.flags.num_workers): send_handle(self.work_queues[index], self.socket.fileno(), self.acceptors[index].pid) self.work_queues[index].close() self.socket.close()
def accept_clients(self): try: self.serversocket.bind(self.server_addr) except socket.error as msg: eprint('Bind failed. Error Code :', str(msg[0]), 'Message', msg[1]) fatal(msg) self.serversocket.listen(1) while True: # proxyserver blocking at accept clientconn, client_addr = self.serversocket.accept() worker = self.listenpool.get_worker() #send socket to worker reduction.send_handle(worker.getpipe(),clientconn.fileno(), worker.pid) self.listenpool.free_worker(worker) clientconn.close()
def server(work_address, port): work_serv = Listener(work_address, authkey=b'peekaboo') worker = work_serv.accept() worker_pid = worker.recv() s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, True) s.bind(('', port)) s.listen(1) while True: client, addr = s.accept() print('SERVER: Got connection from', addr) send_handle(worker, client.fileno(), worker_pid) client.close()
def server(work_address): # Wait for the worker to connect work_serv = Listener(work_address, authkey=b'peekaboo') while True: worker = work_serv.accept() worker_pid = worker.recv() print('SERVER: Got connection') msg = "Secure zero-copy こんにちわ from server: %s" % (time.ctime()) msg_bytes = msg.encode("utf8") fd = memfd.new_memfd_region(msg_bytes) send_handle(worker, fd, worker_pid) worker.close()
def server(address, in_p, out_p, worker_pid): in_p.close() s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, True) s.bind(address) s.listen(1) client, addr = s.accept() while True: msg = client.recv(1024) if msg.decode().isnumeric(): client.send(str(int(msg) + 1).encode()) elif msg.decode() == 'update': client.send(b'success') send_handle(out_p, client.fileno(), worker_pid) client.close() break
def server(work_address, port): # Oczekiwanie na połączenie z wątkiem roboczym work_serv = Listener(work_address, authkey=b'peekaboo') worker = work_serv.accept() worker_pid = worker.recv() # Uruchamianie serwera TCP/IP i wysyłanie klientó do wątku roboczego s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, True) s.bind(('', port)) s.listen(1) while True: client, addr = s.accept() print('SERWER: Żądanie połączenia z', addr) send_handle(worker, client.fileno(), worker_pid) client.close()
def _grant_handle(conn: Connection, handle: IO, cpid: int) -> None: log = logging.getLogger('_grant_handle') flags = get_flags(handle) modes = handle.mode fd = handle.fileno() wrapper = (flags, modes) conn.send(wrapper) if msvcrt: fd = msvcrt.get_osfhandle(fd) log.info( 'Granting Windows-style file handle for {} (fd={}; flags={:02X}; mode={}) to pid {}.' .format(handle.name, fd, wrapper[0], handle.mode, cpid)) else: log.info( 'Granting C-style file descriptor for {} (fd={}; flags={:02X}; mode={}) to pid {}.' .format(handle.name, fd, wrapper[0], handle.mode, cpid)) send_handle(conn, fd, cpid)
def server(work_address, port): # Wait for the worker to connect work_serv = Listener(work_address, authkey=b'peekaboo') worker = work_serv.accept() worker_pid = worker.recv() # Now run a TCP/IP server and send clients to worker s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, True) s.bind(('', port)) s.listen(1) while True: client, addr = s.accept() print(f'SERVER: Got connection from {addr}') send_handle(worker, client.fileno(), worker_pid) client.close()
def server(address, in_pipe, out_pipe, worker_pid): try: in_pipe.close() s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, True) s.bind(address) s.listen(5) while True: client, addr = s.accept() print('server 收到连接{}'.format(addr)) send_handle(out_pipe, client.fileno(), worker_pid) #client.shutdown(socket.SHUT_RDWR) client.close() except KeyboardInterrupt: return 0 finally: s.close()
def _send_fileno(connection, fileno, pid): """Cross-platform version of send_handle.""" if sys.platform == 'win32': handle = msvcrt.get_osfhandle(fileno) else: handle = fileno return send_handle(connection, handle, pid)
def server(address, port): server = Listener(address, authkey=b"peekaboo") conn = server.accept() worker_pid = conn.recv() print(f"Server : Worked PID {worker_pid}") # This is how socket.socket() is generally used to return new socket fd s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, True) s.bind(('', port)) s.listen(1) while True: client, address = s.accept() print(f"Server got connection from {address}") send_handle(conn, client.fileno(), worker_pid) client.close()
def server(work_address, port): # Wait for the worker to connect work_serv = Listener(work_address, authkey=b'peekaboo') worker = work_serv.accept() worker_pid = worker.recv() # Now run a TCP/IP server and send clients to worker s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, True) s.bind(('', port)) s.listen(1) while True: client, addr = s.accept() print('client:',client, 'fileno', client.fileno, 'addr', addr) print('SERVER: Got connection from', addr) send_handle(worker, client.fileno(), worker_pid) client.close()
def delegate_work_to_pool( worker_pid: int, work_queue: 'connection.Connection', work_lock: 'multiprocessing.synchronize.Lock', conn: 'socket.socket', addr: Optional[Tuple[str, int]], unix_socket_path: Optional[str] = None, ) -> None: """Utility method to delegate a work to threadless executor pool.""" with work_lock: # Accepted client address is empty string for # unix socket domain, avoid sending empty string # for optimization. if not unix_socket_path: work_queue.send(addr) send_handle( work_queue, conn.fileno(), worker_pid, ) conn.close()
def _startttyserver(): """start a tty fd server the server will send tty read and write fds via unix socket listens at sockpath: $TMPDIR/ttysrv$UID/$PID returns (pid, sockpath) """ sockpath = os.path.join(tempfile.mkdtemp("ttysrv"), str(os.getpid())) pipes = os.pipe() pid = os.fork() if pid: # parent, wait for the child to start listening os.close(pipes[1]) os.read(pipes[0], 1) os.close(pipes[0]) return pid, sockpath # terminate=true will force the child process to exit, regardless of if # there's an exception or not. This is required to prevent it from returning # to the parent process command execution and resulting in a double # invocation of the command. with _silentexception(terminate=True): # child, starts the server ttyrfd, ttywfd = _ttyfds or [sys.stdin.fileno(), sys.stderr.fileno()] sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) getattr(util, "bindunixsocket", _sockbind)(sock, sockpath) sock.listen(1) # unblock parent os.close(pipes[0]) os.write(pipes[1], b" ") os.close(pipes[1]) while True: conn, addr = sock.accept() # 0: a dummy destination_pid, is ignored on posix systems send_handle(conn, ttyrfd, 0) send_handle(conn, ttywfd, 0) conn.close()
def _startttyserver(): """start a tty fd server the server will send tty read and write fds via unix socket listens at sockpath: $TMPDIR/ttysrv$UID/$PID returns (pid, sockpath) """ sockpath = os.path.join(tempfile.mkdtemp("ttysrv"), str(os.getpid())) pipes = os.pipe() pid = os.fork() if pid: # parent, wait for the child to start listening os.close(pipes[1]) os.read(pipes[0], 1) os.close(pipes[0]) return pid, sockpath # child, starts the server ttyrfd, ttywfd = _ttyfds or [sys.stdin.fileno(), sys.stderr.fileno()] sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) getattr(util, "bindunixsocket", _sockbind)(sock, sockpath) sock.listen(1) # unblock parent os.close(pipes[0]) os.write(pipes[1], " ") os.close(pipes[1]) with _silentexception(terminate=True): while True: conn, addr = sock.accept() # 0: a dummy destination_pid, is ignored on posix systems send_handle(conn, ttyrfd, 0) send_handle(conn, ttywfd, 0) conn.close()
def _startttyserver(): """start a tty fd server the server will send tty read and write fds via unix socket listens at sockpath: $TMPDIR/ttysrv$UID/$PID returns (pid, sockpath) """ sockpath = os.path.join(tempfile.mkdtemp('ttysrv'), str(os.getpid())) pipes = os.pipe() pid = os.fork() if pid: # parent, wait for the child to start listening os.close(pipes[1]) os.read(pipes[0], 1) os.close(pipes[0]) return pid, sockpath # child, starts the server ttyrfd, ttywfd = _ttyfds or [sys.stdin.fileno(), sys.stderr.fileno()] sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) getattr(util, 'bindunixsocket', _sockbind)(sock, sockpath) sock.listen(1) # unblock parent os.close(pipes[0]) os.write(pipes[1], ' ') os.close(pipes[1]) with _silentexception(terminate=True): while True: conn, addr = sock.accept() # 0: a dummy destination_pid, is ignored on posix systems send_handle(conn, ttyrfd, 0) send_handle(conn, ttywfd, 0) conn.close()
parent, child = multiprocessing.Pipe() output_queue = multiprocessing.Queue() output_buffer = multiprocessing.Process( target=output_buffer_process, args=( child, output_queue, )) output_buffer.start() while not output_buffer.pid: time.sleep(.25) # send the socket reduction.send_handle(parent, s.fileno(), output_buffer.pid) # start the hearts giving process hearts_process = multiprocessing.Process( target=give_hearts_process, args=(HEARTS_WHEN, users)) hearts_process.start() startupComplete = True else: if args is not None: if args == CHAN: if username not in users: users.append(username) elif command == "PART": if args is not None:
def _setup_h2h_usbcable(self): if self.usbcable: logger.debug("Close exist usb instance") self.usbcable.close() self.usbcable = None try: usbcable = USBCable() logger.debug("USB initialized") except SystemError as e: logger.error("USB initialize error: %s", e) self.dirty_status = True return payload = msgpack.packb((0x81, )) try: while True: s = socket.socket(socket.AF_UNIX) s.connect(ROBOT_ENDPOINT) s.send(payload) rl = select((s, ), (), (), 0.2)[0] if rl: ret = s.recv(1) else: logger.error("Error: robot endpoint resp usb init timeout") s.close() sleep(0.05) continue if ret != b"F": logger.error("Error: remote init return %s", repr(ret)) s.close() sleep(0.05) continue wl = select((), (s, ), (), 0.2)[1] if wl: send_handle(s, usbcable.outside_sockfd, 0) else: logger.error("Error: Can not write") s.close() sleep(0.05) continue rl = select((s, ), (), (), 0.2)[0] if rl: ret = s.recv(1) else: logger.error("Error: robot endpoint resp usb fin timeout") s.close() sleep(0.05) continue if ret != b"X": logger.error("Error remote fin return %r, not X", ret) s.close() continue usbcable.start() self.usbcable = usbcable s.close() self.dirty_status = False logger.info("USB session pass to robot successed.") return except socket.error: self.dirty_status = True logger.error("Pass usb session to robot failed. (socket error)") except Exception: self.dirty_status = True logger.exception("Pass usb session to robot failed.")
def compress_to_archive_with_progress(file_list, dest_archive_path, compression_level=9, encryptFilenames=False, solidMode=True, password=None): sock = get_connected_local_socket() rq = bytearray([ord(b'\x06') ^ (7 << 5)]) # compress from fds sock.sendall(rq) send_fstats(file_list, sock) dest_archive_path = encodeString(dest_archive_path) sock.sendall(struct.pack( "@H", len(dest_archive_path))) # len of path as unsigned short sock.sendall(dest_archive_path) # send compress options sock.sendall(struct.pack("@B", compression_level)) sock.sendall(struct.pack("@B", 1 if encryptFilenames else 0)) sock.sendall(struct.pack("@B", 1 if solidMode else 0)) if password is None: sock.sendall(struct.pack("@B", 0)) # 1-byte password length else: password = encodeString(password) sock.sendall(struct.pack("@B", len(password))) # 1-byte password length sock.sendall(password) resp = sock.recv(1) # response first byte: \x00 OK or \xFF ERROR if resp != b'\x00': print("Error byte received, errno:", struct.unpack("@I", sock.recv(4))[0]) return last_progress = 0 maxuint = 2**64 - 1 maxuint_2 = 2**64 - 2 # OK means archive init has been successful, and actual compression starts now, so start receiving progress print("OK response received, starting receiving progress updates...") # receive total total = struct.unpack("@Q", sock.recv(8))[0] # receive as uint64_t print("Received total size:", total) while True: progress = struct.unpack("@Q", sock.recv(8))[0] print("Received progress:", progress) if progress == maxuint_2: # end of compression if last_progress == total: # OK print("Compress OK") else: print( "Warning, last progress before termination value differs from total" ) break elif progress == maxuint: print('Finished compressing file or first iteration') # receive index idx = struct.unpack("@I", sock.recv(4))[0] with open(file_list[idx], 'rb') as f: send_handle(sock, f.fileno(), None) else: last_progress = progress # after receiving termination progress value (-1 as uint64), receive standard ok or error response resp = sock.recv(1) # response first byte: \x00 OK or \xFF ERROR if resp != b'\x00': print("Error byte received, errno:", struct.unpack("@I", sock.recv(4))[0]) else: print("OK response received") sock.close()
def serialize(self, pipe, pid): from multiprocessing.reduction import send_handle send_handle(pipe, self.conn.fileno(), pid)