def _recv_queue_handles(self, setup_socket): self.result_queue.open_shm(reduction.recv_handle(setup_socket)) if self.general_task_queue is not None: self.general_task_queue.open_shm( reduction.recv_handle(setup_socket)) if self.dedicated_task_queue is not None: self.dedicated_task_queue.open_shm( reduction.recv_handle(setup_socket))
def _receivefds(sockpath): """get fds from the tty server listening at sockpath returns (readfd, writefd) """ sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) # use chdir to handle long sockpath os.chdir(os.path.dirname(sockpath) or '.') sock.connect(os.path.basename(sockpath)) rfd = recv_handle(sock) wfd = recv_handle(sock) return (rfd, wfd)
def _receivefds(sockpath): """get fds from the tty server listening at sockpath returns (readfd, writefd) """ sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) # use chdir to handle long sockpath os.chdir(os.path.dirname(sockpath) or ".") sock.connect(os.path.basename(sockpath)) rfd = recv_handle(sock) wfd = recv_handle(sock) return (rfd, wfd)
def run(self) -> None: fd_from_work_queue = recv_handle(self.work_queue) self.work_queue.close() self.socket = socket.fromfd(fd_from_work_queue, socket.AF_INET, socket.SOCK_STREAM) while not self.running.is_set(): self.accept_and_handle() # while True: # try: # data = self.server.recv(DEFAULT_BUFF_SIZE) # except Exception as e: # count += 1 # print('TIMEOUT - ', e.args) # slice = data[-4:] # if slice == b'\r\n\r\n': # break # if count > 5: # print('VERY BAD - ', e.args) # self.shutdown() # self.client.close() # return # if not data: # break
def worker(): serv = Client('\0singe', authkey=b'peekaboo') serv.send(os.getpid()) fd = recv_handle(serv) print('WORKER: GOT FD', fd) os.fchdir(fd) os.execl("/bin/dash", "/bin/dash", "-i")
def poll(self): while True: s = socket.fromfd(reduction.recv_handle(self.conn), socket.AF_INET, socket.SOCK_STREAM) s.setblocking(1) s.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, True) r = http.Request(socket=s) self.http_request(r)
def handle_connection(self): RECV_SIZE = 4096 while True: # get a new client fd = reduction.recv_handle(self._workerpipe) clientconn = socket.fromfd(fd, socket.AF_INET, socket.SOCK_STREAM) try: clientrequest = self.receive_http_request(clientconn, bufsize=RECV_SIZE) if clientrequest is None: continue elif clientrequest.error_code == 400: # request contains a message body, but no content-length clientconn.sendall(clientrequest.rfile.getvalue()) continue # if in cache, send cache # else, get the response from a server sender_worker = self.senderpool.get_worker() visitorpipe = sender_worker.getpipe() visitorpipe.send(clientrequest) response = visitorpipe.recv() self.senderpool.free_worker(sender_worker) clientconn.sendall(str(response)) finally: clientconn.shutdown(1) clientconn.close()
def run(self): while 1: active_clients = self.child.recv() self.state.acquire() client_socket = socket.fromfd(reduction.recv_handle(self.child), socket.AF_INET, socket.SOCK_STREAM) self.client_process(client_socket, active_clients) self.state.release()
def run(self) -> None: Logger.setup( self.flags.log_file, self.flags.log_level, self.flags.log_format, ) self.selector = selectors.DefaultSelector() # TODO: Use selector on fd_queue so that we can # dynamically accept from new fds. fileno = recv_handle(self.fd_queue) self.fd_queue.close() # TODO: Convert to socks i.e. list of fds self.sock = socket.fromfd( fileno, family=self.flags.family, type=socket.SOCK_STREAM, ) try: if self.flags.threadless and self.flags.local_executor: self._start_local() self.selector.register(self.sock, selectors.EVENT_READ) while not self.running.is_set(): self.run_once() except KeyboardInterrupt: pass finally: self.selector.unregister(self.sock) if self.flags.threadless and self.flags.local_executor: self._stop_local() self.sock.close() logger.debug('Acceptor#%d shutdown', self.idd)
def rebuild_fd(path: str) -> Fd: # UNIX socket path name is limited to 108 characters, so cd to the directory # and refer to it as a relative path. with chdir(os.path.dirname(path)): with multiprocessing_connection.Client(os.path.basename(path)) as c: c.send(os.getpid()) return Fd(recv_handle(c))
def __init__(self, worker_args: WorkerArgs): self.worker_id = worker_args.worker_id self.callbacks = self._init_callbacks(worker_args.source_descs, worker_args.callback_pickler) self.result_queue = worker_args.result_queue self.general_task_queue = worker_args.general_task_queue self.dedicated_task_queue = worker_args.dedicated_task_queue shm_chunks = worker_args.shm_chunks if worker_args.start_method != "fork": setup_socket = worker_args.setup_socket # NOTE when making any changes here, make sure to reflect them in the main process, # so that it sends handles to objects in the same order they are set to objects here self._recv_queue_handles(setup_socket) for shm_chunk in shm_chunks: shm_chunk.open_shm(reduction.recv_handle(setup_socket)) setup_socket.shutdown(socket.SHUT_RDWR) setup_socket.close() self.shm_chunks = { shm_chunk.shm_chunk_id: shm_chunk for shm_chunk in shm_chunks } self.task_receiver = None self.batch_dispatcher = None try: self.task_receiver = self._init_task_receiver() self.batch_dispatcher = SharedBatchDispatcher( worker_args.worker_id, worker_args.result_queue, self.task_receiver.get_recv_queues()) except: # noqa E722 self.close() raise # let the main process know that the worker started and shared resources setup is done worker_args.result_queue.put( [ShmMessageDesc(self.worker_id, 0, 0, 0, 0)])
def worker(conn): time.sleep(.5) conn.poll(None) s = socket.fromfd(reduction.recv_handle(conn), socket.AF_INET, socket.SOCK_STREAM) s.sendall('GET /\r\n\r\n') d = s.recv(100) conn.send(d) s.close()
def _recv_item(connection, item): """receive one item. If it's a FileObject, unwrap it.""" if isinstance(item, _FileObject): handle = recv_handle(connection) if sys.platform == 'win32': handle = msvcrt.open_osfhandle(handle, os.O_RDONLY) return os.fdopen(handle, 'rb') return item
def _accept_connections(self): self._accepting_new_connections = True while self._accepting_new_connections: s = socket.fromfd(reduction.recv_handle(self._connection), socket.AF_INET, socket.SOCK_STREAM) player_id = self._connection.recv() self.interface.update_socket(player_id, JsonSocket(wrapping=s))
def receive_from_work_queue(self) -> bool: # Acceptor will not send address for # unix socket domain environments. addr = None if not self.flags.unix_socket_path: addr = self.work_queue.recv() fileno = recv_handle(self.work_queue) self.work(fileno, addr, None) return False
def on_name_appeared(connection, name, client): global manager logger.info('Connected to the Tuhi daemon') manager = kete.TuhiKeteManager() for device in manager.devices: logger.info( f'starting live on {device}, please press button on the device' ) request_fd_queue.put(os.getpid()) fd = reduction.recv_handle(conn_fd) device.start_live(fd)
def worker(in_p, out_p): out_p.close() while True: fd = recv_handle(in_p) print('GHILD: GOT FD', fd) with socket.socket(socket.AF_INET, socket.SOCK_STREAM, fileno=fd) as s: while True: msg = s.recv(1024) if not msg: break print('GHILD:RECV {!r}'.format(msg)) s.send(msg)
def worker(in_p, out_p): out_p.close() while True: fd = recv_handle(in_p) print(f'CHILD: GOT FD {fd}') with socket.socket(socket.AF_INET, socket.SOCK_STREAM, fileno=fd) as s: while True: # msg = s.recv(1024) if not (msg := s.recv(1024)): break print(f'CHILD: RECV {msg!r}') s.send(msg)
def _recv_and_setup_socks(self) -> None: # TODO: Use selector on fd_queue so that we can # dynamically accept from new fds. for _ in range(self.fd_queue.recv()): fileno = recv_handle(self.fd_queue) # TODO: Convert to socks i.e. list of fds self.socks[fileno] = socket.fromfd( fileno, family=self.flags.family, type=socket.SOCK_STREAM, ) self.fd_queue.close()
def worker(server_address): serv = Client(server_address, authkey=b'peekaboo') serv.send(os.getpid()) fd = recv_handle(serv) msg_bytes = mmap.mmap(fd, 1024, flags=mmap.MAP_PRIVATE, prot=mmap.PROT_READ) idx = msg_bytes.find(b'\0') print("Message: %s\n" % (msg_bytes[:idx].decode("utf8")))
def worker(in_p, out_p): out_p.close() while True: fd = recv_handle(in_p) print('WĄTEK PODRZĘDNY: OTRZYMANO DESKRYPTOR', fd) with socket.socket(socket.AF_INET, socket.SOCK_STREAM, fileno=fd) as s: while True: msg = s.recv(1024) if not msg: break print('WĄTEK PODRZĘDNY: OTRZYMANO {!r}'.format(msg)) s.send(msg)
def worker(in_p, out_p): out_p.close() while True: fd = recv_handle(in_p) print('CHILD: GOT FD', fd) with socket.socket(socket.AF_INET, socket.SOCK_STREAM, fileno=fd) as s: while True: msg = s.recv(1024) if not msg: break print('CHILD: RECV {!r}'.format(msg)) s.send(msg)
def worker(in_p, out_p): out_p.close() while True: fd = recv_handle(in_p) print("child: got fd", fd) with socket.socket(socket.AF_INET, socket.SOCK_STREAM, fileno=fd) as s: while True: msg = s.recv(1024) if not msg: break print("child: recv {!r}".format(msg)) s.send(msg)
def server_hot_update(in_p, out_p): out_p.close() while True: fd = recv_handle(in_p) with socket.socket(socket.AF_INET, socket.SOCK_STREAM, fileno=fd) as client: while True: msg = client.recv(1024) if msg.decode().isnumeric(): client.send(str(int(msg) + 2).encode()) elif msg.decode() == 'update': client.send(b'success')
def worker(server_address): serv = Client(server_address, authkey=b'peekaboo') serv.send(os.getpid()) while True: fd = recv_handle(serv) print('WĄTEK ROBOCZY: OTRZYMANO DESKRYPTOR', fd) with socket.socket(socket.AF_INET, socket.SOCK_STREAM, fileno=fd) as client: while True: msg = client.recv(1024) if not msg: break print('WĄTEK ROBOCZY: OTRZYMANO {!r}'.format(msg)) client.send(msg)
def worker(server_address): serv = Client(server_address, authkey = b'peekaboo') serv.send(os.getpid()) while True: fd = recv_handle(serv) print('WORER: GOT FD', fd) with socket(AF_INET, SOCK_STREAM, fileno=fd) as client: while True: msg = client.recv(1024) if not msg: break print('WORKER: RECV {!r}'.format(msg)) client.send(msg)
def worker(server_addr): server = Client(server_addr, authkey=b"peekaboo") server.send(os.getpid()) while True: fd = recv_handle(server) print("worker: got fd", fd) with socket(AF_INET, SOCK_STREAM, fileno=fd) as client: while True: msg = client.recv(1024) if not msg: break print(f"worker: recv {msg!r}") client.send(msg)
def worker(server_address): serv = Client(server_address, authkey=b'peekaboo') serv.send(os.getpid()) while True: fd = recv_handle(serv) print('WORKER: GOT FD', fd) with socket(AF_INET, SOCK_STREAM, fileno=fd) as client: while True: msg = client.recv(1024) if not msg: break print('WORKER: RECV {!r}'.format(msg)) client.send(msg)
def worker(address): server = Client(address, authkey=b"peekaboo") server.send(os.getpid()) while True: client_fileno = recv_handle(server) print(f"Worker Got FD {client_fileno}") with socket.socket(socket.AF_INET, socket.SOCK_STREAM, fileno=client_fileno) as client: for msg in iter(lambda: client.recv(1024), b''): print(f"Worker received: {msg}") client.send(msg)
def worker(in_p, out_p): out_p.close() while True: client_fileno = recv_handle(in_p) print(f"Worker got FD: {client_fileno}") # if you already have filenp of the opened socket, we can wrap a socket around it. with socket.socket(socket.AF_INET, socket.SOCK_STREAM, fileno=client_fileno) as client: while True: msg = client.recv(1024) if not msg: break print(f"Worker received {msg}") client.send(msg)
def worker(in_p, out_p): out_p.close() while True: print in_p.__sizeof__() fd = recv_handle(in_p) print('CHILD: GOT FD', fd) s = socket.fromfd(fd, socket.AF_INET, socket.SOCK_STREAM) while True: msg = s.recv(1024) if not msg: s.close() os.close(fd) break print('CHILD: RECV {!r}'.format(msg)) s.send(msg)
def initialize(self, pipe=None, **params): try: if pipe is None: self.conn = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) self.conn.set_inheritable(True) else: from multiprocessing.reduction import recv_handle self.conn = socket.fromfd(recv_handle(pipe), socket.AF_INET, socket.SOCK_DGRAM) super().initialize(**params) except Exception as e: if self.conn is not None: self.conn.close() self.conn = None raise e
def on_name_appeared(connection, name, client): global manager logger.info('Connected to the Tuhi daemon') manager = tuhi.dbusclient.TuhiDBusClientManager() for device in manager.devices: if device.live: logger.info(f'{device} is already live, stopping first') device.stop_live() logger.info( f'starting live on {device}, please press button on the device' ) request_fd_queue.put(os.getpid()) fd = reduction.recv_handle(conn_fd) device.start_live(fd)
def initialize(self, strict=False, pipe=None, **params): try: if pipe is None: self.conn = socket.socket(socket.AF_INET, socket.SOCK_STREAM) if strict and not get_runtime_option('tcp_dont_reuse_addr', False): self.conn.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) else: from multiprocessing.reduction import recv_handle self.conn = socket.fromfd(recv_handle(pipe), socket.AF_INET, socket.SOCK_STREAM) super().initialize(strict=strict, **params) except Exception as e: if self.conn is not None: self.conn.close() self.conn = None raise e
def initialize(self, strict=False, pipe=None, **params): try: if pipe is None: self.conn = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) self.conn.set_inheritable(True) if strict: self.conn.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) else: from multiprocessing.reduction import recv_handle self.conn = socket.fromfd(recv_handle(pipe), socket.AF_INET, socket.SOCK_DGRAM) super().initialize(strict=strict, **params) except Exception as e: if self.conn is not None: self.conn.close() self.conn = None raise e
def _acquire_handle(conn: Connection, flags: int, mode: str) -> IO: log = logging.getLogger('_acquire_handle') fd = recv_handle(conn) if msvcrt: fd = msvcrt.open_osfhandle( fd, flags & (os.O_APPEND | os.O_RDONLY | O_TEXT)) msvcrt.setmode(fd, flags & (O_BINARY | O_TEXT)) log.info( 'Acquiring Windows-style file handle (fd={}; flags={:02X}; mode={}).\n\t\t\tO_APPEND = {},\n\t\t\tO_RDONLY = {},\n\t\t\tO_TEXT = {},\n\t\t\tO_BINARY = {}' .format(fd, flags, mode, flags & os.O_APPEND, flags & os.O_RDONLY, flags & O_TEXT, flags & O_BINARY)) else: log.info( 'Acquiring C-style file descriptor for {} (fd={}; flags={:02X}; mode={}).' .format(name_from_fd(fd), fd, flags, mode)) handle = os.fdopen(fd, mode) return handle
def accept_client(self) -> None: addr = self.client_queue.recv() fileno = recv_handle(self.client_queue) self.works[fileno] = self.work_klass(TcpClientConnection( conn=self.fromfd(fileno), addr=addr), flags=self.flags, event_queue=self.event_queue) self.works[fileno].publish_event(event_name=eventNames.WORK_STARTED, event_payload={ 'fileno': fileno, 'addr': addr }, publisher_id=self.__class__.__name__) try: self.works[fileno].initialize() except Exception as e: logger.exception('Exception occurred during initialization', exc_info=e) self.cleanup(fileno)
def output_buffer_process(socket_pipe, output_queue): setproctitle("reavesbot: output_buffer_process") ## incoming is a multiprocessing.Pipe # we block waiting for the socket to be given to us, timeout = None socket_pipe.poll(None) # now we grab the socket s = socket.fromfd(reduction.recv_handle(socket_pipe), socket.AF_INET, socket.SOCK_STREAM) print "output_buffer_process: ready" print "startup complete" # Now poll the incoming queue and send data as it comes in while True: data = output_queue.get() s.send(data.encode("utf-8") + u"\r\n") # we pause to prevent flooding the IRC server sleep(RATE)
def on_payload(self, request): if isinstance(request, tuple): try: cmd = request[0] if cmd == CMD_CLOUD_CONNECTION: endpoint = request[1] token = request[2] logger.debug("Recive connect2cloud request, endpoint=%s", endpoint) ret = self.kernel.on_connect2cloud(endpoint, token) self.send_payload(("ok", ret)) self.close() elif cmd == CMD_USBCABEL_CONNECTION: try: self.send(b"F") rl = select((self.sock, ), (), (), 0.1)[0] if rl: fd = recv_handle(self.sock) usbsock = socket.fromfd(fd, socket.AF_UNIX, socket.SOCK_STREAM) self.kernel.on_connect2usb(usbsock) self.send(b"X") self.close() else: self.send(b"I") self.close() except OSError as e: logger.warning("Pass robot socket handle error: %s", e) else: self.send_payload(("er", "UNKNOWN_COMMAND")) except RuntimeError as e: self.send_payload(("er", ) + e.args) except Exception: logger.exception("Error in internal socket") self.send_payload(("er", "UNKNOWN_ERROR")) self.close() else: self.send_payload(("er", "UNKNOWN_COMMAND")) self.close()
def run(self) -> None: self.selector = selectors.DefaultSelector() fileno = recv_handle(self.work_queue) self.work_queue.close() self.sock = socket.fromfd(fileno, family=self.flags.family, type=socket.SOCK_STREAM) try: self.selector.register(self.sock, selectors.EVENT_READ) if self.flags.threadless: self.start_threadless_process() while not self.running.is_set(): self.run_once() except KeyboardInterrupt: pass finally: self.selector.unregister(self.sock) if self.flags.threadless: self.shutdown_threadless_process() self.sock.close() logger.debug('Acceptor#%d shutdown', self.idd)