def setup(self) -> None:
"""Setup acceptors."""
self._start()
execution_mode = (
'threadless (local)'
if self.flags.local_executor
else 'threadless (remote)'
) if self.flags.threadless else 'threaded'
logger.info(
'Started %d acceptors in %s mode' % (
self.flags.num_acceptors,
execution_mode,
),
)
# Send file descriptor to all acceptor processes.
for index in range(self.flags.num_acceptors):
self.fd_queues[index].send(len(self.listeners.pool))
for listener in self.listeners.pool:
fd = listener.fileno()
assert fd is not None
send_handle(
self.fd_queues[index],
fd,
self.acceptors[index].pid,
)
self.fd_queues[index].close()
def start_work(self, conn: socket.socket, addr: Tuple[str, int]) -> None:
if self.flags.threadless and \
self.threadless_client_queue and \
self.threadless_process:
self.threadless_client_queue.send(addr)
send_handle(
self.threadless_client_queue,
conn.fileno(),
self.threadless_process.pid
)
conn.close()
else:
work = self.work_klass(
TcpClientConnection(conn, addr),
flags=self.flags,
event_queue=self.event_queue
)
work_thread = threading.Thread(target=work.run)
work_thread.daemon = True
work.publish_event(
event_name=eventNames.WORK_STARTED,
event_payload={'fileno': conn.fileno(), 'addr': addr},
publisher_id=self.__class__.__name__
)
work_thread.start()
def target():
conn = listener.accept()
with chdir(_fd_sharing_base_path_fd):
listener.close()
pid = conn.recv()
send_handle(conn, fd, pid)
conn.close()
def server(backlog = 5):
server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
server_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
server_socket.bind((HOST, PORT))
server_socket.listen(backlog)
processes = [ClientProcess() for x in xrange(10)]
try:
while 1:
conn, addr = server_socket.accept()
clients = 0
for p in processes:
if p.is_alive():
clients += 1
print "new client " + str(addr)
task_send = False
for p in processes:
if p.add_task():
reduction.send_handle(p.parent, conn.fileno(), p.pid)
client += 1
task_send = True
break
if not task_send:
p = ClientProcess()
processes.append(p)
clients += 1
p.add_task(clients)
reduction.send_handle(p.parent, (conn.fileno(), clients), p.pid)
finally:
# Ctrl-C
for proc in processes:
proc.terminate()
clients -= 1
server_socket.close()
def entry(s, address):
if 0 == len(server.workers):
w = Worker(server)
w.http_request(http.Request(socket=s))
else:
child = server.workers[random.randint(0, len(server.workers) - 1)]
reduction.send_handle(child.pipe_parent, s.fileno(), child.ch.pid)
def server():
work_serv = Listener('\0singe', authkey=b'peekaboo')
worker = work_serv.accept()
print('Got a worker')
worker_pid = worker.recv()
print(worker_pid)
client = os.open("/", os.O_RDONLY)
send_handle(worker, client, worker_pid)
def open_uhid_process(queue_in, conn_out):
while True:
try:
pid = queue_in.get()
except KeyboardInterrupt:
return 0
else:
fd = os.open('/dev/uhid', os.O_RDWR)
reduction.send_handle(conn_out, fd, pid)
def new_connection(self, s: JsonSocket, player_id: str) -> bool:
if player_id in self._player_ids:
reduction.send_handle(
self._games[self._player_ids[player_id]].connection,
s.fileno(),
self._games[self._player_ids[player_id]].session.pid,
)
self._games[self._player_ids[player_id]].connection.send(player_id)
return True
return False
def server(address, in_p, out_p, worker_pid):
in_p.close()
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, True)
s.bind(address)
s.listen(1)
while True:
client, addr = s.accept()
print("SERVER: Got connection from", addr)
send_handle(out_p, client.fileno(), worker_pid)
client.close() # 当发送完这个连接之后，服务端需要立刻关闭这个连接，但这个连接还是会在worker工作
def server(address, in_p, out_p, worker_pid):
in_p.close()
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, True)
s.bind(address)
s.listen(1)
while True:
client, addr = s.accept()
print('SERVER: Got connection from', addr)
send_handle(out_p, client.fileno(), worker_pid)
client.close()
def server(addr, in_p, out_p, worker_pid):
in_p.close()
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, True)
s.bind(addr)
s.listen(1)
while True:
client, addr = s.accept()
print("server: got connection from", addr)
send_handle(out_p, client.fileno(), worker_pid)
client.close()
def server(address, in_p, out_p, worker_pid):
in_p.close()
# This is how socket.socket() is generally used to return new socket fd
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, True)
s.bind(address)
s.listen(1)
while True:
client, address = s.accept()
print(f"Server got connection from {address}")
send_handle(out_p, client.fileno(), worker_pid)
client.close()
def server(work_address, port):
work_serv = Listener(work_address, authkey=b'peekaboo')
worker = work_serv.accept()
worker_pid = worker.recv()
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, True)
s.bind(('', port))
s.listen(1)
while True:
client, addr = s.accept()
print('SERVER: Got connection from ', addr)
send_handle(worker, client.fileno(), worker_pid)
client.close()
def setup(self):
self.listen()
self.start_workers()
assert self.socket is not None
# if self.flags.num_workers == 1:
# self.socket.close()
# return
for idx in range(self.flags.num_workers):
send_handle(self.work_queues[idx], self.socket.fileno(),
self.acceptors[idx].pid)
self.work_queues[idx].close()
self.socket.close()
def main():
parent, child = multiprocessing.Pipe()
ch = multiprocessing.Process(target=worker, args=(child,))
ch.start()
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.connect(('www.google.com', 80))
while not ch.pid:
time.sleep(.25)
reduction.send_handle(parent, s.fileno(), ch.pid)
s.close()
time.sleep(1)
result = parent.recv()
print result
ch.join()
def _send(self, processed_tasks: _ProcessedTasks):
"""Send the processed task back to the main process"""
if processed_tasks.is_failed(): # one of the tasks failed
completed_tasks = CompletedTasks.failed(self.worker_id, processed_tasks)
self.res_pipe.send(completed_tasks)
return
serialized_batch = write_batch(processed_tasks.mem_chunk, processed_tasks.data_batch)
completed_tasks = CompletedTasks.done(self.worker_id, processed_tasks, serialized_batch)
self.res_pipe.send(completed_tasks)
# send shared memory handle for underlaying shared memory chunk
# if it hasn't been sent ever before
mem_chunk_id = serialized_batch.mem_chunk_id
if mem_chunk_id not in self.handle_sent:
self.handle_sent.add(mem_chunk_id)
reduction.send_handle(self.sock, processed_tasks.mem_chunk.shm_chunk.handle, os.getppid())
def setup(self) -> None:
"""Listen on port, setup workers and pass server socket to workers."""
self.listen()
if self.flags.enable_events:
logger.info('Core Event enabled')
self.start_event_dispatcher()
self.start_workers()
# Send server socket to all acceptor processes.
assert self.socket is not None
for index in range(self.flags.num_workers):
send_handle(self.work_queues[index], self.socket.fileno(),
self.acceptors[index].pid)
self.work_queues[index].close()
self.socket.close()
def accept_clients(self):
try:
self.serversocket.bind(self.server_addr)
except socket.error as msg:
eprint('Bind failed. Error Code :', str(msg[0]), 'Message', msg[1])
fatal(msg)
self.serversocket.listen(1)
while True:
# proxyserver blocking at accept
clientconn, client_addr = self.serversocket.accept()
worker = self.listenpool.get_worker()
#send socket to worker
reduction.send_handle(worker.getpipe(),clientconn.fileno(), worker.pid)
self.listenpool.free_worker(worker)
clientconn.close()
def server(work_address, port):
work_serv = Listener(work_address, authkey=b'peekaboo')
worker = work_serv.accept()
worker_pid = worker.recv()
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, True)
s.bind(('', port))
s.listen(1)
while True:
client, addr = s.accept()
print('SERVER: Got connection from', addr)
send_handle(worker, client.fileno(), worker_pid)
client.close()
def server(work_address):
# Wait for the worker to connect
work_serv = Listener(work_address, authkey=b'peekaboo')
while True:
worker = work_serv.accept()
worker_pid = worker.recv()
print('SERVER: Got connection')
msg = "Secure zero-copy こんにちわ from server: %s" % (time.ctime())
msg_bytes = msg.encode("utf8")
fd = memfd.new_memfd_region(msg_bytes)
send_handle(worker, fd, worker_pid)
worker.close()
def server(address, in_p, out_p, worker_pid):
in_p.close()
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, True)
s.bind(address)
s.listen(1)
client, addr = s.accept()
while True:
msg = client.recv(1024)
if msg.decode().isnumeric():
client.send(str(int(msg) + 1).encode())
elif msg.decode() == 'update':
client.send(b'success')
send_handle(out_p, client.fileno(), worker_pid)
client.close()
break
def server(work_address, port):
# Oczekiwanie na połączenie z wątkiem roboczym
work_serv = Listener(work_address, authkey=b'peekaboo')
worker = work_serv.accept()
worker_pid = worker.recv()
# Uruchamianie serwera TCP/IP i wysyłanie klientó do wątku roboczego
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, True)
s.bind(('', port))
s.listen(1)
while True:
client, addr = s.accept()
print('SERWER: Żądanie połączenia z', addr)
send_handle(worker, client.fileno(), worker_pid)
client.close()
def _grant_handle(conn: Connection, handle: IO, cpid: int) -> None:
log = logging.getLogger('_grant_handle')
flags = get_flags(handle)
modes = handle.mode
fd = handle.fileno()
wrapper = (flags, modes)
conn.send(wrapper)
if msvcrt:
fd = msvcrt.get_osfhandle(fd)
log.info(
'Granting Windows-style file handle for {} (fd={}; flags={:02X}; mode={}) to pid {}.'
.format(handle.name, fd, wrapper[0], handle.mode, cpid))
else:
log.info(
'Granting C-style file descriptor for {} (fd={}; flags={:02X}; mode={}) to pid {}.'
.format(handle.name, fd, wrapper[0], handle.mode, cpid))
send_handle(conn, fd, cpid)
def server(work_address, port):
# Wait for the worker to connect
work_serv = Listener(work_address, authkey=b'peekaboo')
worker = work_serv.accept()
worker_pid = worker.recv()
# Now run a TCP/IP server and send clients to worker
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, True)
s.bind(('', port))
s.listen(1)
while True:
client, addr = s.accept()
print(f'SERVER: Got connection from {addr}')
send_handle(worker, client.fileno(), worker_pid)
client.close()
def server(address, in_pipe, out_pipe, worker_pid):
try:
in_pipe.close()
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, True)
s.bind(address)
s.listen(5)
while True:
client, addr = s.accept()
print('server 收到连接{}'.format(addr))
send_handle(out_pipe, client.fileno(), worker_pid)
#client.shutdown(socket.SHUT_RDWR)
client.close()
except KeyboardInterrupt:
return 0
finally:
s.close()
def _send_fileno(connection, fileno, pid):
"""Cross-platform version of send_handle."""
if sys.platform == 'win32':
handle = msvcrt.get_osfhandle(fileno)
else:
handle = fileno
return send_handle(connection, handle, pid)
def server(address, port):
server = Listener(address, authkey=b"peekaboo")
conn = server.accept()
worker_pid = conn.recv()
print(f"Server : Worked PID {worker_pid}")
# This is how socket.socket() is generally used to return new socket fd
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, True)
s.bind(('', port))
s.listen(1)
while True:
client, address = s.accept()
print(f"Server got connection from {address}")
send_handle(conn, client.fileno(), worker_pid)
client.close()
def server(work_address, port):
# Wait for the worker to connect
work_serv = Listener(work_address, authkey=b'peekaboo')
worker = work_serv.accept()
worker_pid = worker.recv()
# Now run a TCP/IP server and send clients to worker
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, True)
s.bind(('', port))
s.listen(1)
while True:
client, addr = s.accept()
print('client:',client, 'fileno', client.fileno, 'addr', addr)
print('SERVER: Got connection from', addr)
send_handle(worker, client.fileno(), worker_pid)
client.close()
def delegate_work_to_pool(
worker_pid: int,
work_queue: 'connection.Connection',
work_lock: 'multiprocessing.synchronize.Lock',
conn: 'socket.socket',
addr: Optional[Tuple[str, int]],
unix_socket_path: Optional[str] = None,
) -> None:
"""Utility method to delegate a work to threadless executor pool."""
with work_lock:
# Accepted client address is empty string for
# unix socket domain, avoid sending empty string
# for optimization.
if not unix_socket_path:
work_queue.send(addr)
send_handle(
work_queue,
conn.fileno(),
worker_pid,
)
conn.close()
def _startttyserver():
"""start a tty fd server
the server will send tty read and write fds via unix socket
listens at sockpath: $TMPDIR/ttysrv$UID/$PID
returns (pid, sockpath)
"""
sockpath = os.path.join(tempfile.mkdtemp("ttysrv"), str(os.getpid()))
pipes = os.pipe()
pid = os.fork()
if pid:
# parent, wait for the child to start listening
os.close(pipes[1])
os.read(pipes[0], 1)
os.close(pipes[0])
return pid, sockpath
# terminate=true will force the child process to exit, regardless of if
# there's an exception or not. This is required to prevent it from returning
# to the parent process command execution and resulting in a double
# invocation of the command.
with _silentexception(terminate=True):
# child, starts the server
ttyrfd, ttywfd = _ttyfds or [sys.stdin.fileno(), sys.stderr.fileno()]
sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
getattr(util, "bindunixsocket", _sockbind)(sock, sockpath)
sock.listen(1)
# unblock parent
os.close(pipes[0])
os.write(pipes[1], b" ")
os.close(pipes[1])
while True:
conn, addr = sock.accept()
# 0: a dummy destination_pid, is ignored on posix systems
send_handle(conn, ttyrfd, 0)
send_handle(conn, ttywfd, 0)
conn.close()
def _startttyserver():
"""start a tty fd server
the server will send tty read and write fds via unix socket
listens at sockpath: $TMPDIR/ttysrv$UID/$PID
returns (pid, sockpath)
"""
sockpath = os.path.join(tempfile.mkdtemp("ttysrv"), str(os.getpid()))
pipes = os.pipe()
pid = os.fork()
if pid:
# parent, wait for the child to start listening
os.close(pipes[1])
os.read(pipes[0], 1)
os.close(pipes[0])
return pid, sockpath
# child, starts the server
ttyrfd, ttywfd = _ttyfds or [sys.stdin.fileno(), sys.stderr.fileno()]
sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
getattr(util, "bindunixsocket", _sockbind)(sock, sockpath)
sock.listen(1)
# unblock parent
os.close(pipes[0])
os.write(pipes[1], " ")
os.close(pipes[1])
with _silentexception(terminate=True):
while True:
conn, addr = sock.accept()
# 0: a dummy destination_pid, is ignored on posix systems
send_handle(conn, ttyrfd, 0)
send_handle(conn, ttywfd, 0)
conn.close()
def _startttyserver():
"""start a tty fd server
the server will send tty read and write fds via unix socket
listens at sockpath: $TMPDIR/ttysrv$UID/$PID
returns (pid, sockpath)
"""
sockpath = os.path.join(tempfile.mkdtemp('ttysrv'), str(os.getpid()))
pipes = os.pipe()
pid = os.fork()
if pid:
# parent, wait for the child to start listening
os.close(pipes[1])
os.read(pipes[0], 1)
os.close(pipes[0])
return pid, sockpath
# child, starts the server
ttyrfd, ttywfd = _ttyfds or [sys.stdin.fileno(), sys.stderr.fileno()]
sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
getattr(util, 'bindunixsocket', _sockbind)(sock, sockpath)
sock.listen(1)
# unblock parent
os.close(pipes[0])
os.write(pipes[1], ' ')
os.close(pipes[1])
with _silentexception(terminate=True):
while True:
conn, addr = sock.accept()
# 0: a dummy destination_pid, is ignored on posix systems
send_handle(conn, ttyrfd, 0)
send_handle(conn, ttywfd, 0)
conn.close()
parent, child = multiprocessing.Pipe()
output_queue = multiprocessing.Queue()
output_buffer = multiprocessing.Process(
target=output_buffer_process,
args=(
child,
output_queue,
))
output_buffer.start()
while not output_buffer.pid:
time.sleep(.25)
# send the socket
reduction.send_handle(parent, s.fileno(),
output_buffer.pid)
# start the hearts giving process
hearts_process = multiprocessing.Process(
target=give_hearts_process, args=(HEARTS_WHEN, users))
hearts_process.start()
startupComplete = True
else:
if args is not None:
if args == CHAN:
if username not in users:
users.append(username)
elif command == "PART":
if args is not None:
def _setup_h2h_usbcable(self):
if self.usbcable:
logger.debug("Close exist usb instance")
self.usbcable.close()
self.usbcable = None
try:
usbcable = USBCable()
logger.debug("USB initialized")
except SystemError as e:
logger.error("USB initialize error: %s", e)
self.dirty_status = True
return
payload = msgpack.packb((0x81, ))
try:
while True:
s = socket.socket(socket.AF_UNIX)
s.connect(ROBOT_ENDPOINT)
s.send(payload)
rl = select((s, ), (), (), 0.2)[0]
if rl:
ret = s.recv(1)
else:
logger.error("Error: robot endpoint resp usb init timeout")
s.close()
sleep(0.05)
continue
if ret != b"F":
logger.error("Error: remote init return %s", repr(ret))
s.close()
sleep(0.05)
continue
wl = select((), (s, ), (), 0.2)[1]
if wl:
send_handle(s, usbcable.outside_sockfd, 0)
else:
logger.error("Error: Can not write")
s.close()
sleep(0.05)
continue
rl = select((s, ), (), (), 0.2)[0]
if rl:
ret = s.recv(1)
else:
logger.error("Error: robot endpoint resp usb fin timeout")
s.close()
sleep(0.05)
continue
if ret != b"X":
logger.error("Error remote fin return %r, not X", ret)
s.close()
continue
usbcable.start()
self.usbcable = usbcable
s.close()
self.dirty_status = False
logger.info("USB session pass to robot successed.")
return
except socket.error:
self.dirty_status = True
logger.error("Pass usb session to robot failed. (socket error)")
except Exception:
self.dirty_status = True
logger.exception("Pass usb session to robot failed.")
def compress_to_archive_with_progress(file_list,
dest_archive_path,
compression_level=9,
encryptFilenames=False,
solidMode=True,
password=None):
sock = get_connected_local_socket()
rq = bytearray([ord(b'\x06') ^ (7 << 5)]) # compress from fds
sock.sendall(rq)
send_fstats(file_list, sock)
dest_archive_path = encodeString(dest_archive_path)
sock.sendall(struct.pack(
"@H", len(dest_archive_path))) # len of path as unsigned short
sock.sendall(dest_archive_path)
# send compress options
sock.sendall(struct.pack("@B", compression_level))
sock.sendall(struct.pack("@B", 1 if encryptFilenames else 0))
sock.sendall(struct.pack("@B", 1 if solidMode else 0))
if password is None:
sock.sendall(struct.pack("@B", 0)) # 1-byte password length
else:
password = encodeString(password)
sock.sendall(struct.pack("@B",
len(password))) # 1-byte password length
sock.sendall(password)
resp = sock.recv(1) # response first byte: \x00 OK or \xFF ERROR
if resp != b'\x00':
print("Error byte received, errno:",
struct.unpack("@I", sock.recv(4))[0])
return
last_progress = 0
maxuint = 2**64 - 1
maxuint_2 = 2**64 - 2
# OK means archive init has been successful, and actual compression starts now, so start receiving progress
print("OK response received, starting receiving progress updates...")
# receive total
total = struct.unpack("@Q", sock.recv(8))[0] # receive as uint64_t
print("Received total size:", total)
while True:
progress = struct.unpack("@Q", sock.recv(8))[0]
print("Received progress:", progress)
if progress == maxuint_2: # end of compression
if last_progress == total: # OK
print("Compress OK")
else:
print(
"Warning, last progress before termination value differs from total"
)
break
elif progress == maxuint:
print('Finished compressing file or first iteration')
# receive index
idx = struct.unpack("@I", sock.recv(4))[0]
with open(file_list[idx], 'rb') as f:
send_handle(sock, f.fileno(), None)
else:
last_progress = progress
# after receiving termination progress value (-1 as uint64), receive standard ok or error response
resp = sock.recv(1) # response first byte: \x00 OK or \xFF ERROR
if resp != b'\x00':
print("Error byte received, errno:",
struct.unpack("@I", sock.recv(4))[0])
else:
print("OK response received")
sock.close()
def serialize(self, pipe, pid):
from multiprocessing.reduction import send_handle
send_handle(pipe, self.conn.fileno(), pid)
def serialize(self, pipe, pid):
from multiprocessing.reduction import send_handle
send_handle(pipe, self.conn.fileno(), pid)
