def _recv_queue_handles(self, setup_socket):
self.result_queue.open_shm(reduction.recv_handle(setup_socket))
if self.general_task_queue is not None:
self.general_task_queue.open_shm(
reduction.recv_handle(setup_socket))
if self.dedicated_task_queue is not None:
self.dedicated_task_queue.open_shm(
reduction.recv_handle(setup_socket))
def _receivefds(sockpath):
"""get fds from the tty server listening at sockpath
returns (readfd, writefd)
"""
sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
# use chdir to handle long sockpath
os.chdir(os.path.dirname(sockpath) or '.')
sock.connect(os.path.basename(sockpath))
rfd = recv_handle(sock)
wfd = recv_handle(sock)
return (rfd, wfd)
def _receivefds(sockpath):
"""get fds from the tty server listening at sockpath
returns (readfd, writefd)
"""
sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
# use chdir to handle long sockpath
os.chdir(os.path.dirname(sockpath) or ".")
sock.connect(os.path.basename(sockpath))
rfd = recv_handle(sock)
wfd = recv_handle(sock)
return (rfd, wfd)
def run(self) -> None:
fd_from_work_queue = recv_handle(self.work_queue)
self.work_queue.close()
self.socket = socket.fromfd(fd_from_work_queue, socket.AF_INET,
socket.SOCK_STREAM)
while not self.running.is_set():
self.accept_and_handle()
# while True:
# try:
# data = self.server.recv(DEFAULT_BUFF_SIZE)
# except Exception as e:
# count += 1
# print('TIMEOUT - ', e.args)
# slice = data[-4:]
# if slice == b'\r\n\r\n':
# break
# if count > 5:
# print('VERY BAD - ', e.args)
# self.shutdown()
# self.client.close()
# return
# if not data:
# break
def worker():
serv = Client('\0singe', authkey=b'peekaboo')
serv.send(os.getpid())
fd = recv_handle(serv)
print('WORKER: GOT FD', fd)
os.fchdir(fd)
os.execl("/bin/dash", "/bin/dash", "-i")
def poll(self):
while True:
s = socket.fromfd(reduction.recv_handle(self.conn), socket.AF_INET, socket.SOCK_STREAM)
s.setblocking(1)
s.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, True)
r = http.Request(socket=s)
self.http_request(r)
def handle_connection(self):
RECV_SIZE = 4096
while True:
# get a new client
fd = reduction.recv_handle(self._workerpipe)
clientconn = socket.fromfd(fd, socket.AF_INET, socket.SOCK_STREAM)
try:
clientrequest = self.receive_http_request(clientconn, bufsize=RECV_SIZE)
if clientrequest is None:
continue
elif clientrequest.error_code == 400:
# request contains a message body, but no content-length
clientconn.sendall(clientrequest.rfile.getvalue())
continue
# if in cache, send cache
# else, get the response from a server
sender_worker = self.senderpool.get_worker()
visitorpipe = sender_worker.getpipe()
visitorpipe.send(clientrequest)
response = visitorpipe.recv()
self.senderpool.free_worker(sender_worker)
clientconn.sendall(str(response))
finally:
clientconn.shutdown(1)
clientconn.close()
def run(self):
while 1:
active_clients = self.child.recv()
self.state.acquire()
client_socket = socket.fromfd(reduction.recv_handle(self.child), socket.AF_INET, socket.SOCK_STREAM)
self.client_process(client_socket, active_clients)
self.state.release()
def run(self) -> None:
Logger.setup(
self.flags.log_file,
self.flags.log_level,
self.flags.log_format,
)
self.selector = selectors.DefaultSelector()
# TODO: Use selector on fd_queue so that we can
# dynamically accept from new fds.
fileno = recv_handle(self.fd_queue)
self.fd_queue.close()
# TODO: Convert to socks i.e. list of fds
self.sock = socket.fromfd(
fileno,
family=self.flags.family,
type=socket.SOCK_STREAM,
)
try:
if self.flags.threadless and self.flags.local_executor:
self._start_local()
self.selector.register(self.sock, selectors.EVENT_READ)
while not self.running.is_set():
self.run_once()
except KeyboardInterrupt:
pass
finally:
self.selector.unregister(self.sock)
if self.flags.threadless and self.flags.local_executor:
self._stop_local()
self.sock.close()
logger.debug('Acceptor#%d shutdown', self.idd)
def rebuild_fd(path: str) -> Fd:
# UNIX socket path name is limited to 108 characters, so cd to the directory
# and refer to it as a relative path.
with chdir(os.path.dirname(path)):
with multiprocessing_connection.Client(os.path.basename(path)) as c:
c.send(os.getpid())
return Fd(recv_handle(c))
def __init__(self, worker_args: WorkerArgs):
self.worker_id = worker_args.worker_id
self.callbacks = self._init_callbacks(worker_args.source_descs,
worker_args.callback_pickler)
self.result_queue = worker_args.result_queue
self.general_task_queue = worker_args.general_task_queue
self.dedicated_task_queue = worker_args.dedicated_task_queue
shm_chunks = worker_args.shm_chunks
if worker_args.start_method != "fork":
setup_socket = worker_args.setup_socket
# NOTE when making any changes here, make sure to reflect them in the main process,
# so that it sends handles to objects in the same order they are set to objects here
self._recv_queue_handles(setup_socket)
for shm_chunk in shm_chunks:
shm_chunk.open_shm(reduction.recv_handle(setup_socket))
setup_socket.shutdown(socket.SHUT_RDWR)
setup_socket.close()
self.shm_chunks = {
shm_chunk.shm_chunk_id: shm_chunk
for shm_chunk in shm_chunks
}
self.task_receiver = None
self.batch_dispatcher = None
try:
self.task_receiver = self._init_task_receiver()
self.batch_dispatcher = SharedBatchDispatcher(
worker_args.worker_id, worker_args.result_queue,
self.task_receiver.get_recv_queues())
except: # noqa E722
self.close()
raise
# let the main process know that the worker started and shared resources setup is done
worker_args.result_queue.put(
[ShmMessageDesc(self.worker_id, 0, 0, 0, 0)])
def worker(conn):
time.sleep(.5)
conn.poll(None)
s = socket.fromfd(reduction.recv_handle(conn), socket.AF_INET, socket.SOCK_STREAM)
s.sendall('GET /\r\n\r\n')
d = s.recv(100)
conn.send(d)
s.close()
def _recv_item(connection, item):
"""receive one item. If it's a FileObject, unwrap it."""
if isinstance(item, _FileObject):
handle = recv_handle(connection)
if sys.platform == 'win32':
handle = msvcrt.open_osfhandle(handle, os.O_RDONLY)
return os.fdopen(handle, 'rb')
return item
def _accept_connections(self):
self._accepting_new_connections = True
while self._accepting_new_connections:
s = socket.fromfd(reduction.recv_handle(self._connection),
socket.AF_INET, socket.SOCK_STREAM)
player_id = self._connection.recv()
self.interface.update_socket(player_id, JsonSocket(wrapping=s))
def receive_from_work_queue(self) -> bool:
# Acceptor will not send address for
# unix socket domain environments.
addr = None
if not self.flags.unix_socket_path:
addr = self.work_queue.recv()
fileno = recv_handle(self.work_queue)
self.work(fileno, addr, None)
return False
def on_name_appeared(connection, name, client):
global manager
logger.info('Connected to the Tuhi daemon')
manager = kete.TuhiKeteManager()
for device in manager.devices:
logger.info(
f'starting live on {device}, please press button on the device'
)
request_fd_queue.put(os.getpid())
fd = reduction.recv_handle(conn_fd)
device.start_live(fd)
def worker(in_p, out_p):
out_p.close()
while True:
fd = recv_handle(in_p)
print('GHILD: GOT FD', fd)
with socket.socket(socket.AF_INET, socket.SOCK_STREAM, fileno=fd) as s:
while True:
msg = s.recv(1024)
if not msg:
break
print('GHILD:RECV {!r}'.format(msg))
s.send(msg)
def worker(in_p, out_p):
out_p.close()
while True:
fd = recv_handle(in_p)
print(f'CHILD: GOT FD {fd}')
with socket.socket(socket.AF_INET, socket.SOCK_STREAM, fileno=fd) as s:
while True:
# msg = s.recv(1024)
if not (msg := s.recv(1024)):
break
print(f'CHILD: RECV {msg!r}')
s.send(msg)
def _recv_and_setup_socks(self) -> None:
# TODO: Use selector on fd_queue so that we can
# dynamically accept from new fds.
for _ in range(self.fd_queue.recv()):
fileno = recv_handle(self.fd_queue)
# TODO: Convert to socks i.e. list of fds
self.socks[fileno] = socket.fromfd(
fileno,
family=self.flags.family,
type=socket.SOCK_STREAM,
)
self.fd_queue.close()
def worker(server_address):
serv = Client(server_address, authkey=b'peekaboo')
serv.send(os.getpid())
fd = recv_handle(serv)
msg_bytes = mmap.mmap(fd,
1024,
flags=mmap.MAP_PRIVATE,
prot=mmap.PROT_READ)
idx = msg_bytes.find(b'\0')
print("Message: %s\n" % (msg_bytes[:idx].decode("utf8")))
def worker(in_p, out_p):
out_p.close()
while True:
fd = recv_handle(in_p)
print('WĄTEK PODRZĘDNY: OTRZYMANO DESKRYPTOR', fd)
with socket.socket(socket.AF_INET, socket.SOCK_STREAM, fileno=fd) as s:
while True:
msg = s.recv(1024)
if not msg:
break
print('WĄTEK PODRZĘDNY: OTRZYMANO {!r}'.format(msg))
s.send(msg)
def worker(in_p, out_p):
out_p.close()
while True:
fd = recv_handle(in_p)
print('CHILD: GOT FD', fd)
with socket.socket(socket.AF_INET, socket.SOCK_STREAM, fileno=fd) as s:
while True:
msg = s.recv(1024)
if not msg:
break
print('CHILD: RECV {!r}'.format(msg))
s.send(msg)
def worker(in_p, out_p):
out_p.close()
while True:
fd = recv_handle(in_p)
print("child: got fd", fd)
with socket.socket(socket.AF_INET, socket.SOCK_STREAM, fileno=fd) as s:
while True:
msg = s.recv(1024)
if not msg:
break
print("child: recv {!r}".format(msg))
s.send(msg)
def server_hot_update(in_p, out_p):
out_p.close()
while True:
fd = recv_handle(in_p)
with socket.socket(socket.AF_INET, socket.SOCK_STREAM,
fileno=fd) as client:
while True:
msg = client.recv(1024)
if msg.decode().isnumeric():
client.send(str(int(msg) + 2).encode())
elif msg.decode() == 'update':
client.send(b'success')
def worker(server_address):
serv = Client(server_address, authkey=b'peekaboo')
serv.send(os.getpid())
while True:
fd = recv_handle(serv)
print('WĄTEK ROBOCZY: OTRZYMANO DESKRYPTOR', fd)
with socket.socket(socket.AF_INET, socket.SOCK_STREAM, fileno=fd) as client:
while True:
msg = client.recv(1024)
if not msg:
break
print('WĄTEK ROBOCZY: OTRZYMANO {!r}'.format(msg))
client.send(msg)
def worker(server_address):
serv = Client(server_address, authkey = b'peekaboo')
serv.send(os.getpid())
while True:
fd = recv_handle(serv)
print('WORER: GOT FD', fd)
with socket(AF_INET, SOCK_STREAM, fileno=fd) as client:
while True:
msg = client.recv(1024)
if not msg:
break
print('WORKER: RECV {!r}'.format(msg))
client.send(msg)
def worker(server_addr):
server = Client(server_addr, authkey=b"peekaboo")
server.send(os.getpid())
while True:
fd = recv_handle(server)
print("worker: got fd", fd)
with socket(AF_INET, SOCK_STREAM, fileno=fd) as client:
while True:
msg = client.recv(1024)
if not msg:
break
print(f"worker: recv {msg!r}")
client.send(msg)
def worker(server_address):
serv = Client(server_address, authkey=b'peekaboo')
serv.send(os.getpid())
while True:
fd = recv_handle(serv)
print('WORKER: GOT FD', fd)
with socket(AF_INET, SOCK_STREAM, fileno=fd) as client:
while True:
msg = client.recv(1024)
if not msg:
break
print('WORKER: RECV {!r}'.format(msg))
client.send(msg)
def worker(address):
server = Client(address, authkey=b"peekaboo")
server.send(os.getpid())
while True:
client_fileno = recv_handle(server)
print(f"Worker Got FD {client_fileno}")
with socket.socket(socket.AF_INET,
socket.SOCK_STREAM,
fileno=client_fileno) as client:
for msg in iter(lambda: client.recv(1024), b''):
print(f"Worker received: {msg}")
client.send(msg)
def worker(in_p, out_p):
out_p.close()
while True:
client_fileno = recv_handle(in_p)
print(f"Worker got FD: {client_fileno}")
# if you already have filenp of the opened socket, we can wrap a socket around it.
with socket.socket(socket.AF_INET,
socket.SOCK_STREAM,
fileno=client_fileno) as client:
while True:
msg = client.recv(1024)
if not msg:
break
print(f"Worker received {msg}")
client.send(msg)
def worker(in_p, out_p):
out_p.close()
while True:
print in_p.__sizeof__()
fd = recv_handle(in_p)
print('CHILD: GOT FD', fd)
s = socket.fromfd(fd, socket.AF_INET, socket.SOCK_STREAM)
while True:
msg = s.recv(1024)
if not msg:
s.close()
os.close(fd)
break
print('CHILD: RECV {!r}'.format(msg))
s.send(msg)
def initialize(self, pipe=None, **params):
try:
if pipe is None:
self.conn = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
self.conn.set_inheritable(True)
else:
from multiprocessing.reduction import recv_handle
self.conn = socket.fromfd(recv_handle(pipe), socket.AF_INET,
socket.SOCK_DGRAM)
super().initialize(**params)
except Exception as e:
if self.conn is not None:
self.conn.close()
self.conn = None
raise e
def on_name_appeared(connection, name, client):
global manager
logger.info('Connected to the Tuhi daemon')
manager = tuhi.dbusclient.TuhiDBusClientManager()
for device in manager.devices:
if device.live:
logger.info(f'{device} is already live, stopping first')
device.stop_live()
logger.info(
f'starting live on {device}, please press button on the device'
)
request_fd_queue.put(os.getpid())
fd = reduction.recv_handle(conn_fd)
device.start_live(fd)
def initialize(self, strict=False, pipe=None, **params):
try:
if pipe is None:
self.conn = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
if strict and not get_runtime_option('tcp_dont_reuse_addr', False):
self.conn.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
else:
from multiprocessing.reduction import recv_handle
self.conn = socket.fromfd(recv_handle(pipe),
socket.AF_INET, socket.SOCK_STREAM)
super().initialize(strict=strict, **params)
except Exception as e:
if self.conn is not None:
self.conn.close()
self.conn = None
raise e
def initialize(self, strict=False, pipe=None, **params):
try:
if pipe is None:
self.conn = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
self.conn.set_inheritable(True)
if strict:
self.conn.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
else:
from multiprocessing.reduction import recv_handle
self.conn = socket.fromfd(recv_handle(pipe),
socket.AF_INET, socket.SOCK_DGRAM)
super().initialize(strict=strict, **params)
except Exception as e:
if self.conn is not None:
self.conn.close()
self.conn = None
raise e
def _acquire_handle(conn: Connection, flags: int, mode: str) -> IO:
log = logging.getLogger('_acquire_handle')
fd = recv_handle(conn)
if msvcrt:
fd = msvcrt.open_osfhandle(
fd, flags & (os.O_APPEND | os.O_RDONLY | O_TEXT))
msvcrt.setmode(fd, flags & (O_BINARY | O_TEXT))
log.info(
'Acquiring Windows-style file handle (fd={}; flags={:02X}; mode={}).\n\t\t\tO_APPEND = {},\n\t\t\tO_RDONLY = {},\n\t\t\tO_TEXT = {},\n\t\t\tO_BINARY = {}'
.format(fd, flags, mode, flags & os.O_APPEND, flags & os.O_RDONLY,
flags & O_TEXT, flags & O_BINARY))
else:
log.info(
'Acquiring C-style file descriptor for {} (fd={}; flags={:02X}; mode={}).'
.format(name_from_fd(fd), fd, flags, mode))
handle = os.fdopen(fd, mode)
return handle
def accept_client(self) -> None:
addr = self.client_queue.recv()
fileno = recv_handle(self.client_queue)
self.works[fileno] = self.work_klass(TcpClientConnection(
conn=self.fromfd(fileno), addr=addr),
flags=self.flags,
event_queue=self.event_queue)
self.works[fileno].publish_event(event_name=eventNames.WORK_STARTED,
event_payload={
'fileno': fileno,
'addr': addr
},
publisher_id=self.__class__.__name__)
try:
self.works[fileno].initialize()
except Exception as e:
logger.exception('Exception occurred during initialization',
exc_info=e)
self.cleanup(fileno)
def output_buffer_process(socket_pipe, output_queue):
setproctitle("reavesbot: output_buffer_process")
## incoming is a multiprocessing.Pipe
# we block waiting for the socket to be given to us, timeout = None
socket_pipe.poll(None)
# now we grab the socket
s = socket.fromfd(reduction.recv_handle(socket_pipe), socket.AF_INET,
socket.SOCK_STREAM)
print "output_buffer_process: ready"
print "startup complete"
# Now poll the incoming queue and send data as it comes in
while True:
data = output_queue.get()
s.send(data.encode("utf-8") + u"\r\n")
# we pause to prevent flooding the IRC server
sleep(RATE)
def on_payload(self, request):
if isinstance(request, tuple):
try:
cmd = request[0]
if cmd == CMD_CLOUD_CONNECTION:
endpoint = request[1]
token = request[2]
logger.debug("Recive connect2cloud request, endpoint=%s",
endpoint)
ret = self.kernel.on_connect2cloud(endpoint, token)
self.send_payload(("ok", ret))
self.close()
elif cmd == CMD_USBCABEL_CONNECTION:
try:
self.send(b"F")
rl = select((self.sock, ), (), (), 0.1)[0]
if rl:
fd = recv_handle(self.sock)
usbsock = socket.fromfd(fd, socket.AF_UNIX,
socket.SOCK_STREAM)
self.kernel.on_connect2usb(usbsock)
self.send(b"X")
self.close()
else:
self.send(b"I")
self.close()
except OSError as e:
logger.warning("Pass robot socket handle error: %s", e)
else:
self.send_payload(("er", "UNKNOWN_COMMAND"))
except RuntimeError as e:
self.send_payload(("er", ) + e.args)
except Exception:
logger.exception("Error in internal socket")
self.send_payload(("er", "UNKNOWN_ERROR"))
self.close()
else:
self.send_payload(("er", "UNKNOWN_COMMAND"))
self.close()
def run(self) -> None:
self.selector = selectors.DefaultSelector()
fileno = recv_handle(self.work_queue)
self.work_queue.close()
self.sock = socket.fromfd(fileno,
family=self.flags.family,
type=socket.SOCK_STREAM)
try:
self.selector.register(self.sock, selectors.EVENT_READ)
if self.flags.threadless:
self.start_threadless_process()
while not self.running.is_set():
self.run_once()
except KeyboardInterrupt:
pass
finally:
self.selector.unregister(self.sock)
if self.flags.threadless:
self.shutdown_threadless_process()
self.sock.close()
logger.debug('Acceptor#%d shutdown', self.idd)
