def start_work(self, conn: socket.socket, addr: Tuple[str, int]) -> None:
if self.flags.threadless and \
self.threadless_client_queue and \
self.threadless_process:
self.threadless_client_queue.send(addr)
send_handle(
self.threadless_client_queue,
conn.fileno(),
self.threadless_process.pid
)
conn.close()
else:
work = self.work_klass(
TcpClientConnection(conn, addr),
flags=self.flags,
event_queue=self.event_queue
)
work_thread = threading.Thread(target=work.run)
work_thread.daemon = True
work.publish_event(
event_name=eventNames.WORK_STARTED,
event_payload={'fileno': conn.fileno(), 'addr': addr},
publisher_id=self.__class__.__name__
)
work_thread.start()
def create_file_event(self, fd: socket.socket, mask: int, handler,
privdata):
'''
将给定fd的给定事件添加到events中
添加到events的FileEvent结构由mask, handler, privdata共同决定
如果已存在， 则更新FileEvent
:param fd: 描述符(socket)
:param mask: 表示读写的标志， 根据此更新FileEvent的可读写状态
:param handler: 事件发生时的回调函数
:param privdata: handler需要的数据， 保存在FileEvent中
:return:
'''
# logging.info("new event, fd: {}, mask: {}".format(fd.fileno(), mask))
fe = self.events.setdefault(fd.fileno(), FileEvent())
self.event_add(fd, mask)
fe.mask |= mask
if mask & READABLE:
fe.readhandler = handler
if mask & WRITEABLE:
fe.writehandler = handler
fe.client = privdata
fe.fd = fd
if fd.fileno() > self.maxfd:
self.maxfd = fd.fileno()
def read(sock: socket.socket):
f = Future()
def on_readable():
f.set_result(sock.recv(4096))
selector.register(sock.fileno(), EVENT_READ, on_readable)
chunk = yield f # 读一个chunk
selector.unregister(sock.fileno())
return chunk
async def read(sock: socket.socket) -> bytes:
'''从 socket 中读取bytes
'''
f = Future()
def on_readable():
f.set_result(sock.recv(4096))
selector.register(sock.fileno(), EVENT_READ, on_readable)
chunck = await f
# 移除监听
selector.unregister(sock.fileno())
return chunck
def start_threaded_work(
flags: argparse.Namespace,
conn: socket.socket,
addr: Optional[Tuple[str, int]],
event_queue: Optional[EventQueue] = None,
publisher_id: Optional[str] = None,
) -> Tuple['Work[T]', threading.Thread]:
"""Utility method to start a work in a new thread."""
work = flags.work_klass(
flags.work_klass.create(conn=conn, addr=addr),
flags=flags,
event_queue=event_queue,
upstream_conn_pool=None,
)
# TODO: Keep reference to threads and join during shutdown.
# This will ensure connections are not abruptly closed on shutdown
# for threaded execution mode.
thread = threading.Thread(target=work.run)
thread.daemon = True
thread.start()
work.publish_event(
event_name=eventNames.WORK_STARTED,
event_payload={
'fileno': conn.fileno(),
'addr': addr
},
publisher_id=publisher_id or 'thread#{0}'.format(thread.ident, ),
)
return (work, thread)
def client_handler(client: socket.socket):
logic.menu(client)
while True:
msg = tcp_server.recv_msg(client)
if client is None:
return
if msg is not None:
msg = msg.split()
# Validate Command
if logic.validate_command(msg):
tcp_server.send_msg(client, logic.error_msg_prefix + "Invalid command")
continue
# Validate Registry
if logic.validate_registration(client, msg) and 'exit' not in msg:
tcp_server.send_msg(client, logic.error_msg_prefix + "Please register first. \n")
continue
# Then execute game action
try:
threading.Thread(target=logic.commands_handler.get(msg[0].lower()), args=(client, msg,)).start()
except:
if client is None or client.fileno() == -1:
break
tcp_server.send_msg(client, "There was a problem procesing your request\n")
def accept_handler(serversocket: socket.socket) -> None:
clientsocket, (client_address, client_port) = serversocket.accept()
clientsocket.setblocking(False)
logging.debug(f"New client: {client_address}:{client_port}")
connections[clientsocket.fileno()] = (clientsocket, client_address, client_port)
read_waiters[clientsocket.fileno()] = (recv_handler, (clientsocket.fileno(),))
read_waiters[serversocket.fileno()] = (accept_handler, (serversocket,))
async def send(self, client_socket: socket):
loop = asyncio.get_event_loop()
raw_utf = f'{self._protocol} {self._status} {STATUS_MESSAGES[self._status]}\r\n' + \
'\r\n'.join([f'{key}: {value}' for key, value in self._headers.items()]) + '\r\n\r\n'
raw_bytes = raw_utf.encode()
await loop.sock_sendall(client_socket, raw_bytes)
if self._filepath is not None and self._method == 'GET':
with open(self._filepath, 'rb') as file:
if Config.sendfile:
try:
await loop.run_in_executor(None, os.sendfile,
client_socket.fileno(),
file.fileno(), 0,
getsize(self._filepath))
except (BrokenPipeError, ConnectionResetError) as e:
logging.warning(e)
return
else:
part = file.read(Config.bytes_per_send)
while len(part) > 0:
try:
await loop.sock_sendall(client_socket, part)
except (BrokenPipeError, ConnectionResetError) as e:
logging.warning(e)
return
part = file.read(Config.bytes_per_send)
def del_client(self, conn: socket.socket):
addr = conn.getpeername()
if addr not in self.clients.keys():
raise ClientNotExistException()
self.server.remove_handler(conn.fileno())
del self.clients[addr]
async def connect(sock: socket.socket, address: Tuple) -> None:
f = Future()
# 将socket 设置为非阻塞模式
sock.setblocking(False)
try:
sock.connect(address)
except BlockingIOError:
pass
def on_connected():
f.set_result(None)
selector.register(sock.fileno(), EVENT_WRITE, on_connected)
await f
# 移除监听
selector.unregister(sock.fileno())
def io_transfer(iosock: socket.socket, flag, ifreq):
"""
send & recieve an ifreq struct
:param iosock: io socket
:param flag: sockios control call
:param ifreq: ifreq to send
:returns: an the ifreq struct recieved
"""
try:
return ioctl(iosock.fileno(), flag, ifreq)
except (AttributeError, struct.error) as e:
# either sock is not valid or a bad value passed to ifreq
if e.message.find("fileno"):
raise EnvironmentError(errno.ENOTSOCK, "Bad socket")
else:
raise EnvironmentError(errno.EINVAL, e)
except IOError as e:
# generally device cannot be found sort but can also be
# permissions etc, catch and reraise as our own
if e.errno is not None: # just in case we have a none 2-tuple error
raise EnvironmentError(e.errno, e.strerror)
else:
raise EnvironmentError(-1, "Undefined error")
except Exception as e:
# blanket catchall
raise EnvironmentError(-1, e.args[0])
def unregister_sock(self, sock: socket) -> None:
"""
Unregister the given sock with the polling object, and internal dict.
"""
fileno = sock.fileno()
self.poller.unregister(fileno)
del self.socket_filenos[fileno]
def _work(self, conn: socket.socket, addr: Optional[Tuple[str,
int]]) -> None:
self._total = self._total or 0
if self.flags.threadless:
# Index of worker to which this work should be dispatched
# Use round-robin strategy by default.
#
# By default all acceptors will start sending work to
# 1st workers. To randomize, we offset index by idd.
index = (self._total + self.idd) % self.flags.num_workers
thread = threading.Thread(
target=delegate_work_to_pool,
args=(
self.executor_pids[index],
self.executor_queues[index],
self.executor_locks[index],
conn,
addr,
self.flags.unix_socket_path,
),
)
thread.start()
# TODO: Move me into target method
logger.debug( # pragma: no cover
'Dispatched work#{0}.{1}.{2} to worker#{3}'.format(
conn.fileno(),
self.idd,
self._total,
index,
), )
else:
_, thread = start_threaded_work(
self.flags,
conn,
addr,
event_queue=self.event_queue,
publisher_id=self.__class__.__name__,
)
# TODO: Move me into target method
logger.debug( # pragma: no cover
'Started work#{0}.{1}.{2} in thread#{3}'.format(
conn.fileno(),
self.idd,
self._total,
thread.ident,
), )
self._total += 1
def __init__(self, sock: socket, loop: BaseLoop = None, app=None):
self._socket = sock
self.fd = sock.fileno()
self.loop = loop
self.app = app
self.request: Request = None
self.response: Response = None
self._socket.setblocking(False)
def _on_socket_close(self, client: mqtt.Client, userdata: Any,
sock: socket.socket) -> None:
fileno = sock.fileno()
if fileno > -1:
self._loop.remove_reader(fileno)
if self._misc_task is not None and not self._misc_task.done():
self._loop.call_soon_threadsafe(self._misc_task.cancel)
async def connect(protocol_factory: Callable[..., ConnectionBase],
address: str = None,
port: int = None,
family=AF_INET,
type=SOCK_STREAM,
proto=0,
fileno=None,
bufsize: int = 1024,
sock: socket = None):
if sock is not None:
if fileno is not None:
raise ValueError("You cannot specify a fileno AND a socket!")
try:
sock.getpeername()
connected = False
# We want to check if the sock is connected already
except OSError: # It'll raise an OSError if we try to getpeername of an unconnected sock
if address is not None or port is not None:
raise ValueError(
"You cannot specify both an address/port AND a connected socket!"
) from None
connected = True
else:
sock = socket(family=family, type=type, proto=proto, fileno=fileno)
connected = False
connection = protocol_factory(
socket=sock, host=(address, port),
bufsize=bufsize) # All protos need these args
if not connected:
await connection._connect(
) # If the sock isn't connected, connect "asynchronously"
loop = await get_loop()
if not isinstance(loop, IOCPLoop):
await create_writer(sock, connection._writer_callback
) # Create our reader and writer
else:
if sock.fileno() not in loop._open_ports:
_overlapped.CreateIoCompletionPort(sock.fileno(), loop._port, 0, 0)
loop._open_ports.append(sock.fileno())
await create_reader(sock, connection._reader_callback)
return connection
def __read_socket(self, s: socket.socket):
"""
Handles sockets that has data to be read/incomming connections
"""
if s is self.__socket: # If socket is servers own socket, we have a new client connecting
connection, addr = s.accept() # Accept client
connection.setblocking(0) # Asynchronous input
# Store info about client
self.__sockets_read.append(connection)
self.__response_queues[connection] = queue.Queue()
self.on_client_connected(addr)
else: # A clients socket has data to be read
header = self.__header_buffer.get(
s.getpeername(), None
) # Has this client sent data before? Get the previous data (header) sent
n = header.size if header is not None else PacketHeader.SIZE # Get the expected size to read (as stated in the header) or (if the header doesnt exist, which means this is the first data from that client to be received) set the size to header size
avail = self._available_bytes(
s.fileno()) # Get size of the data awaiting to be read
if avail < n: # Available data size is not (yet) sufficient
if avail <= 0: # If we have data request with no size, the client has disconnected
self.on_client_disconnected(s.getpeername())
self.__close_socket(s)
return # If size of the data is less than expected, wait for more in the next call
data = s.recv(n) if n > 0 else None # Receive the acctual data
if header is None: # If we don't have previous data form this client
if data is None: # This is not expected
return
header = PacketHeader._make(
struct.unpack(PacketHeader.FORMAT,
data)) # Unpack header data
self.__header_buffer[s.getpeername(
)] = header # Associente this header to client
if header.size > 0: # If expecting more data receive it in the next call
return
# Here we are guaranteed to have a header, but not necessarily any data after that (i.e: header != None and (data == None or data != None))
if self.on_data_received(header, data): # Call callback
response = struct.pack(PacketHeader.FORMAT, int(PacketID.TRUE),
0) # Success packet to be sent back
else:
response = struct.pack(PacketHeader.FORMAT,
int(PacketID.FALSE),
0) # Fail packet to be sent back
self.__response_queues[s].put(response) # Queue response packet
if s not in self.__sockets_write:
self.__sockets_write.append(
s) # Append cliet socket to write queue
del self.__header_buffer[s.getpeername(
)] # Remove associated header from client as it's already processed
def hci_le_set_scan_enable(sock: socket.socket, enable: bool,
filter_duplicates: bool, timeout: int) -> None:
"""
Enable or disable LE scanning.
"""
err = BLUEZ.hci_le_set_scan_enable(sock.fileno(), enable,
filter_duplicates, timeout)
if err < 0:
raise BluezError("hci_le_set_scan_enable")
def add_client(self, conn: socket.socket):
addr = conn.getpeername()
if addr in self.clients.keys():
raise ClientExistException()
client = Client(self, conn)
self.server.add_handler(conn.fileno(), client.recv, EVENT_READ)
self.clients[addr] = client
def __init__(self,
loop: asyncio.BaseEventLoop,
rawsock: socket.socket,
protocol: asyncio.Protocol,
ssl_context_factory: SSLContextFactory,
waiter: typing.Optional[asyncio.Future] = None,
use_starttls: bool = False,
post_handshake_callback: typing.
Optional[PostHandshakeCallback] = None,
peer_hostname: typing.Optional[str] = None,
server_hostname: typing.Optional[str] = None):
if not use_starttls and not ssl_context_factory:
raise ValueError("Cannot have STARTTLS disabled (i.e. immediate "
"TLS connection) and without SSL context.")
super().__init__()
self._rawsock = rawsock
self._raw_fd = rawsock.fileno()
self._trace_logger = logger.getChild("trace.fd={}".format(
self._raw_fd))
self._sock = rawsock # type: typing.Union[socket.socket, OpenSSL.SSL.Connection] # noqa
self._send_wrap = SendWrap(self._sock)
self._protocol = protocol
self._loop = loop
self._extra = {
"socket": rawsock,
} # type: typing.Dict[str, typing.Any]
self._waiter = waiter
self._conn_lost = 0
self._buffer = bytearray()
self._ssl_context_factory = ssl_context_factory
self._extra.update(sslcontext=None,
ssl_object=None,
peername=self._rawsock.getpeername(),
peer_hostname=peer_hostname,
server_hostname=server_hostname)
# this is a list set of tasks which will also be cancelled if the
# _waiter is cancelled
self._chained_pending = set() # type: typing.Set[asyncio.Future]
self._paused = False
self._closing = False
self._tls_conn = None # type: typing.Optional[OpenSSL.SSL.Connection]
self._tls_read_wants_write = False
self._tls_write_wants_read = False
self._tls_post_handshake_callback = post_handshake_callback
self._state = None # type: typing.Optional[_State]
if not use_starttls:
self._ssl_context = ssl_context_factory(self)
self._extra.update(sslcontext=self._ssl_context, )
self._initiate_tls()
else:
self._initiate_raw()
def _netmask_for_ifacename(name: str, sock: socket.socket) -> Optional[str]:
siocgifnetmask = 0x891b
bytebuf = struct.pack('256s', name.encode('utf-8'))
try:
ret = fcntl.ioctl(sock.fileno(), siocgifnetmask, bytebuf)
except OSError:
logging.error('siocgifnetmask failed')
return None
return socket.inet_ntoa(ret[20:24])
def __init__(self, clientsocket: socket.socket):
super().__init__(fileno=clientsocket.fileno())
self.session_count = ClientSession.session_count + 1
ClientSession.session_count += 1
self.write_buffer = bytearray()
self.read_buffer = bytearray()
logging.info(
f"{self.getpeername()} connected as client #{self.session_count}")
def create_client(self, sock: socket.socket):
if self.client_exists(sock):
return
client = Client(sock)
self.clients[sock.fileno()] = client
self.sockets[sock.fileno()] = sock
self.poller.register(sock, select.POLLIN)
print(str(client), "connected.")
def handler(self, cli_sock: socket.socket, cli_addr):
client_name = 'fd={}'.format(cli_sock.fileno())
LOG.info(f'client {client_name} enter backdoor!')
handler = BackdoorHandler(self, cli_sock, cli_addr)
_BACKDOOR_HANDLER.handler = handler
try:
handler.handle()
finally:
LOG.info(f'client {client_name} exit backdoor!')
handler.close()
_BACKDOOR_HANDLER.handler = None
def _launch_repo_server(
*,
repo_server_bin: Path,
sock: socket.socket,
snapshot_dir: Path,
debug: bool,
):
'''
Invokes `repo-server` with the given snapshot; passes it ownership of
the bound TCP socket -- it listens & accepts connections.
'''
# This could be a thread, but it's probably not worth the risks
# involved in mixing threads & subprocess (yes, lots of programs do,
# but yes, far fewer do it safely).
with sock, subprocess.Popen([
repo_server_bin,
'--socket-fd', str(sock.fileno()),
'--snapshot-dir', snapshot_dir,
*(['--debug'] if debug else []),
], pass_fds=[sock.fileno()]) as server_proc:
try:
log.info('Waiting for repo server to listen')
while server_proc.poll() is None:
if sock.getsockopt(socket.SOL_SOCKET, socket.SO_ACCEPTCONN):
break
time.sleep(0.1)
yield
finally:
# Although `repo-server` is a read-only proxy, give it the
# chance to do graceful cleanup.
log.info('Trying to gracefully terminate `repo-server`')
# `atexit` (used in an FB-specific `repo-server` plugin) only
# works with SIGINT. We signal once, and need to wait for it to
# clean up the resources it must to free. Signaling twice would
# interrupt cleanup (because this is Python, lol).
server_proc.send_signal(signal.SIGINT) # `atexit` needs this
try:
server_proc.wait(60.0)
except subprocess.TimeoutExpired: # pragma: no cover
log.info('Killing unresponsive `repo-server`')
server_proc.kill()
def __init__(self): global server_socket, serverpid server_socket = Socket(AF_INET, SOCK_STREAM) server_socket.setblocking(0) try: server_socket.setsockopt(SOL_SOCKET, SO_REUSEADDR, server_socket.getsockopt(SOL_SOCKET, SO_REUSEADDR) | 1) except error: pass serverpid = server_socket.fileno() pollster.register(serverpid, RE)
def _repo_server(sock: socket.socket, storage_cfg: str, snapshot_dir: Path):
'''
Invokes `repo-server` with the given storage & snapshot; passes it
ownership of the bound TCP socket -- it listens & accepts connections.
'''
# This could be a thread, but it's probably not worth the risks
# involved in mixing threads & subprocess (yes, lots of programs do,
# but yes, far fewer do it safely).
with sock, subprocess.Popen([
os.path.join(os.path.dirname(__file__), 'repo-server'),
'--socket-fd',
str(sock.fileno()),
'--storage',
storage_cfg,
'--snapshot-dir',
snapshot_dir,
],
pass_fds=[sock.fileno()]) as server_proc:
try:
yield server_proc
finally:
server_proc.kill() # It's a read-only proxy, abort ASAP
def read(conn: socket.socket, mask: int):
with conn:
s_file = fdopen(conn.fileno(),
"rb",
buffering=0)
try:
obj = pickle.Unpickler(s_file).load()
# False obj means skip
if obj:
self.tool_calls.append(obj)
formatted_out(f_out, obj)
finally:
sel.unregister(conn)
async def send(self, sock: socket.socket):
await asyncio.get_event_loop().sock_sendall(sock, self.head_to_string().encode(C.ENCODING))
# send body
if self.status == C.HTTP_STATUS_CODE_OK and self.path and self.method == C.METHOD_GET:
with open(self.path, 'rb') as file:
# Using lib sendfile
offset = 0
blocksize = os.path.getsize(self.path)
while True:
sent = sendfile.sendfile(sock.fileno(), file.fileno(), offset, blocksize)
offset += sent
if sent == 0:
break
def hci_le_set_scan_parameters(sock: socket.socket, scan_type: ScanType,
scan_interval: int, scan_window: int,
address_type: AddressType,
filter_policy: FilterPolicy,
timeout: int) -> None:
"""
Set LE scanning parameters.
"""
err = BLUEZ.hci_le_set_scan_parameters(sock.fileno(), scan_type.value,
scan_interval, scan_window,
address_type.value,
filter_policy.value, timeout)
if err < 0:
raise BluezError("hci_le_set_scan_parameters")
def __init__(self, sock: socket.socket, host: bytes, port: int, loop: LoopBase = None) -> None:
EventSource.__init__(self, loop)
self.socket = sock
self.host = host
self.port = port
self.tcp_connected = True # we assume a connected socket
self._input_paused = True # we start with input paused
self._output_paused = False
self._closing = False
self._write_buffer = [] # type: list[bytes]
self.register_fd(sock.fileno())
self.on("fd_readable", self.handle_readable)
self.on("fd_writable", self.handle_writable)
self.on("fd_close", self._handle_close)
def add_socket(self, sock: socket.socket):
self.other_sockets[sock.fileno()] = sock
self.poll.register(sock.fileno(), select.EPOLLIN)
def remove_socket(self, sock: socket.socket):
del self.other_sockets[sock.fileno()]
self.poll.unregister(sock.fileno())
