def selector():
sel = DefaultSelector()
sel.register(proc.stdout, EVENT_READ, 0)
sel.register(proc.stderr, EVENT_READ, 1)
while True:
for key, mask in sel.select():
yield key.data, decode(key.fileobj.readline(linesize))
def redirect(fd, callback):
save_fd = os.dup(fd)
s_in, s_out = socket.socketpair()
os.dup2(s_in.fileno(), fd)
os.close(s_in.fileno())
s_out.setblocking(False)
sel = DefaultSelector()
sel.register(s_out, EVENT_READ)
def target():
while running:
_, = sel.select() # There's only one event.
callback(s_out.recv(4096))
running = True
thread = Thread(target=target, daemon=True)
thread.start()
try:
yield
finally:
running = False
os.dup2(save_fd, fd)
os.close(save_fd)
def selector():
sel = DefaultSelector()
sel.register(proc.stdout, EVENT_READ, 0)
sel.register(proc.stderr, EVENT_READ, 1)
while True:
ready = sel.select(line_timeout)
if not ready and line_timeout:
raise ProcessLineTimedOut("popen line timeout expired", getattr(proc, "argv", None), getattr(proc, "machine", None))
for key, mask in ready:
yield key.data, decode(key.fileobj.readline(linesize))
class Loop:
def __init__(self):
self.ready = deque()
self.selector = DefaultSelector()
self.futures = {}
def create_task(self, task):
self.ready.append(task)
def run_forever(self):
while True:
while not self.ready:
completed_futures = [future for future in self.futures if not future.running()]
for future in completed_futures:
self.ready.append(self.futures.pop(future))
# so select() is blocking. If set a negative time out it won't block.
events = self.selector.select(-1)
# add these socket events and unregister them from listened to:
for key, _ in events:
self.ready.append(key.data) # add the task to the ready queue
self.selector.unregister(key.fileobj)
while self.ready:
self.current_task = self.ready.popleft()
# try to run current_task...
try:
# run task to next yield point
reason, what = self.current_task.send(None)
# reason, what = self.current_task.send(None)
if reason == 'waiting_to_accept':
self.selector.register(what, EVENT_READ, self.current_task)
elif reason == 'waiting_to_read':
self.selector.register(what, EVENT_READ, self.current_task)
elif reason == 'waiting_to_write':
self.selector.register(what, EVENT_WRITE, self.current_task)
elif reason == 'waiting_for_future':
self.futures[what] = self.current_task
else:
raise RuntimeError(
'Something bad happened... er. reason={}'.format(reason))
except StopIteration:
pass
async def sock_recv(self, sock, maxbytes):
# wait to read from the socket
await read_wait(sock)
return sock.recv(maxbytes)
async def sock_accept(self, sock):
# wait to read/hear from the socket
await accept_wait(sock)
return sock.accept()
async def sock_sendall(self, sock, data):
while data:
# wait to be able to write to the socket
await write_wait(sock)
nsent = sock.send(data)
data = data[nsent:]
def run_in_executor(self, executor, func, *args):
return executor.submit(func, *args)
class Loop:
def __init__(self):
self.sel = DefaultSelector()
self.queue = deque()
def create_task(self, coro):
sock = coro.send(None)
self.sel.register(sock, EVENT_WRITE, coro)
self.queue.append(coro)
def run(self):
while self.queue:
events = self.sel.select()
for key, _ in events:
coro = key.data
event = key.events
file_obj = key.fileobj
self.sel.unregister(file_obj)
try:
if event == EVENT_WRITE:
self.sel.register(file_obj, EVENT_READ, coro)
elif event == EVENT_READ:
self.sel.register(file_obj, EVENT_WRITE, coro)
coro.send(None)
except StopIteration:
self.queue.popleft()
except:
pass
else: return
class IRC(Service):
DEFAULT_PORT = 6667
TIMEOUT = 3
def __init__(self,events):
super().__init__(events,'irc')
self.servers = set()
self.selector = DefaultSelector()
self.events = events
self.has_servers = Event()
self.channels = {}#server:[channels]
def respond(self,addr,command):
if command['action']=='connect':
server = command['host']
nick = command['nick']
port = command.get('port',self.DEFAULT_PORT)
self.connect((server,port),nick)
if command['action']=='status':
print(self.servers)
self.broadcast({
'kind':'status',
'status':{
'servers':{
s: {
'channels':v
} for s,v in self.channels.items()
}
}
})
def connect(self,addr,nick):
host = addr[0]
if host not in self.servers:
print(nick)
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
print(addr)
sock.connect(addr)
self.events.listen(host,self.check_channels)
self.selector.register(sock,EVENT_READ|EVENT_WRITE,ChatServerInterface(addr[0],nick,self.events))
self.servers.add(host)
self.channels[host] = list()
if not self.has_servers.is_set():
self.has_servers.set()
def check_channels(self,addr,event):
if event['kind']=='irc':
if event['command']=='332':
print(event['event'],event['arguments'])
self.channels[event['event']].append(event['arguments'][1].lower())
def loop(self):
while True:
self.has_servers.wait()
for key,mask in self.selector.select():
if mask&EVENT_READ==EVENT_READ:
key.data.read_data(key.fileobj)
if mask&EVENT_WRITE==EVENT_WRITE:
key.data.write_data(key.fileobj)
def get(path, jobs):
jobs += 1
selector = DefaultSelector()
client = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
client.setblocking(False)
try:
client.connect(('localhost', 5000))
except BlockingIOError:
pass
f = Future()
selector.register(client.fileno(), EVENT_READ, f)
def loop():
pre_loop_barrier.wait(1)
selector = DefaultSelector()
selector.register(sig_r_fd, EVENT_READ)
with suppress(StopIteration):
while True:
events = selector.select(timeout=5)
if len(events) == 0:
raise StopIteration
for key, mask in events:
if key.fd == sig_r_fd:
data = os.read(sig_r_fd, 4)
self._sigchld_caught = True
raise StopIteration
self._post_loop_barrier.wait(1)
def __init__(self):
"""
Конструктор событийного цикла, который хранит очередь
:return:
"""
self.ready = deque()
self.selector = DefaultSelector()
def __init__(self,events):
super().__init__(events,'irc')
self.servers = set()
self.selector = DefaultSelector()
self.events = events
self.has_servers = Event()
self.channels = {}#server:[channels]
def __init__(self, port):
self.port = port
self.sock = None
self.selector = DefaultSelector()
# Advance self.listen() method. A future has been attached to the
# server socket, that has been registered in selector. And a callback
# has been attached to this future.
Task(self.listen())
def __init__(self, port, callback):
logging.basicConfig(level=logging.INFO)
self.clients_kept_for_sending = []
self.selector = DefaultSelector()
self.callback = callback
self.server_socket = socket.socket(family=socket.AF_INET6,
type=socket.SOCK_STREAM)
self.server_socket.bind(("::0", port))
self.server_socket.listen(5)
self.selector.register(self.server_socket, EVENT_READ, data=None)
self.server_socket.setblocking(False)
class Loop:
def __init__(self):
self.ready = deque()
self.selectors = DefaultSelector() # a way to watch socket IO
async def sock_recv(self, sock, maxbytes):
await read_wait(sock) # wait for something happen
return sock.recv(maxbytes)
async def sock_accept(self, sock):
await read_wait(sock)
return sock.sock_accept()
async def sock_sendall(self, sock, data):
while data :
try:
nsent = sock.send(data)
data = data[nsent:] # send partial data
except BlockingIOError:
await write_wait(sock)
def create_task(self, coro):
self.ready.append(coro)
def run_forever(self):
while True:
while not self.ready:
events = self.selectors.select()
for key, _ in events:
self.ready.append(key.data)
self.selectors.unregister(key.fileobj)
while self.ready:
self.current_task = self.ready.popleft()
try:
op, *args = self.current_task.send(None) # run to the yield
getattr(self,op)(*args) # Sneaky method call
except StopIteration:
pass
def read_wait(self, sock):
self.selectors.register(sock, EVENT_READ, self.current_task)
def write_wait(self, sock):
self.selectors.register
class Loop:
def __init__(self):
self.ready = deque()
self.selector = DefaultSelector()
def create_task(self, task):
self.ready.append(task)
def run_forever(self):
while True:
# hmmn, nothing to run -> must be waiting on stuff...
while not self.ready:
events = self.selector.select()
# add these events and unregister them from listened to:
for key, _ in events:
self.ready.append(key.data)
self.selector.unregister(key.fileobj)
while self.ready:
self.current_task = self.ready.popleft()
# try to run current_task...
try:
# run task to next yield point
reason, sock = next(self.current_task)
if reason == 'waiting_to_accept':
self.selector.register(sock, EVENT_READ, self.current_task)
elif reason == 'waiting_to_read':
self.selector.register(sock, EVENT_READ, self.current_task)
elif reason == 'waiting_to_write':
self.selector.register(sock, EVENT_WRITE, self.current_task)
else:
raise RuntimeError('Something bad happened... er. reason={}'.format(reason))
except StopIteration:
pass
def sock_recv(self, sock, maxbytes):
# wait to read from the socket
return sock.recv(maxbytes)
def sock_accept(self, sock):
# wait to read/hear from the socket
return sock.accept()
def sock_sendall(self, sock, data):
while data:
# wait to be able to write to the socket
nsent = sock.send(data)
data = data[nsent:]
def __init__(self, fb):
assert isinstance(fb, Framebuffer)
self._fb = fb
# get all input devices
devices = [InputDevice(fn) for fn in evdev.list_devices()]
# filter out non key devices
for device in devices.copy():
cap = device.capabilities()
if ecodes.EV_KEY not in cap:
devices.remove(device)
continue
self._selector = DefaultSelector()
# This works because InputDevice has a `fileno()` method.
for device in devices:
self._selector.register(device, EVENT_READ)
def __init__(self):
self.selector = DefaultSelector()
self.init_hardware()
self.mouseevents = []
self.prev_value = 1
self.keynames = {
127: "menu",
28: "ok",
103: "up",
108: "down",
105: "left",
106: "right",
1: "back",
172: "home",
114: "volumedown",
115: "volumeup",
116: "power",
272: "btn_left"}
class Loop:
def __init__(self):
self.ready = deque()
self.selector = DefaultSelector()
async def sock_recv(self, sock, maxbytes):
await read_wait(sock)
return sock.recv(maxbytes)
async def sock_accept(self, sock):
await read_wait(sock)
return sock.accept()
async def sock_sendall(self, sock, data):
while data:
try:
nsent = sock.send(data)
data = data[nsent:]
except BlockingIOError:
await write_wait(sock)
def create_task(self, coro):
self.ready.append(coro)
def run_forever(self):
while True:
while not self.ready:
events = self.selector.select()
for key, _ in events:
self.ready.append(key.data)
self.selector.unregister(key.fileobj)
while self.ready:
self.current_task = self.ready.popleft()
try:
op, *args = self.current_task.send(None)
getattr(self, op)(*args)
except StopIteration:
pass
def read_wait(self, sock):
self.selector.register(sock, EVENT_READ, self.current_task)
def write_wait(self, sock):
self.selector.register(sock, EVENT_WRITE, self.current_task)
class Connection(LibConnection):
# Reference to the handling app on the side of the server
TYPE_HANDLER = 'client-handler'
LL_EVENT_ACCEPT = 'accept'
LL_EVENT_READ = 'read'
LL_EVENT_WRITE = 'write'
LL_EVENT_DISCONNECTED = 'disconnected'
_sel: DefaultSelector = None
_perform_select: bool = True
_subscribed_events: dict = None
_t_selecting_events: Thread = None
_t_read: Thread = None
_t_write: Thread = None
_read_lock: Lock = None
_write_lock: Lock = None
def __init__(self, *args, **kwargs):
super(Connection, self).__init__(*args, **kwargs)
self._subscribed_events = {
self.LL_EVENT_ACCEPT: list(),
self.LL_EVENT_READ: list(),
self.LL_EVENT_WRITE: list(),
self.LL_EVENT_DISCONNECTED: list(),
}
def unsubscribe_from_event(self, event_name: str, cb: callable) -> bool:
if event_name not in self._subscribed_events:
return False
if cb in self._subscribed_events[event_name]:
self._subscribed_events[event_name].remove(cb)
return True
def subscribe_to_event(self, event_name: str, cb: callable) -> bool:
"""
Subscribing to low-level communication events (accept, read, write).
:param event_name: One of: 'accept', 'read' or 'write'
:param cb: Callback that should be used for the subscription
:return: Returns True in case of successful subscription or False otherwise (Basically indication if subscribed)
"""
if event_name not in self._subscribed_events:
return False
if cb not in self._subscribed_events[event_name]:
self._subscribed_events[event_name].append(cb)
# Already subscribed
return True
def _after_open_connection(self):
super(Connection, self)._after_open_connection()
self._register_events()
self._t_selecting_events = Thread(target=self._selecting_events,
daemon=True)
def ready_to_operate(self):
"""
This method must be explicitly invoked after reconnection.
Otherwise the reading/writing to socket will work incorrectly
:return:
"""
self._t_selecting_events.start()
def _reading(self):
self._read_lock.acquire()
while self._is_connected:
# print('R')
self._read_lock.acquire()
self._invoke_events_callbacks(self.LL_EVENT_READ)
if self._read_lock.locked():
self._read_lock.release()
def _writing(self):
self._write_lock.acquire()
# print('[W2]', self._is_connected)
while self._is_connected:
# print('[W]')
self._write_lock.acquire()
self._invoke_events_callbacks(self.LL_EVENT_WRITE)
if self._write_lock.locked():
self._write_lock.release()
def _register_events(self):
self._sel = DefaultSelector()
if self._type == self.TYPE_SERVER:
self._sel.register(self._socket, EVENT_READ, self._event_accept)
if self._type in (self.TYPE_HANDLER, self.TYPE_CLIENT):
self._sel.register(self._socket, EVENT_READ | EVENT_WRITE,
self._event_communicate)
self._read_lock = Lock(
) if self._read_lock is None else self._read_lock
self._write_lock = Lock(
) if self._write_lock is None else self._write_lock
self._t_read = Thread(target=self._reading, daemon=True)
self._t_read.start()
self._t_write = Thread(target=self._writing, daemon=True)
self._t_write.start()
def _disconnected(self):
print('Disconnected happened')
self._unregister_events()
def _unregister_events(self):
# self._sel = DefaultSelector()
if self._perform_select:
# print('Unregistering stuff')
self._perform_select = False
self._sel.unregister(self._socket)
if self._read_lock and self._read_lock.locked():
self._read_lock.release()
if self._write_lock and self._write_lock.locked():
self._write_lock.release()
if self._t_read and self._t_read.is_alive():
del self._t_read
if self._t_write and self._t_write.is_alive():
del self._t_write
def _selecting_events(self):
while self._perform_select:
for key, mask in self._sel.select():
key.data(mask)
def _invoke_events_callbacks(self, event_name, *args, **kwargs):
# print(f'LL EVENT Invoked: {event_name}')
if self._subscribed_events[event_name]:
# print('cb: ', self._subscribed_events, event_name)
for cb in self._subscribed_events[event_name]:
cb(*args, **kwargs)
else:
if event_name == self.LL_EVENT_WRITE:
# If there is no write subscriptions, this code
# would decrease unnecessary CPU usage by just a simple timeout.
# Should not be considered as a hack, though if you have
# a better solution - please propose.
# print('No write subscription')
sleep(10)
def _event_accept(self, mask):
if mask & EVENT_READ:
new_connection = self.accept(self.TYPE_HANDLER)
self._invoke_events_callbacks(self.LL_EVENT_ACCEPT, new_connection)
def _event_communicate(self, mask):
if mask & EVENT_READ:
if self._read_lock is not None and self._read_lock.locked():
# print('MREAD')
self._read_lock.release()
# Thread(target=self._invoke_events_callbacks, args=(self.LL_EVENT_READ, ), daemon=True).start()
if mask & EVENT_WRITE:
if self._write_lock is not None and self._write_lock.locked():
# print('MWRITE')
self._write_lock.release()
# self._invoke_events_callbacks(self.LL_EVENT_WRITE)
def collect_frame(self):
frame = None
try:
frame = self._gather_frame_from_socket(SimpleFrame)
if frame is None:
self._invoke_events_callbacks(self.LL_EVENT_DISCONNECTED)
except SSLWantReadError as e:
# todo move to ll
pass
if isinstance(frame, SimpleFrame):
return frame
return False
import socket
from selectors import DefaultSelector, EVENT_WRITE
selector = DefaultSelector()
sock = socket.socket()
sock.setblocking(False)
sock2 = socket.socket()
sock2.setblocking(False)
try:
sock.connect(('xkcd.com', 80))
except BlockingIOError:
pass
try:
sock2.connect(('xkcd.com', 80))
except BlockingIOError:
pass
def connected(key, mask):
selector.unregister(key.fd)
print('connected!')
selector.register(sock.fileno(), EVENT_WRITE, connected)
selector.register(sock2.fileno(), EVENT_WRITE, connected)
def loop():
class TorSocketLoop(threading.Thread):
def __init__(self, our_sock, send_func):
super().__init__(name='SocketLoop{:x}'.format(our_sock.fileno()))
self._our_sock = our_sock
self._send_func = send_func
self._do_loop = True
self._cntrl_l = threading.Lock()
self._cntrl_r, self._cntrl_w = socket.socketpair()
self._selector = DefaultSelector()
self._selector.register(self._our_sock, EVENT_READ, self._do_recv)
self._selector.register(self._cntrl_r, EVENT_READ, self._do_stop)
def _do_recv(self, sock):
try:
data = sock.recv(1024)
self._send_func(data)
except ConnectionResetError:
logger.debug('Client was badly disconnected...')
def _do_stop(self, sock):
self._do_loop = False
def _cleanup(self):
with self._cntrl_l:
logger.debug('Cleanup')
self._selector.unregister(self._cntrl_r)
self._cntrl_w.close()
self._cntrl_r.close()
self.close_sock()
self._selector.close()
@property
def fileno(self):
if not self._our_sock:
return None
return self._our_sock.fileno()
def close_sock(self):
if not self._our_sock:
return
self._selector.unregister(self._our_sock)
self._our_sock.close()
self._our_sock = None
def stop(self):
# Because stop could be called twice
with self._cntrl_l:
logger.debug('Stopping...')
self._cntrl_w.send(b'\1')
def run(self):
logger.debug('Starting...')
while self._do_loop:
events = self._selector.select()
for key, _ in events:
callback = key.data
callback(key.fileobj)
self._cleanup()
logger.debug('Stopped!')
def append(self, data):
self._our_sock.send(data)
async def run(self):
l = [False]*4
selector = DefaultSelector()
selector.register(self.mouse, EVENT_READ)
selector.register(self.keyboard, EVENT_READ)
while True:
for key, mask in selector.select(0):
device = key.fileobj
for event in device.read():
if event is not None:
# mouse movement
if event.type == ecodes.EV_REL:
if event.code == ecodes.REL_X:
dx = event.value
self.controller_state.axis_state.dx += dx
elif event.code == ecodes.REL_Y:
dy = event.value
self.controller_state.axis_state.dy += dy
self.controller_state.axis_state.sum_y += dy
# mouse click
elif event.type == ecodes.EV_KEY:
pushed = True
if event.value == 0x01:#key down
pushed = True
elif event.value == 0x00:
pushed = False
#left click -> ZR
if event.code == ecodes.BTN_LEFT:
self.button_state.zr(pushed)
#right click -> ZL
elif event.code == ecodes.BTN_RIGHT:
self.button_state.zl(pushed)
#middle click -> Y
if event.code == ecodes.BTN_MIDDLE:
self.button_state.y(pushed)
#WASD -> up, left, right, down
if event.code == ecodes.KEY_W:
if pushed:
self.lstick_state.set_up()
l[0] = True
else:
if l[2]:
self.lstick_state.set_down()
else:
self.lstick_state.set_v(self.lstick_state._calibration.v_center)
l[0] = False
elif event.code == ecodes.KEY_A:
if pushed:
self.lstick_state.set_left()
l[1] = True
else:
if l[3]:
self.lstick_state.set_right()
else:
self.lstick_state.set_h(self.lstick_state._calibration.h_center)
l[1] = False
elif event.code == ecodes.KEY_S:
if pushed:
self.lstick_state.set_down()
l[2] = True
else:
if l[0]:
self.lstick_state.set_up()
else:
self.lstick_state.set_v(self.lstick_state._calibration.v_center)
l[2] = False
elif event.code == ecodes.KEY_D:
if pushed:
self.lstick_state.set_right()
l[3] = True
else:
if l[1]:
self.lstick_state.set_up()
else:
self.lstick_state.set_h(self.lstick_state._calibration.h_center)
l[3] = False
#FVE -> a,b,x
elif event.code == ecodes.KEY_F:
self.button_state.a(pushed)
elif event.code == ecodes.KEY_V:
self.button_state.b(pushed)
elif event.code == ecodes.KEY_E:
self.button_state.x(pushed)
#TG -> rstick, lstick
elif event.code == ecodes.KEY_T:
self.button_state.l_stick(pushed)
elif event.code == ecodes.KEY_G:
self.button_state.r_stick(pushed)
#QR -> l,r
elif event.code == ecodes.KEY_Q:
self.button_state.l(pushed)
elif event.code == ecodes.KEY_R:
self.button_state.r(pushed)
#ZX -> plus, minus
elif event.code == ecodes.KEY_Z:
self.button_state.minus(pushed)
elif event.code == ecodes.KEY_X:
self.button_state.plus(pushed)
#TAB -> zl
elif event.code == ecodes.KEY_TAB:
self.button_state.zl(pushed)
#SPACE -> b
elif event.code == ecodes.KEY_SPACE:
self.button_state.b(pushed)
#esc -> home
elif event.code == ecodes.KEY_ESC:
self.button_state.home(pushed)
#del -> break
elif event.code == ecodes.KEY_DELETE:
self.mouse.ungrab()
return
try:
await self.controller_state.send()
except NotConnectedError:
logger.info('Connection was lost.')
self.mouse.ungrab()
return
class Loop:
"""
Событийный цикл, который использует очередь сдвухсторонним доступом
Цикл опрашивает селектор на предмет готовности событий для чтения и записи
в сокет
"""
def __init__(self):
"""
Конструктор событийного цикла, который хранит очередь
:return:
"""
self.ready = deque()
self.selector = DefaultSelector()
async def sock_recv(self, sock, max_bytes):
"""
курутина для чтения данных из сокета в асинхронный способ
:param sock: дескриптор сокета
:param max_bytes: максимальное количество байт, который могут быть
прочитаны за один раз без блокировки
:return: принятые из сокета данные в банарном виде
"""
await read_wait(sock)
return sock.recv(max_bytes)
async def sock_accept(self, sock):
await read_wait(sock)
return sock.accept()
async def sock_sendall(self, sock, data):
while data:
try:
n_sent = sock.send(data)
data = data[n_sent:]
except BlockingIOError:
await write_wait(sock)
def create_task(self, coro):
self.ready.append(coro)
def run_forever(self):
while True:
while not self.ready:
events = self.selector.select()
for key, _ in events:
self.ready.append(key.data)
self.selector.unregister(key.fileobj)
while self.ready:
self.current_task = self.ready.popleft()
try:
# запускаем генератор до появления yield
op, *args = self.current_task.send(None)
getattr(self, op)(*args)
except StopIteration:
pass
def read_wait(self, sock):
self.selector.register(sock, EVENT_READ, self.current_task)
def write_wait(self, sock):
self.selector.register(sock, EVENT_WRITE, self.current_task)
import socket
from selectors import DefaultSelector, EVENT_READ, EVENT_WRITE
selector = DefaultSelector()
stopped = False
urls_todo = {"/", "/1", "/2", "/3"}
class Future:
def __init__(self):
self.result = None
self._callbacks = []
def add_done_callback(self, fn):
self._callbacks.append(fn)
def set_result(self, result):
self.result = result
for fn in self._callbacks:
fn(self)
def __iter__(self):
"""
yield的出现使得__iter__函数变成一个生成器，生成器本身就有next方法，所以不需要额外实现。
yield from x语句首先调用iter(x)获取一个迭代器（生成器也是迭代器）
"""
yield self # 外面使用yield from把f实例本身返回
return self.result, 'x' # 在Task.step中send(result)的时候再次调用这个生成器，但是此时会抛出stopInteration异常，并且把self.result返回
def read(sock):
class TorReceiver(threading.Thread):
def __init__(self, tor_socket, handler_mgr):
super().__init__(name='RecvLoop_{}'.format(tor_socket.ip_address[0:7]))
self._tor_socket = tor_socket
self._handler_mgr = handler_mgr
self._do_loop = False
# fmt: off
self._regs_funcs_map = {
'reg': {
socket.socket: self.register_socket,
TorStream: self.register_stream
},
'unreg': {
socket.socket: self.unregister_socket,
TorStream: self.unregister_stream
}
}
# fmt: on
self._stream_to_callback = {}
self._selector = DefaultSelector()
self._cntrl_r, self._cntrl_w = socket.socketpair()
self._selector.register(self._cntrl_r, EVENT_READ, self._do_stop)
self._selector.register(self._tor_socket.ssl_socket, EVENT_READ,
self._do_recv)
def _cleanup(self):
self._selector.unregister(self._cntrl_r)
self._cntrl_w.close()
self._cntrl_r.close()
self._selector.unregister(self._tor_socket.ssl_socket)
self._selector.close()
def start(self):
self._do_loop = True
super().start()
def stop(self):
logger.debug('Stopping receiver thread...')
self._cntrl_w.send(b'\1')
self.join()
def register(self, sock_or_stream, events, callback):
func = self._regs_funcs_map['reg'].get(type(sock_or_stream))
if not func:
raise Exception('Unknown object for register')
return func(sock_or_stream, events, callback)
def register_socket(self, sock, events, callback):
return self._selector.register(sock, events, callback)
def register_stream(self, stream: TorStream, events, callback):
if events & EVENT_WRITE:
raise Exception('Write event not supported yet')
stream.register(callback)
if stream not in self._stream_to_callback:
self._stream_to_callback[stream] = []
self._stream_to_callback[stream].append(callback)
def unregister(self, sock_or_stream):
func = self._regs_funcs_map['unreg'].get(type(sock_or_stream))
if not func:
raise Exception('Unknown object for unregister')
return func(sock_or_stream)
def unregister_socket(self, sock):
return self._selector.unregister(sock)
def unregister_stream(self, stream):
callbacks = self._stream_to_callback.pop(stream, None)
if not callbacks:
raise Exception('There is no such stream registered')
for callback in callbacks:
stream.unregister(callback)
def _do_stop(self, raw_socket, mask):
self._do_loop = False
def _do_recv(self, raw_socket, mask):
for cell in self._tor_socket.recv_cell_async():
logger.debug('Cell received: %r', cell)
try:
self._handler_mgr.handle(cell)
except BaseException:
logger.exception('Some handle errors')
def run(self):
logger.debug('Starting...')
while self._do_loop:
events = self._selector.select()
for key, mask in events:
callback = key.data
callback(key.fileobj, mask)
self._cleanup()
logger.debug('Stopped...')
def __init__(self):
self.address = ('127.0.0.1', 8080)
self.selector = DefaultSelector()
self.decoder = DecoderHandler()
self.server = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
class Crawler:
select = DefaultSelector()
finished = False
def __init__(self, url):
self.client = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.client.setblocking(False)
self.url = url
self.response = b''
global urls
# getUrl 这个委派生成器中创建的了3个yield from 通道用于调用方和3个子生成器的通信。
# 但是这3个通道不是同时建立的，而是先建立一个完了再建立下一个，同一时刻只有一个通道存在。也就是说同一时刻调用方只和1个子生成器通信（同一时刻调用方只运行1个协程）
def getUrl(self):
self.__parseUrl()
# 先暂停执行连接
yield from self.__connect()
# 连接完成后暂停执行发请求
yield from self.__sendReq()
# 发送请求后执行接收响应
yield from self.__recvResponse()
def __connect(self):
try:
self.client.connect((self.host, 80))
except:
pass
f = Future()
self.select.register(self.client, EVENT_WRITE, data=f)
yield f
def __sendReq(self):
self.select.unregister(self.client)
self.client.send(self.send_msg.encode('utf-8'))
f = Future()
self.select.register(self.client, EVENT_READ, data=f)
yield f
def __recvResponse(self):
while True:
self.select.unregister(self.client)
data = self.client.recv(1024)
if data:
self.response += data
f = Future()
self.select.register(self.client, EVENT_READ, data=f)
yield f
else:
self.client.close()
urls.remove(self.url)
self.__class__.finished = False if len(urls) else True
self.__saveHtml()
break
def __parseUrl(self):
url_component = urlparse(self.url)
self.host = url_component.netloc
self.path = '/' if url_component.path == '' else url_component.path
self.send_msg = "GET %s HTTP/1.1\r\nHost: %s\r\nConnection: close\r\n\r\n" % (
self.path, self.host)
def __saveHtml(self):
try:
dir_path = './crawled_page2/'
fname = 'index.html' if self.path == '/' else self.path.strip(
'/').strip('.html') + '.html'
content_arr = self.response.decode('utf-8').split("\r\n\r\n")
content_arr[0] = ''
content = ''.join(content_arr)
if not os.path.isdir(dir_path):
os.mkdir(dir_path)
with open(dir_path + fname, 'w', encoding='utf-8') as f:
f.write(content)
print("%s 爬取成功" % str(fname))
except BaseException as e:
print(e)
finally:
del self
@classmethod
def loopEvents(cls):
while not cls.finished:
events = cls.select.select()
for key, mask in events:
f = key.data
f.runCallback()
class Main(object):
def __init__(self, fb):
assert isinstance(fb, Framebuffer)
self._fb = fb
# get all input devices
devices = [InputDevice(fn) for fn in evdev.list_devices()]
# filter out non key devices
for device in devices.copy():
cap = device.capabilities()
if ecodes.EV_KEY not in cap:
devices.remove(device)
continue
self._selector = DefaultSelector()
# This works because InputDevice has a `fileno()` method.
for device in devices:
self._selector.register(device, EVENT_READ)
def _color565(self, r, g, b):
"""Convert red, green, blue components to a 16-bit 565 RGB value. Components
should be values 0 to 255.
"""
return (((r & 0xF8) << 8) | ((g & 0xFC) << 3) | (b >> 3))
def _img_to_rgb565_bytes(self):
pixels = [self._color565(r, g, b) for (r, g, b) in self._img.getdata()]
return pack('H' * len(pixels), *pixels)
def _write_image(self, img):
if (1 == self._fb.bits_per_pixel):
image_bytes = img.tobytes("raw", "1;IR", self._fb.line_length)
else:
self._img = img
image_bytes = self._img_to_rgb565_bytes()
self._fb.write_raw(image_bytes)
def _do_countdown(self):
if (1 == self._fb.bits_per_pixel):
img = Image.new("1", self._fb.resolution, 1)
else:
img = Image.new("RGB", self._fb.resolution, 'black')
fnt = ImageFont.truetype(font="/usr/share/fonts/truetype/dejavu/DejaVuSans.ttf", size=96)
d = ImageDraw.Draw(img)
for i in range(3, 0, -1):
# clear the image
d.rectangle(((0, 0), self._fb.resolution), fill=1)
# draw the text
text_width, text_height = d.textsize(str(i), font=fnt)
x = (self._fb.resolution.x - text_width) / 2
y = (self._fb.resolution.y - text_height) / 2
d.text((x, y), str(i), font=fnt)
self._write_image(img)
# give some time to read
time.sleep(1.25)
def _draw_text(self, text, size):
if (1 == self._fb.bits_per_pixel):
img = Image.new("1", self._fb.resolution, 1)
else:
img = Image.new("RGB", self._fb.resolution, 'black')
fnt = ImageFont.truetype(font="/usr/share/fonts/truetype/dejavu/DejaVuSans.ttf", size=size)
d = ImageDraw.Draw(img)
text_width, text_height = d.textsize(text, font=fnt)
x = (self._fb.resolution.x - text_width) / 2
y = (self._fb.resolution.y - text_height) / 2
d.text((x, y), text, font=fnt)
self._write_image(img)
def _take_picture(self):
args = 'fswebcam --quiet --no-banner'
args += ' --scale {0}x{1}'.format(self._fb.resolution.x, self._fb.resolution.y)
if self._fb.grayscale or self._fb.bits_per_pixel == 1:
args += ' --greyscale'
args += ' --png --save {0}'.format(self._filename)
fswebcam = Popen(args, shell=True)
fswebcam.wait()
def _show_picture(self):
if (1 == self._fb.bits_per_pixel):
self._write_image(ImageMath.eval('convert(img, "1")', img=Image.open(self._filename)))
else:
self._write_image(img=Image.open(self._filename))
def run(self):
self._draw_text('Ready!', 36)
# TODO: listen for keypresses on a background thread, otherwise they pile up while taking a picture.
while True:
for key, mask in self._selector.select():
device = key.fileobj
for event in device.read():
if event.type != ecodes.EV_KEY:
# ignore non-key events
continue
if not event.value:
# ignore key up events
continue
if event.code in [ecodes.KEY_ENTER, ecodes.KEY_CAMERA]:
# enter button or camera button takes a picture
self._do_countdown()
self._draw_text('Cheese!', 36)
self._filename = 'raw-{0}.png'.format(time.strftime("%Y%m%d-%H%M%S"))
self._take_picture()
self._draw_text('Please wait...', 24)
self._show_picture()
elif event.code == ecodes.KEY_BACKSPACE:
# back button to exit as usual.
exit(0)
class MySocket:
def get_conn(self):
self.sel = DefaultSelector()
self.host_port = ('0.0.0.0', 45633)
self.ss = socket.socket()
self.ss.setblocking(False)
self.ss.bind(self.host_port)
self.ss.listen(10)
self.sel.register(self.ss.fileno(), EVENT_READ, self.get_accept)
while 1:
ready = self.sel.select()
for key, mask in ready:
print(key.data)
call_back = key.data
call_back(key)
def get_accept(self, mask):
self.sel.unregister(mask.fd)
self.conn, self.addr = self.ss.accept()
self.conn.setblocking(False)
self.sel.register(self.conn.fileno(), EVENT_READ, self.get_recv)
def get_recv(self, mask):
# self.sel.unregister(mask.fd)
self.data = self.conn.recv(1024)
if self.data:
# self.conn.send(self.data)
self.sel.unregister(mask.fd)
self.sel.register(self.conn.fileno(), EVENT_WRITE, self.get_send)
else:
self.conn.close()
self.sel.unregister(mask.fd)
def get_send(self, mask):
self.sel.unregister(mask.fd)
self.conn.send(self.data)
self.sel.register(self.conn.fileno(), EVENT_READ, self.get_recv)
def audio(dev):
for event in dev.read_loop():
if event.value == 1 and event.type == 1:
print('按下键', evdev.ecodes.KEY[event.code], 'device:', dev.fn)
def detectInputKey(dev_keyboards):
while True:
select([dev], [], [])
for event in dev.read():
print("code:{} value:{}".format(event.code, event.value))
if __name__ == '__main__':
keyboards = check_keyboard()
selector = DefaultSelector()
#selector.register(keyboards[0],EVENT_READ)
#selector.register(keyboards[1],EVENT_READ)
for kb in keyboards:
selector.register(kb, EVENT_READ)
try:
while True:
for key, mask in selector.select():
dev = key.fileobj
print('有事件发生')
for event in dev.read():
print(event)
if event.value == 1 and event.type == 1:
pass
#print('按下键',evdev.ecodes.KEY[event.code],'device:',dev.fn)
# * non blocking sockets 2°
# * callbacks, allowing multiples operations to be waiting concurrently for i/o operations 3°
# * event loop 3°
# * coroutines 4°
# --> Future
# --> generators
# --> Task, responsible for calling next() on the generators
import socket
import time
# 2° to wait for some event on a non blocking socket
# jusk ask for the default selector for your system
from selectors import DefaultSelector, EVENT_WRITE, EVENT_READ
selector = DefaultSelector()
n_tasks = 0
# a Future represent some pending event we're waiting for
class Future:
def __init__(self):
self.callbacks = [] # what to do when event occurs
def resolve(self):
for c in self.callbacks:
c()
class Task:
def __init__(self, gen):
self.gen = gen
本代码使用 I/O多路复用 中的select完成http请求 单线程
*事件循环使用场景
tornado
twisted (scrapy、django channels)
gevent
asyncio
"""
import socket
from urllib.parse import urlparse
# selectors是对select库的进一步封装
from selectors import DefaultSelector, EVENT_READ, EVENT_WRITE
# import select
# select.select()
selector = DefaultSelector() # win用poll ,linux用epoll
urls = []
stop = False
class Fetcher:
def connected(self, key):
selector.unregister(key.fd) # fd == self.client.fileno()
self.client.send(
"GET {} HTTP/1.1\r\nHost:{}\r\nConnection:close\r\n\r\n".format(
self.path, self.host).encode("utf8"))
# 注册一个事件: 先监听等待，当当前socket变为可写EVENT_READ时候，调用self.readable方法
# (里边有recv data方法)里边的逻辑
selector.register(self.client.fileno(), EVENT_READ, self.readable)
def readable(self, key):
def selector():
sel = DefaultSelector()
sel.register(proc.stdout.channel, EVENT_READ)
while True:
for key, mask in sel.select():
yield
def __init__(self):
self.tasks = deque()
self.selector = DefaultSelector()
User-Agent: PPTViPhone/7.8.0.1911 CFNetwork/811.5.4 Darwin/16.7.0
Accept-Encoding: gzip, deflate
Content-Length: 2
ab
'''
host='ulogs.umeng.com'
clientcert=path.join(path.abspath(path.dirname(path.dirname(__file__))),'certs','soul_client.pem')
servercert=path.join(path.abspath(path.dirname(path.dirname(__file__))),'certs','sn_new_server.crt')
serverkey=path.join(path.abspath(path.dirname(path.dirname(__file__))),'certs','sn_new_server_key.pem')
selector=DefaultSelector()
ctx=SSLContext(PROTOCOL_SSLv23)
ctx.load_cert_chain(certfile=clientcert)
sock=socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.setblocking(False)
selector.register(sock,EVENT_WRITE)
try:
sock.connect((host,443))
except BlockingIOError:
pass
while True:
events=selector.select(2)
if events:
break
selector.unregister(sock)
class TCPServer:
def __init__(self, port):
self.port = port
self.sock = None
self.selector = DefaultSelector()
# Advance self.listen() method. A future has been attached to the
# server socket, that has been registered in selector. And a callback
# has been attached to this future.
Task(self.listen())
def run(self):
while True:
ready = self.selector.select()
for key, events in ready:
fut = key.data
fut.resolve()
def listen(self):
server_sock = self._create_socket()
with server_sock:
try:
client_sock, remote_addr = self.sock.accept()
except BlockingIOError:
pass
fut = Future()
self.selector.register(self.sock, EVENT_READ, fut)
while True:
yield fut
Task(self.handle_client())
def handle_client(self):
client_sock, remote_addr = self.sock.accept()
client_sock.setblocking(False)
client_name = '{} {}'.format(*client_sock.getpeername())
print('Connected', client_name)
# Register client socket for reading and wait for this event
fut = Future()
self.selector.register(client_sock, EVENT_READ, fut)
yield fut
buffer = []
while True:
try:
chunk = client_sock.recv(1024)
buffer.append(chunk)
except BlockingIOError:
# Yield the same future, so we don't need to register
# it again. By the way, callback is the same for all
# futures, so it doesn't matter in my case.
yield fut
continue
print('Received chunk from client', client_name)
if chunk.endswith(b'end'):
print('Got all data from client', client_name)
request = b''.join(buffer).decode('utf8')
response = (request.upper() * 100).encode('utf8')
self.selector.unregister(client_sock)
fut = Future()
self.selector.register(client_sock, EVENT_WRITE, fut)
break
# This future was attached to the client_sock
yield fut
while True:
try:
bytes_sent = client_sock.send(response)
print(bytes_sent, 'bytes sent to client', client_name)
response = response[bytes_sent:]
except BlockingIOError:
print('Socket blockes for', client_name)
yield fut
continue
else:
if not response:
# All response sent
print('Response sent to client', client_name)
self.selector.unregister(client_sock)
client_sock.close()
break
def _create_socket(self):
self.sock = socket.socket()
self.sock.setsockopt(socket.SOL_SOCKET, socket.SO_SNDBUF, 10)
self.sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
self.sock.bind(('', self.port))
self.sock.setblocking(False)
self.sock.listen(10)
self.responses = {}
return self.sock
def start(self):
_sel = Selector()
for n in self.nodes.values():
_sel.register(n['sock'], EVENT_READ, {})
Thread(target=self._loop, args=(_sel, ), daemon=True).start()
# 使用socket获取http报文
# 把setblocking设为false 让学员自己解决问题
import socket
from urllib.parse import urlparse
from selectors import DefaultSelector, EVENT_READ, EVENT_WRITE
# 申明select
sel = DefaultSelector()
class Downloader:
def __init__(self, sel):
self.domain = ""
self.url = ""
self.path = ""
self.host = ""
self.selector = sel
self.ss = ""
self.data = b""
def get_client(self):
# 初始化socket
self.ss = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
# 使用非阻塞io，socket不等待连接建立好 直接向下执行
self.ss.setblocking(False)
# 虽然申明了一个非阻塞操作,但这个还是阻塞操作,不能交给select
try:
self.ss.connect((self.host, 18001))
except Exception:
#1. epoll并不代表一定比select好
# 在并发高的情况下，连接活跃度不是很高， epoll比select
# 并发性不高，同时连接很活跃， select比epoll好
#通过非阻塞io实现http请求
# select + 回调 + 事件循环
# 并发性高
# 虽然使用单线程，但是省去了I/O等待时线程切换的开销，因为几十个线程切换的开销是很大的，但是成千上万个回调却是很快。这这种方式是高并发编程的核心。
import socket
from urllib.parse import urlparse
from selectors import DefaultSelector, EVENT_READ, EVENT_WRITE # DefaultSelector是选择select、poll还是epoll根据平台自己选择。
selector = DefaultSelector() # 全局变量
#使用select完成http请求
urls = []
stop = False
class Fetcher:
def connected(self, key):
selector.unregister(key.fd) # 注销掉监控的描述符
print("=========key.fd:", key.fd)
print("=========self.client.fileno():", self.client.fileno())
self.client.send("GET {} HTTP/1.1\r\nHost:{}\r\nConnection:close\r\n\r\n".format(self.path, self.host).encode("utf8"))
selector.register(self.client.fileno(), EVENT_READ, self.readable) # 注册读取监控
# 全都是cpu操作，速度远远快于I/O操作，尤其是网络I/O
class Loop:
def __init__(self):
self.ready = collections.deque()
self.selector = DefaultSelector()
async def sock_accept(self, sock):
'''
Wait for the server socket to become readable, accept a new
client and return her socket, addr pair.
'''
await read_wait(sock)
return sock.accept()
async def sock_recv(self, sock, maxbytes):
'''
Wait for the socket to become readable, read data from the
socket and return data.
'''
await read_wait(sock)
return sock.recv(maxbytes)
async def sock_sendall(self, sock, data):
'''
While we have some data to send, wait for the socket to become
writeable and send some data.
'''
while data:
await write_wait(sock)
nbytes = sock.send(data)
# Modify data according to the number of bytes sent this time
data = data[nbytes:]
def create_task(self, coro):
self.ready.append(coro)
def run_forever(self):
while True:
while not self.ready:
events = self.selector.select()
for key, _ in events:
self.selector.unregister(key.fileobj)
self.ready.append(key.data)
while self.ready:
# I need this task in another method, so I attach it here
self.current_task = self.ready.popleft()
try:
# drive the coroutine to the next yield
op, *args = self.current_task.send(None)
except StopIteration:
pass
else:
# call method on myself
# this methods will register sockets with selector
getattr(self, op)(*args)
def read_wait(self, sock):
# When this socket is ready, drive this task to the next yield
self.selector.register(sock, EVENT_READ, self.current_task)
def write_wait(self, sock):
self.selector.register(sock, EVENT_WRITE, self.current_task)
class TorSocketLoop(threading.Thread):
def __init__(self, our_sock, send_func):
super().__init__(name='SocketLoop{:x}'.format(our_sock.fileno()))
self._our_sock = our_sock
self._send_func = send_func
self._do_loop = True
self._cntrl_r, self._cntrl_w = socket.socketpair()
self._selector = DefaultSelector()
self._selector.register(self._our_sock, EVENT_READ, self._do_recv)
self._selector.register(self._cntrl_r, EVENT_READ, self._do_stop)
def _do_recv(self, sock):
data = sock.recv(1024)
self._send_func(data)
def _do_stop(self, sock):
self._do_loop = False
def _cleanup(self):
self._selector.unregister(self._cntrl_r)
self._cntrl_w.close()
self._cntrl_r.close()
self._selector.unregister(self._our_sock)
self._our_sock.shutdown(socket.SHUT_WR)
self._our_sock.close()
self._selector.close()
def stop(self):
self._cntrl_w.send(b'\1')
def run(self):
logger.debug("Starting...")
while self._do_loop:
events = self._selector.select()
for key, _ in events:
callback = key.data
callback(key.fileobj)
self._cleanup()
logger.debug("Stopped...")
def append(self, data):
self._our_sock.send(data)
#!/usr/env/bin python
import socket
import re
from selectors import DefaultSelector, EVENT_WRITE, EVENT_READ
try:
from urlparse import urlparse
except ImportError:
from urllib.parse import urlparse
selector = DefaultSelector() # Linux system will use epoll()
class Task():
def __init__(self, gen):
self.gen = gen
f = Future()
f.set_result(None)
self.step(f)
def step(self, future):
try:
f = self.gen.send(future.result)
except StopIteration:
return
f.add_done_callback(self.step)
class Future():
def __init__(self):
def __init__(self):
self.selector = DefaultSelector()
self.clients = {}
class Peer(HandleableMixin, CommandableMixin, LanTopologyMixin,
DefaultAuthenticatorMixin):
"""
Attributes:
server_info (PeerInfo): Peer's peer info.
pkt_handlers (Dict[str, Handler]): All handlers which peer have ability
to process.
peer_pool (Dict[Tuple[str, int], PeerInfo]): All peers currently avai-
lable in net.
"""
@property
def server_info(self):
return self.__server_info
@property
def program_hash(self):
return self.__program_hash
@property
def send_queue(self):
# def packet_queue(self):
return self.__packet_queue
@property
def connectlist(self) -> List: # Remember to remove `List` import.
return self.peer_pool.values()
def __init__(
self,
host: Tuple[str, int],
name: str,
role: str,
cert: Tuple[str, str],
program_hash: str,
ns: str,
auto_register: bool = False,
logger: "logging.Logger" = getLogger(__name__),
) -> None:
"""Init of PeerManager
Args:
host: Binding host.
name: Peer's name in net.
role: Peer's role in net.
cert: Cert file's path.
program_hash: Program self hash to send in packet.
ns: Nameserver address for resolve DNS.
logger: Logger for logging.
"""
super().__init__()
self.logger = getLogger(name)
self.__auto_register = auto_register
self.__selector = DefaultSelector()
self.__packet_queue = {}
self.__cert = cert
self.__program_hash = program_hash
self.__server_info = PeerInfo(host=host, name=name, role=role)
self.__tcp_server = self.__bind_socket(cert=self.__cert)
self.peer_pool = {}
self.pkt_handlers = {}
self.commands = {}
self.logger.info("Program hash: {{{}...{}}}".format(
self.__program_hash[:6], self.__program_hash[-6:]))
self.dns_resolver = DNSResolver(ns="127.0.0.1" if ns is None else ns,
role=role)
self.monitor = Monitor(peer=self, logger=getLogger(name + ".MONITOR"))
if self.__auto_register is False:
self.logger.warning(
("auto_register parameter is set to False,\n You may need to r"
"egister them through _register_command & _register_handler m"
"ethod."))
def _preregister_handler(self) -> None:
self.topology_register_handler()
installing_handlers = [MessageHandler(self)]
for each in installing_handlers:
self.register_handler(handler=each)
def _preregister_command(self) -> None:
installing_commands = [
HelpCmd(self),
JoinCmd(self),
SendCmd(self),
ListCmd(self),
LeaveNetCmd(self),
]
for each in installing_commands:
self.register_command(command=each)
def pend_packet(self, sock: "Socket", pkt: "Packet", **kwargs) -> None:
"""Pending pkt's raw_data to queue's with sepecific sock.
Any exception when wrapping handler to packet whould cause this connec-
tion been close and thread maintaining loop terminate.
Args:
sock: A Socket which wants to pend on its queue.
pkt: A Packet ready to be pend.
**kwargs: Any additional arguments needs by handler object.
Raises:
AssertionError:
If given pkt variable is not in proper Packet type.
"""
assert type(pkt) is Packet
try:
self.__packet_queue[sock].put_nowait(pkt)
except Exception:
self.logger.info(format_exc())
def register_socket(self, sock: "Socket") -> None:
"""Register a new socket with packet queue & selector.
Init a packet queue and put into dict for further handling of packets.
And the given socket will be register in selector for IO process.
Args:
sock: A Socket object which wants to be register.
"""
self.__packet_queue[sock] = Queue()
self.__selector.register(sock, EVENT_READ | EVENT_WRITE,
self.__on_handle)
def unregister_socket(self, sock: "Socket") -> None:
del self.__packet_queue[sock]
self.__selector.unregister(sock)
def _on_packet(self, sock: "Socket", pkt: "Packet",
handler: "Handler") -> None:
"""Method use to process passed packet to higher application layer.
This method will call by AuthenticatorMixin when a packet is passed examination.
This is 3rd layer to process packet to handler. This is last layer to application layer.
"""
handler.on_recv(src=pkt.src, pkt=pkt, sock=sock)
self.monitor.on_recv_pkt(addr=pkt.src, pkt=pkt, conn=sock)
def new_tcp_long_conn(self, dst: Tuple[str, int]) -> "SSLSocket":
"""Create a ssl-wrapped TCP socket with given destination host
Args:
dst: Specified socket destination.
Returns:
A SSLSocket object which connected to destination host with
non-blocking.
Raises:
AssertionError:
If given dst variable is not in proper Tuple[str, int] type.
"""
assert host_valid(dst) is True
unwrap_socket = socket(AF_INET, SOCK_STREAM)
sock = wrap_socket(unwrap_socket,
cert_reqs=CERT_REQUIRED,
ca_certs=self.__cert[0])
sock.connect(dst)
sock.setblocking(False)
return sock
def loop_start(self):
self.logger.info(self.__server_info)
self.__selector.register(self.__tcp_server, EVENT_READ,
self.__on_accept)
if self.__auto_register is True:
self._preregister_handler()
self._preregister_command()
if self.monitor.is_start() is False:
self.monitor.start()
self.logger.info("Peer started.")
def loop(self):
return self._loop()
def _loop(self):
"""Called inside infinite loop from outside inherited class.
It's use to call the method which given event is triggered.
"""
events = self.__selector.select(timeout=0)
for key, mask in events:
if callable(key.data):
key.data(key.fileobj, mask)
def loop_stop(self):
for _, value in self.__packet_queue.items():
if value.empty() is False:
sleep(2)
return self.loop_stop()
self.__selector.unregister(self.__tcp_server)
self.leave_net()
self.monitor.stop()
def loop_stop_post(self):
self.__tcp_server.close()
self.__selector.close()
def __bind_socket(self, cert: Tuple[str, str]) -> "SSLSocket":
unwrap_socket = socket(AF_INET, SOCK_STREAM)
unwrap_socket.setsockopt(SOL_SOCKET, SO_REUSEADDR, 1)
unwrap_socket.setsockopt(SOL_SOCKET, SO_REUSEPORT, 1)
unwrap_socket.bind(self.__server_info.host)
unwrap_socket.listen(5)
unwrap_socket.setblocking(False)
self.logger.info("Peer prepared")
self.logger.info(
"This peer is running with certificate at path {}".format(cert[0]))
self.logger.info("Please make sure other peers have same certicate.")
return wrap_socket(unwrap_socket,
certfile=cert[0],
keyfile=cert[1],
server_side=True)
def __on_accept(self, sock: "Socket", mask, **kwargs):
"""Call when a new socket is connection by waiter socket.
This will accept all sockets from outside, but it doesn't mean every socket's
packet be process by higher layer.
This is the 1st layer to process sockets.
"""
conn, _ = sock.accept()
conn.setblocking(False)
self.register_socket(sock=conn)
def __on_handle(self, sock: "Socket", mask, **kwargs):
"""Decide whether send or recv."""
if mask & EVENT_READ == EVENT_READ:
self.__on_recv(sock=sock, mask=mask, **kwargs)
if mask & EVENT_WRITE == EVENT_WRITE:
self.__on_send(sock=sock, mask=mask, **kwargs)
def __on_recv(self, sock: "Socket", mask, **kwargs):
"""Method use when recieve socket data.
This is 2th layer to process Packets.
"""
try:
raw_data = sock.recv(4096)
if raw_data == b"":
return
pkt = Packet.deserilize(raw_data=raw_data)
return self._authenticate_packet(sock=sock, pkt=pkt)
except SSLWantReadError:
return
except sock_error as sock_err:
if sock_err.errno == ECONNRESET:
peer_info = self.get_peer_info_by_conn(conn=sock)
if peer_info is not None:
peer_info.status.update(status_type=StatusType.NO_RESP)
self.logger.warning("Peer {} Connection Reseted.".format(
peer_info.host))
else:
raise sock_err
except Exception:
self.logger.warning(str(self.server_info) + format_exc())
def __on_send(self, sock: "Socket", mask, **kwargs):
"""Method use when sending data to socket."""
q = self.__packet_queue[sock] if sock in self.__packet_queue else None
while q is not None and q.empty() is False:
try:
pkt = q.get_nowait()
handler = self.select_handler(pkt_type=pkt._type)
handler.pre_send(pkt=pkt)
data = Packet.serilize(obj=pkt)
sock.send(data)
handler.post_send(pkt=pkt, sock=sock)
except sock_error as sock_err:
if sock_err.errno == ECONNRESET:
q.put_nowait(pkt)
self.monitor.peer_status_update_by_host(
host=pkt.src, status_type=StatusType.NO_RESP)
self.logger.warning("Peer {} Connection Reseted.".format(
pkt.src))
else:
raise sock_err
except Exception:
self.logger.warning(format_exc())
class DBusConnection:
def __init__(self, sock):
self.sock = sock
self.parser = Parser()
self.outgoing_serial = count(start=1)
self.selector = DefaultSelector()
self.select_key = self.selector.register(sock, EVENT_READ)
self._stop_r, self._stop_w = os.pipe()
self.stop_key = self.selector.register(self._stop_r, EVENT_READ)
self.send_lock = Lock()
self.rcv_lock = Lock()
self.unique_name = None
def __enter__(self):
return self
def __exit__(self, exc_type, exc_val, exc_tb):
self.close()
return False
def send(self, message: Message, serial=None):
"""Serialise and send a :class:`~.Message` object"""
if serial is None:
serial = next(self.outgoing_serial)
data = message.serialise(serial=serial)
with self.send_lock:
self.sock.sendall(data)
def receive(self, *, timeout=None) -> Message:
"""Return the next available message from the connection
If the data is ready, this will return immediately, even if timeout<=0.
Otherwise, it will wait for up to timeout seconds, or indefinitely if
timeout is None. If no message comes in time, it raises TimeoutError.
If the connection is closed from another thread, this will raise
ReceiveStopped.
"""
if timeout is not None:
deadline = time.monotonic() + timeout
else:
deadline = None
with self.rcv_lock:
while True:
msg = self.parser.get_next_message()
if msg is not None:
return msg
if deadline is not None:
timeout = deadline - time.monotonic()
b = self._read_some_data(timeout)
self.parser.add_data(b)
def _read_some_data(self, timeout=None):
# Wait for data or a signal on the stop pipe
for key, ev in self.selector.select(timeout):
if key == self.select_key:
return unwrap_read(self.sock.recv(4096))
elif key == self.stop_key:
raise ReceiveStopped(
"DBus receive stopped from another thread")
raise TimeoutError
def interrupt(self):
"""Make any threads waiting for a message raise ReceiveStopped"""
os.write(self._stop_w, b'a')
def reset_interrupt(self):
"""Allow calls to .receive() again after .interrupt()
To avoid race conditions, you should typically wait for threads to
respond (e.g. by joining them) between interrupting and resetting.
"""
# Clear any data on the stop pipe
while (self.stop_key, EVENT_READ) in self.selector.select(timeout=0):
os.read(self._stop_r, 1024)
def close(self):
"""Close the connection"""
self.interrupt()
self.selector.close()
self.sock.close()
def __init__(self, server_id, handler: Handler):
self.select = DefaultSelector()
self.server_id = server_id
self.handler = handler
class SimpleServer(object):
""" This is a simple server receiving and sending utf-8 plain text lines
over TCP/IP. It should use epoll or other efficient mechanisms if
available to support a large number of clients without cpu hogging.
All data sent to connections will be utf-8 encoded. All received lines
from clients are expected to be utf-8 encoded.
Lines longer than 10KB will be quietly dropped.
"""
def __init__(self, port, callback):
logging.basicConfig(level=logging.INFO)
self.clients_kept_for_sending = []
self.selector = DefaultSelector()
self.callback = callback
self.server_socket = socket.socket(family=socket.AF_INET6,
type=socket.SOCK_STREAM)
self.server_socket.bind(("::0", port))
self.server_socket.listen(5)
self.selector.register(self.server_socket, EVENT_READ, data=None)
self.server_socket.setblocking(False)
def close(self):
self.server_socket.close()
def _remove_socket(self, socket):
try:
self.selector.unregister(socket)
except KeyError:
pass
def _handle_new_client(self, socket, addr):
client = SimpleClient(socket, self, addr)
self.selector.register(socket, EVENT_READ, data=client)
self.callback(client)
def _mark_client_for_sending(self, client, marked):
if marked:
self.selector.modify(client.socket, EVENT_READ | EVENT_WRITE,
data=client)
else:
self.selector.modify(client.socket, EVENT_READ, data=client)
def tick(self, timeout=0):
""" This will check for any server events and process them.
If timeout is greater than zero, it will wait at most that
amount of time in seconds until something happens.
(This is useful if you want to save some cpu time and you have
nothing else to do)
For debugging purposes, it returns the amount of events processed.
"""
# get rid of old connections we kept around for sending:
entry_removed = True
while entry_removed:
entry_removed = False
for entry in self.clients_kept_for_sending:
if entry[1] <= time.time() or len(entry[0].send_buffer) == 0:
try:
entry[0].close()
except:
pass
self.clients_kept_for_sending.remove(entry)
entry_removed = True
# wait for socket events:
events = self.selector.select(timeout=timeout)
events_processed = 0
for key, mask in events:
if mask & EVENT_READ != 0:
events_processed = events_processed + 1
# handle accept for server socket:
if key.data == None:
conn, addr = self.server_socket.accept()
self._handle_new_client(conn, addr)
continue
# handle client socket receive:
key.data._do_recv()
continue
if mask & EVENT_WRITE != 0:
events_processed = events_processed + 1
# handle client socket send:
key.data._do_outstanding_send()
continue
logging.debugging("unknown socket event happening, bug?")
return events_processed
import argparse
from selectors import DefaultSelector, EVENT_WRITE
from socket import socket
# 网络连接的例子（socket和selectors结合）
def connected():
selector.unregister(sock.fileno())
print('connected!')
url = "http://www.baicu.com"
sock = socket()
sock.setblocking(False)
sock.connect(('xkcd.com', 80))
selector = DefaultSelector()
selector.register(sock.fileno(), connected())
def __init__(self):
self.sel = DefaultSelector()
self.queue = deque()
def selector():
sel = DefaultSelector()
sel.register(proc.stdout.channel, EVENT_READ)
while True:
for key, mask in sel.select():
yield
def __init__(self):
self.ready = collections.deque()
self.selector = DefaultSelector()
import socket
from selectors import DefaultSelector, EVENT_WRITE, EVENT_READ
sock = socket.socket()
sock.setblocking(False)
selector = DefaultSelector()
urls_todo = set(['/'])
seen_urls = set(['/'])
class Fetcher:
def __init__(self, url):
self.response = b''
self.url = url
self.sock = None
def fetch(self):
# This method fetches the url
self.sock = socket.socket()
self.sock.setblocking(False)
try:
self.sock.connect(('xkcd.com'), 80)
except BlockingIOError:
pass
selector.register(self.sock.fileno(),
EVENT_WRITE,
self.connected)
# coding=utf-8
import socket
from selectors import DefaultSelector, EVENT_READ, EVENT_WRITE
epoller = DefaultSelector()
stopped = False
urls_todo = {'/' + str(i) for i in range(10)}
class Future:
def __init__(self):
self.result = None
self._callbacks = []
def add_done_callback(self, fn):
self._callbacks.append(fn)
def set_result(self, result):
self.result = result
for fn in self._callbacks:
fn(self)
# 改造为一个可迭代对象
def __iter__(self):
yield self
return self.result
# 抽象sock连接功能
def connect(sock, address):
f = Future()
class LibInput(object):
"""A base/factory class for libinput context.
Context is used to manage devices and get events.
"""
_libc = CDLL('libc.so.6')
_libc.vsprintf.argtypes = (c_char_p, c_char_p, c_void_p)
_libc.vsprintf.restype = c_int
_libudev = CDLL('libudev.so.1')
_libudev.udev_new.argtypes = None
_libudev.udev_new.restype = c_void_p
_libudev.udev_unref.argtypes = (c_void_p, )
_libudev.udev_unref.restype = None
# libinput is not available on RTD so we prevent failing immediately
# when it's not available. Trying to instantiate the class will still
# throw an exception.
try:
_libinput = CDLL('libinput.so.10')
_libinput.libinput_udev_create_context.argtypes = (POINTER(
Interface.Interface), c_void_p, c_void_p)
_libinput.libinput_udev_create_context.restype = c_void_p
_libinput.libinput_udev_assign_seat.argtypes = (c_void_p, c_char_p)
_libinput.libinput_udev_assign_seat.restype = c_int
_libinput.libinput_path_create_context.argtypes = (POINTER(
Interface.Interface), c_void_p)
_libinput.libinput_path_create_context.restype = c_void_p
_libinput.libinput_unref.argtypes = (c_void_p, )
_libinput.libinput_unref.restype = c_void_p
_libinput.libinput_log_set_handler.argtypes = (c_void_p, c_void_p)
_libinput.libinput_log_set_handler.restype = None
_libinput.libinput_log_set_priority.argtypes = (c_void_p, LogPriority)
_libinput.libinput_log_set_priority.restype = None
_libinput.libinput_get_fd.argtypes = (c_void_p, )
_libinput.libinput_get_fd.restype = c_int
_libinput.libinput_suspend.argtypes = (c_void_p, )
_libinput.libinput_suspend.restype = None
_libinput.libinput_resume.argtypes = (c_void_p, )
_libinput.libinput_resume.restype = c_int
_libinput.libinput_path_add_device.argtypes = (c_void_p, c_char_p)
_libinput.libinput_path_add_device.restype = c_void_p
_libinput.libinput_path_remove_device.argtypes = (c_void_p, )
_libinput.libinput_path_remove_device.restype = None
_libinput.libinput_dispatch.argtypes = (c_void_p, )
_libinput.libinput_dispatch.restype = c_int
_libinput.libinput_get_event.argtypes = (c_void_p, )
_libinput.libinput_get_event.restype = c_void_p
_libinput.libinput_event_get_type.argtypes = (c_void_p, )
_libinput.libinput_event_get_type.restype = EventType
_libinput.libinput_next_event_type.argtypes = (c_void_p, )
_libinput.libinput_next_event_type.restype = c_int
except OSError:
pass
def __new__(cls, context_type=ContextType.PATH, debug=False):
if context_type == ContextType.PATH:
return LibInputPath()
elif context_type == ContextType.UDEV:
return LibInputUdev()
else:
raise TypeError('Unsupported context type')
def __init__(self, context_type=ContextType.PATH, debug=False):
"""Initialize context.
Args:
context_type (~libinput.constant.ContextType): If
:attr:`~libinput.constant.ContextType.UDEV` devices are
added/removed from udev seat. If
:attr:`~libinput.constant.ContextType.PATH` devices have to be
added/removed manually.
debug (bool): If false, only errors are printed.
"""
self._selector = DefaultSelector()
self._interface = Interface()
if context_type == ContextType.UDEV:
self._udev = self._libudev.udev_new()
self._li = self._libinput.libinput_udev_create_context(
byref(self._interface), None, self._udev)
elif context_type == ContextType.PATH:
self._li = self._libinput.libinput_path_create_context(
byref(self._interface), None)
self._log_handler = lambda pr, strn: print(pr.name, ': ', strn)
self._set_default_log_handler()
if debug:
self._libinput.libinput_log_set_priority(self._li,
LogPriority.DEBUG)
self._selector.register(self._libinput.libinput_get_fd(self._li),
EVENT_READ)
def __del__(self):
while self._libinput.libinput_unref(self._li):
pass
def _set_default_log_handler(self):
def default_log_handler(li, priority, fmt, args):
string = create_string_buffer(2048)
self._libc.vsprintf(string, fmt, args)
self._log_handler(LogPriority(priority), string.value.decode())
CMPFUNC = CFUNCTYPE(None, c_void_p, c_int, c_char_p, c_void_p)
self._default_log_handler = CMPFUNC(default_log_handler)
self._libinput.libinput_log_set_handler(self._li,
self._default_log_handler)
@property
def log_handler(self):
"""Callable that handles error/info/debug messages.
Args:
priority (~libinput.constant.LogPriority): Message priority.
message (str): The message.
Default handler prints messages to stdout.
"""
return self._log_handler
@log_handler.setter
def log_handler(self, handler):
self._log_handler = handler
def suspend(self):
"""Suspend monitoring for new devices and close existing devices.
This all but terminates libinput but does keep the context valid to be
resumed with :meth:`resume`.
"""
self._libinput.libinput_suspend(self._li)
def resume(self):
"""Resume a suspended libinput context.
This re-enables device monitoring and adds existing devices.
"""
rc = self._libinput.libinput_resume(self._li)
assert rc == 0, 'Failed to resume current context'
@property
def events(self):
"""Yield events from the internal libinput's queue.
Yields device events that are subclasses of
:class:`~libinput.event.Event`.
Yields:
:class:`~libinput.event.Event`: Device event.
"""
while True:
events = self._selector.select()
for nevent in range(len(events) + 1):
self._libinput.libinput_dispatch(self._li)
hevent = self._libinput.libinput_get_event(self._li)
if hevent:
type_ = self._libinput.libinput_event_get_type(hevent)
self._libinput.libinput_dispatch(self._li)
if type_.is_pointer():
yield PointerEvent(hevent, self._libinput)
elif type_.is_keyboard():
yield KeyboardEvent(hevent, self._libinput)
elif type_.is_touch():
yield TouchEvent(hevent, self._libinput)
elif type_.is_gesture():
yield GestureEvent(hevent, self._libinput)
elif type_.is_tablet_tool():
yield TabletToolEvent(hevent, self._libinput)
elif type_.is_tablet_pad():
yield TabletPadEvent(hevent, self._libinput)
elif type_.is_switch():
yield SwitchEvent(hevent, self._libinput)
elif type_.is_device():
yield DeviceNotifyEvent(hevent, self._libinput)
def next_event_type(self):
"""Return the type of the next event in the internal queue.
This method does not pop the event off the queue and the next call
to :attr:`events` returns that event.
Returns:
~libinput.constant.EventType: The event type of the next available
event or :obj:`None` if no event is available.
"""
type_ = self._libinput.libinput_next_event_type(self._li)
if type_ == 0:
return None
else:
return EventType(type_)
class Loop:
def __init__(self):
self._tasks = deque()
self._sleeping = []
self._stop = False
self._selector = DefaultSelector()
def create_task(self, coroutine):
if isinstance(coroutine, Task):
return coroutine
task = Task(coroutine)
self._tasks.append(task)
return task
def remove(self, task):
self.tasks.remove(task)
def _stop_loop_cb(self, task):
self._stop = True
def _wakeup_sleeping_tasks(self):
now = time.monotonic()
while self._sleeping and self._sleeping[0][0] <= now:
_, task = heapq.heappop(self._sleeping)
self._tasks.append(task)
def io_poll(self):
if self._sleeping:
sleep_time = self._sleeping[0][0] - time.monotonic()
else:
sleep_time = None
events = self._selector.select(sleep_time)
for key, mask in events:
task = key.data
self._selector.unregister(key.fileobj)
self._tasks.append(task)
# Wake up time!
self._wakeup_sleeping_tasks()
def run_until_complete(self, coroutine):
task = self.create_task(coroutine)
task.add_done_callback(self._stop_loop_cb)
self.run_forever()
task.remove_done_callback(self._stop_loop_cb)
return task.result
def run_forever(self):
self._stop = False # Just in case we are restarted
while not self._stop:
while not self._tasks:
self.io_poll()
# Let's see if there any task who wants to wake up
self._wakeup_sleeping_tasks()
current = self._tasks.popleft()
try:
# We assume that each Task will yield some method name and
# an argument. Based on that, we decide what to do.
method_name, arg = next(current)
except StopIteration as e:
current.result = e.value
except Exception as e:
current.exception = e
print(f'Warning: Task {current.id} raised {e!r}')
else:
method = getattr(self, method_name, None)
assert method, f'unknown method name {method_name}'
method(arg, current)
finally:
if current.done:
for callback in current.done_callbacks:
callback(current)
# These are the methods which can be "invoked" by Tasks by yielding their
# name, together with an argument.
def sleep_task(self, seconds, task):
# We simply add the task to the self._sleeping priority queue.
# The use of a priority queue here is very convenient: we push
# (time to wake up, task) into the queue. When we get/pop an item from
# the queue, we automatically get the fisrt task to wake up together
# with its wakeup time.
heapq.heappush(self._sleeping, (time.monotonic() + seconds, task))
def read_wait(self, resource, task):
self._selector.register(resource, EVENT_READ, task)
def write_wait(self, resource, task):
self._selector.register(resource, EVENT_WRITE, task)
# Socket stuff
def sock_recv(self, sock, maxsize):
yield 'read_wait', sock
return sock.recv(maxsize)
def sock_sendall(self, sock, data):
while data:
try:
n = sock.send(data)
data = data[n:]
except BlockingIOError:
yield 'write_wait', sock
def sock_accept(self, sock):
yield 'read_wait', sock
return sock.accept()
def sock_connect(self, sock, address):
yield 'write_wait', sock
return sock.connect(address)
class EventLoop:
def __init__(self):
self.tasks = deque()
self.selector = DefaultSelector()
def run_until_complete(self, task):
self.tasks.append(task)
self.run()
def pause(self):
return "pause", None
def schedule(self, target):
return "schedule", target
def sock_accept(self, sock):
yield ("read", sock)
return sock.accept()
def sock_recv(self, sock):
yield ("read", sock)
return sock.recv(1024)
def sock_sendall(self, sock, data):
yield ("write", sock)
return sock.sendall(data)
def start_server(self, handler, host, port, backlog=0):
handler = partial(handler, self)
with socket.socket() as sock:
sock.bind((host, port))
sock.listen(backlog)
print("Listening on {}:{}".format(host, port))
while True:
conn, addr = yield from self.sock_accept(sock)
print("Accepted client from", addr)
yield self.schedule(handler(conn))
def run(self):
while self.tasks or self.selector.get_map():
for _ in range(len(self.tasks)):
try:
task = self.tasks.popleft()
tag, value = next(task)
if tag == "schedule":
self.tasks.append(value)
self.tasks.append(task)
elif tag == "read":
self.selector.register(value, EVENT_READ, data=task)
elif tag == "write":
self.selector.register(value, EVENT_WRITE, data=task)
elif tag == "pause":
self.tasks.append(task)
else:
raise ValueError("Incorrect tag")
except StopIteration:
continue
if self.selector.get_map():
for key, event in self.selector.select():
if event & EVENT_READ or event & EVENT_WRITE:
self.tasks.append(key.data)
self.selector.unregister(key.fileobj)
def __init__(self):
self._tasks = deque()
self._sleeping = []
self._stop = False
self._selector = DefaultSelector()
def __init__(self):
self.ready = deque()
self.selector = DefaultSelector()
#python3.4 or higher is required in order to use selectors
import socket
from selectors import DefaultSelector, EVENT_READ, EVENT_WRITE
from datetime import datetime
sel = DefaultSelector()
request = "GET / HTTP/1.0\r\n\r\n"
request_callback = lambda sock: sock.send(request.encode())
response_callback = lambda sock: \
print((sock.recv(1000)).decode().split("\n")[-1], 'finish at %s' % str(datetime.now())[:-7])
def get():
sock = socket.socket()
sock.setblocking(False)
try:
sock.connect(('localhost', 3000))
except:
pass
sel.register(sock, EVENT_WRITE, lambda: request_callback(sock))
get_count = 4
print("starts at %s" % str(datetime.now())[:-7])
for i in range(get_count):
get()
while True:
events = sel.select()
for key, _ in events:
callback = key.data
event = key.events
def __init__(self):
self.ready = deque()
self.selector = DefaultSelector()
self.futures = {}
def distribute(cmd, max_bytes, max_procs, chunk_size, round_robin, verbose):
"""
Blocking function that manages all the delegation of chunks from
stdin to subprocesses, spawning of subprocess, and collation of
stdout and stderr from each subprocess.
Broadly speaking, gatling reads chunks of data from stdin and
disperses those among subprocesses to achieve parallel execution.
Stdin is read in chunks of chunk_size bytes and truncated to the
last newline, keeping the remainder to be prepended on the next
chunk. Multiple chunks without newline is allowed, nothing is
passed on until a newline is found or stdin closes.
The stdout and stderr from each child is read in a similar
fashion. Whenever a newline is found the output is written onto
stdout or stderr, respectively. This collation preserves the
format of the output, but only weakly adheres to the chronology.
The output from gatling is the result of the subprocesses with a
line-by-line integrity preserved with no guarantee that the order
is exactly maintained.
There are two different behaviors for subprocess spawning,
manifold or gatling:
* In manifold-mode each new chunk read from stdin spawns a new
subprocess until max_procs is reached and then each of the
subprocesses are fed a chunk in round robin fashion. This is an
excellent model for programs that do not tax an external
resource and programs that can act on stdin as soon as it is
available.
* In gatling-mode chunks are fed to a single subprocess until that
processes' max_bytes is reached, the subprocess' stdin is closed
and on the next chunk a new subprocess is spawned and fed until
its max_bytes is reached and so on until max_procs is reached.
If max_procs is reached gatling is blocked until a subprocess
finishes. This mode works well for programs that connect to an
external service or programs that don't start processing stdin
until it is closed.
cmd (list): The full command line for spawning subprocesses.
max_bytes (int): Maximum bytes to pass to each subprocess before
closing the subprocess' stdin. Increase this value if
subprocesses do not have memory management problems from large
input and if subprocesses process stdin as stream. Decrease
this value if programs can not handle a large input set on
stdin or if programs do not start processing until stdin is
closed.
max_procs (int): Maximum number of simultaneos subprocesses.
Increase this number if the subprocess is largely CPU bound,
decrease it to match the hardware if the subprocesses are IO
bound.
chunk_size (int): Stdin content streamed to gatling is consumed in
chunks of this size (bytes) for efficiency reasons. Originally
it was line by line, but that was too slow to keep
subprocesses fed continuously. To force line-by-line behavior,
set chunk to a size always less than the length of an input
line (worst case 1, but try to keep it as high as possible).
Experiment with this number to maximize pipeline throughput.
round_robin (bool): True is manifold, false is gatling, see above
for details.
"""
from sys import stdin, stdout, stderr
from subprocess import Popen, PIPE
from time import sleep, time
from selectors import DefaultSelector, EVENT_READ
from threading import Thread
from fcntl import fcntl, F_SETFL, F_GETFL
from os import O_NONBLOCK
def sink(selector, not_done):
n = 0
while len(not_done) > 1 or list(selector.get_map()):
for (fileobj, _, _, p), _ in selector.select():
chunk = fileobj.read(4096)
if chunk:
i = chunk.rfind('\n') + 1
if i:
n += fileobj._trg.write(fileobj._buf)
n += fileobj._trg.write(chunk[:i])
fileobj._buf = chunk[i:]
else:
fileobj._buf += chunk
else:
if p.returncode is not None:
n += fileobj._trg.write(fileobj._buf)
selector.unregister(fileobj)
not_done[0] = n
my_name = 'manifold' if round_robin else 'gatling'
selector = DefaultSelector()
res = []
p_filled = [] # Child processes that have had their maximum input supplied
# (p_open) Child processes that can take more input
# In non-round-robin mode, there will only be one such child,
# which will continually be popped and re-appended.
# In round-robin mode, this list is rotated each time there is
# a buffer to be written to a child.
p_open = []
b_in = 0
buf = ''
not_done = [0, 1]
sel_t = None
t0 = time()
try:
for chunk in iter(lambda: stdin.read(chunk_size), ''):
b_in += len(chunk)
i = chunk.rfind('\n') + 1
if i:
p = None
if round_robin:
if len(p_open) + len(p_filled) == max_procs:
p = p_open.pop(0)
else:
if p_open:
p = p_open.pop(0)
if not p:
if verbose:
running = len(p_filled) + len(p_open)
print(
f"# {my_name} STARTED A PROCESS (1 + {running} + {len(res)}):",
*cmd,
file=sys.stderr)
p = Popen(cmd,
encoding=stdout.encoding,
stdin=PIPE,
stdout=PIPE,
stderr=PIPE)
p._n = 0
p._t0 = time()
for fo, trg in [(p.stdout, stdout), (p.stderr, stderr)]:
fo._buf = ''
fo._trg = trg
fcntl(fo, F_SETFL, fcntl(fo, F_GETFL) | O_NONBLOCK)
selector.register(fo, EVENT_READ, p)
if not sel_t:
sel_t = Thread(target=sink, args=(selector, not_done))
sel_t.daemon = True
sel_t.start()
p._n += p.stdin.write(buf)
p._n += p.stdin.write(chunk[:i])
buf = chunk[i:]
if p._n >= max_bytes:
t1 = time()
p._t1 = t1
td = (t1 - t0) * 1024
ptd = (t1 - p._t0) * 1024
p.stdin.close()
p_filled.append(p)
if verbose and ptd and td:
running = len(p_filled) + len(p_open)
print(
f"# {my_name} PROCESS LIMIT {p._n:,}/{max_bytes:,}",
f"({p._n/ptd:.2f} kb/s).",
f"INPUT: {b_in:,} ({b_in/td:.2f} kb/s) OUTPUT: {not_done[0]:,}.",
f"PROCESSES: {running}/{running+len(res)}",
file=sys.stderr)
while len(p_filled) == max_procs:
done = [d for d in p_filled if d.poll() is not None]
if done and verbose:
print(f"# {my_name} CLOSED {len(done)} PROCESSES",
file=sys.stderr)
for d in done:
if verbose and p._t0 != t1:
print(
f"# {my_name} CLOSED PROCESS INPUT: {p._n:,} TIME:",
f"{t1-p._t0:.1f}/{t1-p._t1:.1f}",
f"KB/S: {p._n/(t1-p._t0)/1024:.2f}",
file=sys.stderr)
p_filled.remove(d)
res.append(d.returncode)
if not done:
sleep(0.5)
else:
p_open.append(p)
else:
buf += chunk
for p in p_open:
p.stdin.write(buf)
buf = ''
p.stdin.close()
p_filled.append(p)
except (KeyboardInterrupt, SystemExit):
not_done.pop()
for p in p_open:
p.stdin.close()
p.kill()
for d in p_filled:
d.kill()
raise
while p_filled:
res.append(p_filled.pop(0).wait())
if sel_t:
not_done.pop()
sel_t.join()
selector.close()
return res