class Select(object):
'''创建selector,负责注册和取消注册socket读写事件、'''
def __init__(self):
self.selector = DefaultSelector()
self.clients = {}
def startListening(self, client):
# add client,monitor socked wtite event
self.clients[client.sock] = client
# 注册socket写事件
self.selector.register(client.sock, EVENT_WRITE, self.connected)
def connected(self, key):
# connect host,and monitor socket read event
self.selector.unregister(key.fileobj)
sock = key.fileobj
self.clients[sock].send_request()
self.selector.register(sock, EVENT_READ, self.do_read)
def do_read(self, key):
# read data ,if data is empty, then unregister socket and end
sock = key.fileobj
client = self.clients[sock]
if client.recv_response():
return
self.selector.unregister(sock)
client.end()
def wait(gen: PQGen[RV], timeout: Optional[float] = None) -> RV:
"""
Wait for a generator using the best option available on the platform.
:param gen: a generator performing database operations and yielding
(fd, `Ready`) pairs when it would block.
:param timeout: timeout (in seconds) to check for other interrupt, e.g.
to allow Ctrl-C.
:type timeout: float
:return: whatever *gen* returns on completion.
"""
sel = DefaultSelector()
try:
fd, s = next(gen)
while 1:
sel.register(fd, s)
ready = None
while not ready:
ready = sel.select(timeout=timeout)
sel.unregister(fd)
assert len(ready) == 1
fd, s = gen.send(ready[0][1])
except StopIteration as ex:
rv: RV = ex.args[0] if ex.args else None
return rv
class TorSocketLoop(threading.Thread):
def __init__(self, our_sock, send_func):
super().__init__(name='SocketLoop{:x}'.format(our_sock.fileno()))
self._our_sock = our_sock
self._send_func = send_func
self._do_loop = True
self._cntrl_r, self._cntrl_w = socket.socketpair()
self._selector = DefaultSelector()
self._selector.register(self._our_sock, EVENT_READ, self._do_recv)
self._selector.register(self._cntrl_r, EVENT_READ, self._do_stop)
def _do_recv(self, sock):
try:
data = sock.recv(1024)
self._send_func(data)
except ConnectionResetError:
logger.debug('Client was badly disconnected...')
def _do_stop(self, sock):
self._do_loop = False
def _cleanup(self):
self._selector.unregister(self._cntrl_r)
self._cntrl_w.close()
self._cntrl_r.close()
self.close_sock()
self._selector.close()
@property
def fileno(self):
if not self._our_sock:
return None
return self._our_sock.fileno()
def close_sock(self):
if not self._our_sock:
return
self._selector.unregister(self._our_sock)
# self._our_sock.shutdown(socket.SHUT_WR)
self._our_sock.close()
self._our_sock = None
def stop(self):
self._cntrl_w.send(b'\1')
def run(self):
logger.debug('Starting...')
while self._do_loop:
events = self._selector.select()
for key, _ in events:
callback = key.data
callback(key.fileobj)
self._cleanup()
logger.debug('Stopped...')
def append(self, data):
self._our_sock.send(data)
def wait_selector(gen: PQGen[RV],
fileno: int,
timeout: Optional[float] = None) -> RV:
"""
Wait for a generator using the best strategy available.
:param gen: a generator performing database operations and yielding
`Ready` values when it would block.
:param fileno: the file descriptor to wait on.
:param timeout: timeout (in seconds) to check for other interrupt, e.g.
to allow Ctrl-C.
:type timeout: float
:return: whatever *gen* returns on completion.
Consume *gen*, scheduling `fileno` for completion when it is reported to
block. Once ready again send the ready state back to *gen*.
"""
try:
s = next(gen)
sel = DefaultSelector()
while 1:
sel.register(fileno, s)
ready = None
while not ready:
ready = sel.select(timeout=timeout)
sel.unregister(fileno)
s = gen.send(ready[0][1])
except StopIteration as ex:
rv: RV = ex.args[0] if ex.args else None
return rv
class Loop:
def __init__(self):
self.sel = DefaultSelector()
self.queue = deque()
def create_task(self, coro):
sock = coro.send(None)
self.sel.register(sock, EVENT_WRITE, coro)
self.queue.append(coro)
def run(self):
while self.queue:
events = self.sel.select()
for key, _ in events:
coro = key.data
event = key.events
file_obj = key.fileobj
self.sel.unregister(file_obj)
try:
if event == EVENT_WRITE:
self.sel.register(file_obj, EVENT_READ, coro)
elif event == EVENT_READ:
self.sel.register(file_obj, EVENT_WRITE, coro)
coro.send(None)
except StopIteration:
self.queue.popleft()
except:
pass
else: return
def run_until_complete(task):
tasks = [(task, None)]
# NEW: prepare for I/O multiplexing.
selector = DefaultSelector()
# NEW: watch out, all tasks might be suspended at the same time.
while tasks or selector.get_map():
# NEW: poll I/O operation status and resume tasks when ready.
timeout = 0.0 if tasks else None
for key, events in selector.select(timeout):
tasks.append((key.data, None))
selector.unregister(key.fileobj)
queue, tasks = tasks, []
for task, data in queue:
try:
data = task.send(data)
except StopIteration:
pass
else:
# NEW: register for I/O and suspend the task.
if data and data[0] == EVENT_READ:
selector.register(data[1], EVENT_READ, task)
elif data and data[0] == EVENT_WRITE:
selector.register(data[1], EVENT_WRITE, task)
else:
tasks.append((task, None))
class Loop:
def __init__(self):
self.ready = deque()
self.selector = DefaultSelector()
self.futures = {}
def create_task(self, task):
self.ready.append(task)
def run_forever(self):
while True:
while not self.ready:
completed_futures = [future for future in self.futures if not future.running()]
for future in completed_futures:
self.ready.append(self.futures.pop(future))
# so select() is blocking. If set a negative time out it won't block.
events = self.selector.select(-1)
# add these socket events and unregister them from listened to:
for key, _ in events:
self.ready.append(key.data) # add the task to the ready queue
self.selector.unregister(key.fileobj)
while self.ready:
self.current_task = self.ready.popleft()
# try to run current_task...
try:
# run task to next yield point
reason, what = self.current_task.send(None)
# reason, what = self.current_task.send(None)
if reason == 'waiting_to_accept':
self.selector.register(what, EVENT_READ, self.current_task)
elif reason == 'waiting_to_read':
self.selector.register(what, EVENT_READ, self.current_task)
elif reason == 'waiting_to_write':
self.selector.register(what, EVENT_WRITE, self.current_task)
elif reason == 'waiting_for_future':
self.futures[what] = self.current_task
else:
raise RuntimeError(
'Something bad happened... er. reason={}'.format(reason))
except StopIteration:
pass
async def sock_recv(self, sock, maxbytes):
# wait to read from the socket
await read_wait(sock)
return sock.recv(maxbytes)
async def sock_accept(self, sock):
# wait to read/hear from the socket
await accept_wait(sock)
return sock.accept()
async def sock_sendall(self, sock, data):
while data:
# wait to be able to write to the socket
await write_wait(sock)
nsent = sock.send(data)
data = data[nsent:]
def run_in_executor(self, executor, func, *args):
return executor.submit(func, *args)
def wait_conn(gen: PQGenConn[RV], timeout: Optional[float] = None) -> RV:
"""
Wait for a connection generator using the best strategy available.
:param gen: a generator performing database operations and yielding
(fd, `Ready`) pairs when it would block.
:param timeout: timeout (in seconds) to check for other interrupt, e.g.
to allow Ctrl-C.
:type timeout: float
:return: whatever *gen* returns on completion.
Behave like in `wait()`, but take the fileno to wait from the generator
itself, which might change during processing.
"""
try:
fileno, s = next(gen)
sel = DefaultSelector()
while 1:
sel.register(fileno, s)
ready = None
while not ready:
ready = sel.select(timeout=timeout)
sel.unregister(fileno)
fileno, s = gen.send(ready[0][1])
except StopIteration as ex:
rv: RV = ex.args[0] if ex.args else None
return rv
def run_until_complete(task):
tasks = [(task, None)]
selector = DefaultSelector()
while tasks or selector.get_map():
print("___________________________________")
# poll I/O and resume when ready
timeout = 0.0 if tasks else None
for key, events in selector.select(timeout):
tasks.append((key.data, None))
selector.unregister(key.fileobj)
queue, tasks = tasks, []
for task, data in queue:
try:
print(f"loop: send {data} into {task.__name__}")
data = task.send(data)
print(f"loop: received {data} from {task.__name__}")
except StopIteration as res:
pass
except Exception as exp:
print(repr(exp))
else:
if data:
req, _ = data
if req == EVENT_READ:
stream = data[1]
selector.register(stream, EVENT_READ, task)
elif req == EVENT_WRITE:
stream = data[1]
selector.register(data[1], EVENT_WRITE, task)
else:
tasks.append((task, None))
def run_until_complete(task):
tasks = [(task, None)]
selector = DefaultSelector()
while tasks or selector.get_map():
timeout = 0 if tasks else None
if selector.get_map():
for key, events in selector.select(timeout):
tasks.append((key.data, None))
selector.unregister(key.fileobj)
queue, tasks = tasks, []
for task, data in queue:
try:
data = task.send(data)
except StopIteration:
pass
else:
if data and data[0] == EVENT_READ:
selector.register(data[1], EVENT_READ, task)
elif data and data[0] == EVENT_WRITE:
selector.register(data[1], EVENT_WRITE, task)
elif data and data[0] == 'spawn':
tasks.append((data[1], None))
tasks.append((task, data[1]))
else:
tasks.append((task, None))
class Loop:
def __init__(self):
self.ready = deque()
self.selector = DefaultSelector()
async def sock_recv(self, sock, maxbytes):
await read_wait(sock)
return sock.recv(maxbytes)
async def sock_accept(self, sock):
await read_wait(sock)
return sock.accept()
#old version
"""
async def sock_sendall(self, sock, data):
while data:
await write_wait(sock)
nsent = sock.send(data)
data = data[nsent:]
"""
#speed up version
async def sock_sendall(self, sock, data):
while data:
try:
nsent = sock.send(data)
data = data[nsent:]
except BlockingIOError:
await write_wait(sock)
def create_task(self, coro):
self.ready.append(coro)
def run_forever(self):
while True:
while not self.ready:
events = self.selector.select()
for key, _ in events:
self.ready.append(key.data)
self.selector.unregister(key.fileobj)
while self.ready:
self.current_task = self.ready.popleft()
try:
op, *args = self.current_task.send(None)
getattr(self, op)(*args)
except StopIteration:
pass
def read_wait(self, sock):
self.selector.register(sock, EVENT_READ, self.current_task)
def write_wait(self, sock):
self.selector.register(sock, EVENT_WRITE, self.current_task)
def get_clean_paths(max_wait: int, arg_paths):
from os import getpid
from selectors import DefaultSelector, EVENT_READ
from socket import socket
from time import monotonic
from filelock import FileLock
from psutil import Process, NoSuchProcess
from ds_tools.fs.paths import get_user_cache_dir
cache_dir = Path(get_user_cache_dir('music_manager'))
with FileLock(cache_dir.joinpath('init.lock').as_posix()):
pid = getpid()
active_path = cache_dir.joinpath('active_pid_port.txt')
try:
with active_path.open('r') as f:
active_pid, port = map(int, f.read().split(','))
except OSError:
active = True
else:
try:
active = not Process(active_pid).is_running()
except NoSuchProcess:
active = True
sock = socket()
if active:
sock.bind(('localhost', 0))
sock.listen(100)
sock.setblocking(False)
port = sock.getsockname()[1]
log.info(f'Primary instance with {pid=} {port=}')
with active_path.open('w') as f:
f.write(f'{pid},{port}')
else:
log.info(f'Follower instance with {pid=} {port=}')
if active:
paths = list(arg_paths)
selector = DefaultSelector()
def accept(sock, mask):
conn, addr = sock.accept()
conn.setblocking(False)
selector.register(conn, EVENT_READ, read)
def read(conn, mask):
if data := conn.recv(2000):
paths.append(data.decode('utf-8'))
# log.debug(f'Received path={data!r} from other instance')
else:
selector.unregister(conn)
conn.close()
class Loop:
def __init__(self):
# queue of ready tasks
self.ready = deque()
# interface for the operating system's `select` calls,
# informing which file handles (sockets) are ready to read/write
self.selector = DefaultSelector()
async def sock_recv(self, sock, maxbytes):
await read_wait(sock)
return sock.recv(maxbytes)
async def sock_accept(self, sock):
await read_wait(sock)
return sock.accept()
async def sock_sendall(self, sock, data):
while data:
try:
nsent = sock.send(data)
data = data[nsent:]
except BlockingIOError:
await write_wait(sock)
def create_task(self, coro):
self.ready.append(coro)
def run_forever(self):
while True:
while not self.ready:
events = self.selector.select()
for key, _ in events:
self.ready.append(key.data)
self.selector.unregister(key.fileobj)
while self.ready:
self.current_task = self.ready.popleft()
try:
op, *args = self.current_task.send(
None) # Run to the yield
# call the operation yielded by the pending task
getattr(self, op)(*args)
except StopIteration:
pass
def read_wait(self, sock):
# The current_tasks expects a signal that the socket `sock` is readable
self.selector.register(sock, EVENT_READ, self.current_task)
def write_wait(self, sock):
self.selector.register(sock, EVENT_WRITE, self.current_task)
class EventLoop:
def __init__(self, server_id, handler: Handler):
self.select = DefaultSelector()
self.server_id = server_id
self.handler = handler
def register(self, fd: Socket, events: SelectorKey, data=None):
fd.setblocking(False)
self.select.register(fd, events, data)
def modify(self, fd: Socket, events: SelectorKey, data=None):
self.select.modify(fd, events, data)
def unregister(self, fd):
self.select.unregister(fd)
def run(self):
self.register(self.server_id, selectors.EVENT_READ)
while True:
events = self.select.select()
for key, mask in events:
if mask == EVENT_READ:
if key.fileobj == self.server_id:
self.accept(key.fileobj)
else:
self.read(key.fileobj)
elif mask == EVENT_WRITE:
print(f"EventLoop, key={key}, data={key.data}")
self.write(key.fileobj, key.data)
def accept(self, sock):
"""sock 要处理的 fd, mask 为事件类型 READ or WRITE"""
conn = self.handler.accept(sock) # Should be ready
self.register(conn, EVENT_READ, self.read)
def read(self, conn):
for data in self.handler.read(conn):
if data and data not in ('q', 'Q'):
resp = self.handler.handle(conn, data)
self.modify(conn, EVENT_WRITE, resp)
else:
self.unregister(conn)
self.handler.close(conn)
def write(self, conn, data):
self.handler.send(conn, data)
self.modify(conn, EVENT_READ)
def close(self):
self.select.close()
class Loop:
def __init__(self):
self.ready = deque()
self.selector = DefaultSelector()
async def sock_recv(self, sock, maxbytes):
# wait for something to happen
await read_wait(sock)
return sock.recv(maxbytes)
async def sock_accept(self, sock):
# wait for something to happen
await read_wait(sock)
return sock.accept()
async def sock_sendall(self, sock, data):
while data:
await write_wait(sock)
nsent = sock.send(data)
data = data[nsent:]
def create_task(self, coro):
self.ready.append(coro)
def run_forever(self):
while True:
while not self.ready:
# if there's not nothing to ready to run, go get all IO
events = self.selector.select()
for key, _ in events:
self.ready.append(key.data)
self.selector.unregister(key.fileobj)
while self.ready:
self.current_task = self.ready.popleft()
try:
op, *args = self.current_task.send(
None) # Run to the yield
getattr(self, op)(*args) # Sneaky method
except StopIteration:
pass
def read_wait(self, sock):
# the current task is interested in reading on that socket
self.selector.register(sock, EVENT_READ, self.current_task)
def write_wait(self, sock):
# the current task is intertested in writing on that socket
self.selector.register(sock, EVENT_WRITE, self.current_task)
def wait(
gen: Union[PQGen[RV], PQGenConn[RV]], timeout: Optional[float] = None
) -> RV:
"""
Wait for a generator using the best option available on the platform.
:param gen: a generator performing database operations and yielding
(fd, `Ready`) pairs when it would block.
:param timeout: timeout (in seconds) to check for other interrupt, e.g.
to allow Ctrl-C.
:type timeout: float
:return: whatever *gen* returns on completion.
"""
fd: int
s: Wait
sel = DefaultSelector()
try:
# Use the first generated item to tell if it's a PQgen or PQgenConn.
# Note: mypy gets confused by the behaviour of this generator.
item = next(gen)
if isinstance(item, tuple):
fd, s = item
while 1:
sel.register(fd, s)
ready = None
while not ready:
ready = sel.select(timeout=timeout)
sel.unregister(fd)
assert len(ready) == 1
fd, s = gen.send(ready[0][1])
else:
fd = item # type: ignore[assignment]
s = next(gen) # type: ignore[assignment]
while 1:
sel.register(fd, s)
ready = None
while not ready:
ready = sel.select(timeout=timeout)
sel.unregister(fd)
assert len(ready) == 1
s = gen.send(ready[0][1]) # type: ignore[arg-type,assignment]
except StopIteration as ex:
rv: RV = ex.args[0] if ex.args else None
return rv
class Loop:
def __init__(self):
self.ready = deque()
self.selector = DefaultSelector()
async def sock_recv(self, sock, maxbytes):
await read_wait(sock)
return sock.recv(maxbytes)
async def sock_accept(self, sock):
await read_wait(sock)
return sock.accept()
async def sock_sendall(self, sock, data):
while data:
try:
nsent = sock.send(data)
data = data[nsent:]
except BlockingIOError:
await write_wait(sock)
def create_task(self, coro):
self.ready.append(coro)
def run_forever(self):
while True:
while not self.ready:
events = self.selector.select()
for key, _ in events:
self.ready.append(key.data)
self.selector.unregister(key.fileobj)
while self.ready:
self.current_task = self.ready.popleft()
try:
op, *args = self.current_task.send(None)
getattr(self, op)(*args)
except StopIteration:
pass
def read_wait(self, sock):
self.selector.register(sock, EVENT_READ, self.current_task)
def write_wait(self, sock):
self.selector.register(sock, EVENT_WRITE, self.current_task)
class Selector:
def __init__(self):
self._sel = DefaultSelector()
self._fds = {}
def register_read(self, task, fileno):
return self._register(task, fileno, EVENT_READ)
def register_write(self, task, fileno):
return self._register(task, fileno, EVENT_WRITE)
def _unregister(self, fd):
fileno = fd.fileno()
if fileno in self._fds:
LOG.debug('unregister fd %r', fd)
del self._fds[fileno]
self._sel.unregister(fileno)
def _register(self, task, fileno, events):
fd = self._fds.get(fileno, None)
if fd is None:
fd = KernelFd(self, fileno, task)
self._fds[fileno] = fd
if fd.task is not task:
raise RuntimeError(
'file descriptor already registered by other task')
if fd.selector_key is None:
LOG.debug('register fd %r, events=%r', fd, events)
fd.selector_key = self._sel.register(fd, events)
elif fd.events != events:
LOG.debug('modify fd %r, events=%r', fd, events)
fd.selector_key = self._sel.modify(fd, events)
return fd
def poll(self, timeout):
io_events = self._sel.select(timeout=timeout)
for key, mask in io_events:
fd = key.fileobj
if fd.task.is_alive and fd.events & mask:
LOG.debug('task %r wakeup by fd %r', fd.task, fd)
yield fd.task
def close(self):
self._sel.close()
class TorSocketLoop(threading.Thread):
def __init__(self, our_sock, send_func):
super().__init__(name='SocketLoop{:x}'.format(our_sock.fileno()))
self._our_sock = our_sock
self._send_func = send_func
self._do_loop = True
self._cntrl_r, self._cntrl_w = socket.socketpair()
self._selector = DefaultSelector()
self._selector.register(self._our_sock, EVENT_READ, self._do_recv)
self._selector.register(self._cntrl_r, EVENT_READ, self._do_stop)
def _do_recv(self, sock):
data = sock.recv(1024)
self._send_func(data)
def _do_stop(self, sock):
self._do_loop = False
def _cleanup(self):
self._selector.unregister(self._cntrl_r)
self._cntrl_w.close()
self._cntrl_r.close()
self._selector.unregister(self._our_sock)
self._our_sock.shutdown(socket.SHUT_WR)
self._our_sock.close()
self._selector.close()
def stop(self):
self._cntrl_w.send(b'\1')
def run(self):
logger.debug("Starting...")
while self._do_loop:
events = self._selector.select()
for key, _ in events:
callback = key.data
callback(key.fileobj)
self._cleanup()
logger.debug("Stopped...")
def append(self, data):
self._our_sock.send(data)
class Loop:
def __init__(self):
self.ready = deque()
self.selector = DefaultSelector()
def create_task(self, task):
self.ready.append(task)
def run_forever(self):
while True:
# hmmn, nothing to run -> must be waiting on stuff...
while not self.ready:
events = self.selector.select()
# add these events and unregister them from listened to:
for key, _ in events:
self.ready.append(key.data)
self.selector.unregister(key.fileobj)
while self.ready:
self.current_task = self.ready.popleft()
# try to run current_task...
try:
# run task to next yield point
reason, sock = next(self.current_task)
if reason == 'waiting_to_accept':
self.selector.register(sock, EVENT_READ, self.current_task)
elif reason == 'waiting_to_read':
self.selector.register(sock, EVENT_READ, self.current_task)
elif reason == 'waiting_to_write':
self.selector.register(sock, EVENT_WRITE, self.current_task)
else:
raise RuntimeError('Something bad happened... er. reason={}'.format(reason))
except StopIteration:
pass
def sock_recv(self, sock, maxbytes):
# wait to read from the socket
return sock.recv(maxbytes)
def sock_accept(self, sock):
# wait to read/hear from the socket
return sock.accept()
def sock_sendall(self, sock, data):
while data:
# wait to be able to write to the socket
nsent = sock.send(data)
data = data[nsent:]
class Loop:
def __init__(self):
self.ready = deque()
self.selectors = DefaultSelector() # a way to watch socket IO
async def sock_recv(self, sock, maxbytes):
await read_wait(sock) # wait for something happen
return sock.recv(maxbytes)
async def sock_accept(self, sock):
await read_wait(sock)
return sock.sock_accept()
async def sock_sendall(self, sock, data):
while data :
try:
nsent = sock.send(data)
data = data[nsent:] # send partial data
except BlockingIOError:
await write_wait(sock)
def create_task(self, coro):
self.ready.append(coro)
def run_forever(self):
while True:
while not self.ready:
events = self.selectors.select()
for key, _ in events:
self.ready.append(key.data)
self.selectors.unregister(key.fileobj)
while self.ready:
self.current_task = self.ready.popleft()
try:
op, *args = self.current_task.send(None) # run to the yield
getattr(self,op)(*args) # Sneaky method call
except StopIteration:
pass
def read_wait(self, sock):
self.selectors.register(sock, EVENT_READ, self.current_task)
def write_wait(self, sock):
self.selectors.register
class MySocket:
def get_conn(self):
self.sel = DefaultSelector()
self.host_port = ('0.0.0.0', 45633)
self.ss = socket.socket()
self.ss.setblocking(False)
self.ss.bind(self.host_port)
self.ss.listen(10)
self.sel.register(self.ss.fileno(), EVENT_READ, self.get_accept)
while 1:
ready = self.sel.select()
for key, mask in ready:
print(key.data)
call_back = key.data
call_back(key)
def get_accept(self, mask):
self.sel.unregister(mask.fd)
self.conn, self.addr = self.ss.accept()
self.conn.setblocking(False)
self.sel.register(self.conn.fileno(), EVENT_READ, self.get_recv)
def get_recv(self, mask):
# self.sel.unregister(mask.fd)
self.data = self.conn.recv(1024)
if self.data:
# self.conn.send(self.data)
self.sel.unregister(mask.fd)
self.sel.register(self.conn.fileno(), EVENT_WRITE, self.get_send)
else:
self.conn.close()
self.sel.unregister(mask.fd)
def get_send(self, mask):
self.sel.unregister(mask.fd)
self.conn.send(self.data)
self.sel.register(self.conn.fileno(), EVENT_READ, self.get_recv)
class localKeyboard:
def __init__(self):
self.headUpDownAngle = 90
self.headLeftRightAngle = 90
self.PWM = Motor()
self.servo = Servo()
self.horn = Buzzer()
self.speed = 1000
# corrected servo positions
# adjust these to suit your car
# so head is front and centre at start
self.headLRcorrect = -3
self.headUDcorrect = 4
self.reset_head()
self.selector = DefaultSelector()
# set true for mecanum wheels
self.mecanum = False
self.useLights = True
self.led = Led()
self.mouse = evdev.InputDevice('/dev/input/event1')
self.keybd = evdev.InputDevice('/dev/input/event0')
self.readingKeys = False
self.led.colorWipe(self.led.strip, Color(0,0,0),0)
self.brake = False
self.reverse = False
self.indicating = False
self.leftTurn = False
self.rightTurn = False
self.moving = False
self.indi_time = datetime.now()
self.indi_off = True
self.brake_time = datetime.now()
self.brake_off = True
atexit.register(self.keybd.ungrab) # Don't forget to ungrab the keyboard on exit!
atexit.register(self.mouse.ungrab)
self.keybd.grab() # Grab, i.e. prevent the keyboard from emitting original events.#
self.mouse.grab()
# This works because InputDevice has a `fileno()` method.
self.selector.register(self.mouse, EVENT_READ)
self.selector.register(self.keybd, EVENT_READ)
def read_keys_loop(self):
self.readingKeys = True
while self.readingKeys:
self.read_keys()
# only manage lights after a key press so brake lights, if on,
# will stay on until next key event
if self.useLights: self.manage_lights()
def manage_lights(self):
# indicators
if not self.indicating and not self.reverse:
self.led.colorWipe(self.led.strip, Color(0,0,0),0)
else:
if self.indicating:
if (datetime.now() - self.indi_time).microseconds > 250000:
self.indi_off = not self.indi_off
self.indi_time = datetime.now()
if self.indi_off:
if self.leftTurn:
self.led.strip.setPixelColor(2, Color(125,85,0) )
self.led.strip.setPixelColor(5, Color(125,85,0) )
if self.rightTurn:
self.led.strip.setPixelColor(1, Color(125,85,0) )
self.led.strip.setPixelColor(6, Color(125,85,0) )
self.led.strip.show()
else:
self.led.colorWipe(self.led.strip, Color(0,0,0),0)
if self.reverse:
self.led.strip.setPixelColor(1, Color(255,255,255) )
self.led.strip.setPixelColor(2, Color(255,255,255) )
self.led.strip.show()
if self.brake:
self.brake = False
if self.brake_off:
self.brake_off = False
self.brake_time = datetime.now()
self.led.strip.setPixelColor(1, Color(255,0,0) )
self.led.strip.setPixelColor(2, Color(255,0,0) )
self.led.strip.show()
if not self.brake_off:
#this is a minimum time on, they stay on until next key press
if (datetime.now() - self.brake_time).microseconds > 250000:
self.led.colorWipe(self.led.strip, Color(0,0,0),0)
self.brake = False
self.brake_off = True
def read_keys(self):
for key, mask in self.selector.select():
device =key.fileobj
for event in device.read():
if event.type == evdev.ecodes.EV_KEY:
# print("key press")
# print(evdev.ecodes.bytype[evdev.ecodes.EV_KEY][event.code])
if event.value == 1 or event.value == 2:
self.key_press(event, self.keybd)
elif event.value == 0:
self.drive_stop()
elif event.type == evdev.ecodes.EV_REL:
if event.code == evdev.ecodes.REL_X:
if event.value < 0:
self.head_left()
else:
self.head_right()
if event.code == evdev.ecodes.REL_Y:
if event.value < 0:
self.head_down()
else:
self.head_up()
else:
pass
#print(event)
def key_press(self, ev, kbd):
if (ev.value == 1 or ev.value == 2): # 1 PRESS or 2 HOLD
# EVENTS CALLED ON PRESS AND ON HOLD
# HEAD POSITION
if ev.code == evdev.ecodes.KEY_Z: self.head_down()
elif ev.code == evdev.ecodes.KEY_A: self.head_left()
elif ev.code == evdev.ecodes.KEY_S: self.head_right()
elif ev.code == evdev.ecodes.KEY_W: self.head_up()
# HORN
elif ev.code == evdev.ecodes.KEY_T: self.toot()
# not interested in any other held keys
elif ev.value == 2: pass
#EVENTS THAT SHOULD ONLY BE CALLED ON PRESS AND NOT HOLD
# SPEED SETTING
elif ev.code == evdev.ecodes.KEY_1: self.speed = 1000
elif ev.code == evdev.ecodes.KEY_2: self.speed = 1200
elif ev.code == evdev.ecodes.KEY_3: self.speed = 1400
elif ev.code == evdev.ecodes.KEY_4: self.speed = 1700
elif ev.code == evdev.ecodes.KEY_5: self.speed = 2000
elif ev.code == evdev.ecodes.KEY_6: self.speed = 2400
elif ev.code == evdev.ecodes.KEY_7: self.speed = 2800
elif ev.code == evdev.ecodes.KEY_8: self.speed = 3200
elif ev.code == evdev.ecodes.KEY_9: self.speed = 3600
elif ev.code == evdev.ecodes.KEY_0: self.speed = 4000
# DRIVE FUNCTIONS
elif ev.code == evdev.ecodes.KEY_UP: self.drive_forward()
elif ev.code == evdev.ecodes.KEY_DOWN: self.drive_backward()
elif ev.code == evdev.ecodes.KEY_LEFT: self.turn_left()
elif ev.code == evdev.ecodes.KEY_RIGHT: self.turn_right()
elif ev.code == evdev.ecodes.KEY_COMMA: self.crab_left()
elif ev.code == evdev.ecodes.KEY_DOT: self.crab_right()
elif ev.code == evdev.ecodes.KEY_SEMICOLON: self.diag_right()
elif ev.code == evdev.ecodes.KEY_K: self.diag_left()
elif ev.code == evdev.ecodes.KEY_SLASH: self.diag_rev_right()
elif ev.code == evdev.ecodes.KEY_M: self.diag_rev_left()
elif ev.code == evdev.ecodes.KEY_U: self.curve_right()
elif ev.code == evdev.ecodes.KEY_Y: self.curve_left()
elif ev.code == evdev.ecodes.KEY_J: self.curve_rev_right()
elif ev.code == evdev.ecodes.KEY_H: self.curve_rev_left()
# USE OR DONT USE LIGHTS
elif ev.code == evdev.ecodes.KEY_L:
self.useLights = not self.useLights
if not self.useLights: self.led.colorWipe(self.led.strip, Color(0,0,0),0)
# RESET TO START STATE
elif ev.code == evdev.ecodes.KEY_HOME: #RESET TO START STATE
self.drive_stop()
self.servo.setServoPwm('0', int(self.headLeftRightAngle))
self.servo.setServoPwm('1', int(self.headUpDownAngle))
self.speed = 1000
self.led.colorWipe(self.led.strip, Color(0,0,0),0)
# PROG FUNCTIONS
elif ev.code == evdev.ecodes.KEY_LEFTMETA: self.close()
elif ev.code == evdev.ecodes.KEY_END: self.shutdown_pi()
elif ev.code == evdev.ecodes.KEY_SYSRQ: self.reboot_pi()
else:
print("UNUSED KEY CODE")
print(evdev.ecodes.bytype[evdev.ecodes.EV_KEY][ev.code])
if ev.value == 0:
self.drive_stop()
# flush backed up key presses
while kbd.read_one() is not None:
if ev.value == 0 :
self.drive_stop()
def close(self):
self.readingKeys = False
self.selector.unregister(self.mouse)
self.selector.unregister(self.keybd)
self.led.colorWipe(self.led.strip, Color(0,0,0),0)
# kbd should be ungrabbed by atexit
# but belt and braces
try:
self.keybd.ungrab
self.mouse.ungrab
except:
pass
sys.exit()
def shutdown_pi(self):
self.readingKeys = False
self.toot()
time.sleep(0.2)
self.toot()
call("sudo nohup shutdown -h now", shell=True)
def reboot_pi(self):
self.readingKeys = False
self.toot()
call("sudo nohup reboot", shell=True)
def toot(self):
self.horn.run('1')
time.sleep(0.2)
self.horn.run('0')
def drive_forward(self):
self.moving = True
PWM.setMotorModel(self.speed, self.speed, self.speed, self.speed)
def turn_left(self):
self.moving = True
self.indicating = True
self.leftTurn = True
self.rightTurn = False
PWM.setMotorModel(-self.speed, -self.speed, self.speed, self.speed)
def drive_backward(self):
self.moving = True
self.reverse = True
PWM.setMotorModel(-self.speed, -self.speed, -self.speed, -self.speed)
def turn_right(self):
self.moving = True
self.indicating = True
self.leftTurn = False
self.rightTurn = True
PWM.setMotorModel(self.speed, self.speed, -self.speed, -self.speed)
def curve_left(self, biasPcent=20):
self.moving = True
PWM.setMotorModel(int(self.speed * (100 - biasPcent) / 100), int(self.speed * (100 - biasPcent) / 100),
int(self.speed * (100 + biasPcent) / 100), int(self.speed * (100 + biasPcent) / 100))
def curve_right(self, biasPcent=20):
self.moving = True
PWM.setMotorModel(int(self.speed * (100 + biasPcent) / 100), int(self.speed * (100 + biasPcent) / 100),
int(self.speed * (100 - biasPcent) / 100), int(self.speed * (100 - biasPcent) / 100))
def curve_rev_left(self, biasPcent=20):
self.moving = True
self.reverse = True
PWM.setMotorModel(-int(self.speed * (100 - biasPcent) / 100), -int(self.speed * (100 - biasPcent) / 100),
-int(self.speed * (100 + biasPcent) / 100), -int(self.speed * (100 + biasPcent) / 100))
def curve_rev_right(self, biasPcent=20):
self.moving = True
self.reverse = True
PWM.setMotorModel(-int(self.speed * (100 + biasPcent) / 100), -int(self.speed * (100 + biasPcent) / 100),
-int(self.speed * (100 - biasPcent) / 100), -int(self.speed * (100 - biasPcent) / 100))
def crab_left(self): #REQUIRES MECANUM WHEELS
if self.mecanum:
self.moving = True
self.indicating = True
self.leftTurn = True
self.rightTurn = False
PWM.setMotorModel(-self.speed, self.speed, self.speed, -self.speed)
def crab_right(self): #REQUIRES MECANUM WHEELS
if self.mecanum:
self.moving = True
self.indicating = True
self.leftTurn = False
self.rightTurn = True
PWM.setMotorModel(self.speed, -self.speed, -self.speed, self.speed)
def diag_right(self): #REQUIRES MECANUM WHEELS
if self.mecanum:
self.moving = True
PWM.setMotorModel(self.speed, 0, 0, self.speed)
def diag_left(self): #REQUIRES MECANUM WHEELS
if self.mecanum:
self.moving = True
PWM.setMotorModel(0, self.speed, self.speed, 0)
def diag_rev_left(self): #REQUIRES MECANUM WHEELS
if self.mecanum:
self.moving = True
self.reverse = True
PWM.setMotorModel(-self.speed, 0, 0, -self.speed)
def diag_rev_right(self): #REQUIRES MECANUM WHEELS
if self.mecanum:
self.moving = True
self.reverse = True
PWM.setMotorModel(0, -self.speed, -self.speed, 0)
def drive_stop(self):
if self.moving:
self.brake = True
self.moving = False
PWM.setMotorModel(0, 0, 0, 0)
self.reverse = False
self.indicating = False
self.leftTurn = False
self.rightTurn = False
def head_up(self):
self.headUpDownAngle += 1
if self.headUpDownAngle > 180 + self.headUDcorrect:
self.headUpDownAngle = 180 + self.headUDcorrect
self.servo.setServoPwm('1', self.headUpDownAngle)
# print("Up/down " + str(self.headUpDownAngle))
def head_down(self):
self.headUpDownAngle -= 1
if self.headUpDownAngle < 80 + self.headUDcorrect:
self.headUpDownAngle = 80 + self.headUDcorrect
self.servo.setServoPwm('1', self.headUpDownAngle)
# print("Up/down " + str(self.headUpDownAngle))
def head_left(self):
self.headLeftRightAngle -= 1
if self.headLeftRightAngle < 10 + self.headLRcorrect:
self.headLeftRightAngle = 10 + self.headLRcorrect
self.servo.setServoPwm('0', self.headLeftRightAngle)
# print("Left/Right " + str(self.headLeftRightAngle))
def head_LRpos(self, angle):
# print("Move head to " + str(self.headLeftRightAngle))
self.headLeftRightAngle = angle + self.headLRcorrect
self.servo.setServoPwm('0', self.headLeftRightAngle)
def head_right(self):
self.headLeftRightAngle += 1
if self.headLeftRightAngle > 170 + self.headLRcorrect:
self.headLeftRightAngle = 170 + self.headLRcorrect
self.servo.setServoPwm('0', self.headLeftRightAngle)
# print("Left/Right " + str(self.headLeftRightAngle))
def reset_head(self):
self.headLeftRightAngle = 90 + self.headLRcorrect
self.headUpDownAngle = 90 + self.headUDcorrect
self.servo.setServoPwm('0', int(self.headLeftRightAngle))
self.servo.setServoPwm('1', int(self.headUpDownAngle))
class TCPServer:
def __init__(self, port):
self.port = port
self.sock = None
self.selector = DefaultSelector()
# Advance self.listen() method. A future has been attached to the
# server socket, that has been registered in selector. And a callback
# has been attached to this future.
Task(self.listen())
def run(self):
while True:
ready = self.selector.select()
for key, events in ready:
fut = key.data
fut.resolve()
def listen(self):
server_sock = self._create_socket()
with server_sock:
try:
client_sock, remote_addr = self.sock.accept()
except BlockingIOError:
pass
fut = Future()
self.selector.register(self.sock, EVENT_READ, fut)
while True:
yield fut
Task(self.handle_client())
def handle_client(self):
client_sock, remote_addr = self.sock.accept()
client_sock.setblocking(False)
client_name = '{} {}'.format(*client_sock.getpeername())
print('Connected', client_name)
# Register client socket for reading and wait for this event
fut = Future()
self.selector.register(client_sock, EVENT_READ, fut)
yield fut
buffer = []
while True:
try:
chunk = client_sock.recv(1024)
buffer.append(chunk)
except BlockingIOError:
# Yield the same future, so we don't need to register
# it again. By the way, callback is the same for all
# futures, so it doesn't matter in my case.
yield fut
continue
print('Received chunk from client', client_name)
if chunk.endswith(b'end'):
print('Got all data from client', client_name)
request = b''.join(buffer).decode('utf8')
response = (request.upper() * 100).encode('utf8')
self.selector.unregister(client_sock)
fut = Future()
self.selector.register(client_sock, EVENT_WRITE, fut)
break
# This future was attached to the client_sock
yield fut
while True:
try:
bytes_sent = client_sock.send(response)
print(bytes_sent, 'bytes sent to client', client_name)
response = response[bytes_sent:]
except BlockingIOError:
print('Socket blockes for', client_name)
yield fut
continue
else:
if not response:
# All response sent
print('Response sent to client', client_name)
self.selector.unregister(client_sock)
client_sock.close()
break
def _create_socket(self):
self.sock = socket.socket()
self.sock.setsockopt(socket.SOL_SOCKET, socket.SO_SNDBUF, 10)
self.sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
self.sock.bind(('', self.port))
self.sock.setblocking(False)
self.sock.listen(10)
self.responses = {}
return self.sock
class EventLoop:
"""event loop, handles IO"""
def __init__(self):
self.co_result = {}
self.selector = DefaultSelector()
def run_until_complete(self, coroutine):
self._continue(coroutine, None)
while coroutine not in self.co_result:
self._tick()
result = self.co_result[coroutine]
del self.co_result[coroutine]
return result
def run_forever(self):
while True:
self._tick()
def start_coroutine(self, co):
self._continue(co, None)
def _continue(self, co, awaited):
try:
cmd = co.send(awaited)
except StopIteration as e:
self.co_result[co] = e.value
else:
if isinstance(cmd, EventLoopCommand.StreamAccept):
self._register(EVENT_READ, cmd.sock, (cmd, co))
elif isinstance(cmd, EventLoopCommand.StreamConnect):
try:
cmd.sock.connect(cmd.addr)
except BlockingIOError:
self._register(EVENT_WRITE, cmd.sock, (cmd, co))
except:
raise
else:
# connect finished immediately
self._continue(co, None)
elif isinstance(cmd, EventLoopCommand.StreamRecv):
self._register(EVENT_READ, cmd.sock, (cmd, co))
elif isinstance(cmd, EventLoopCommand.StreamSend):
self._register(EVENT_WRITE, cmd.sock, (cmd, co))
else:
raise RuntimeError('unknown cmd')
def _register(self, event, sock, data):
try:
key = self.selector.get_key(sock)
except KeyError:
self.selector.register(sock, event, {event: data})
else:
assert key.events & event == 0
key.data[event] = data
self.selector.modify(sock, key.events | event, key.data)
def _unregister(self, event, sock):
key = self.selector.get_key(sock)
assert key.events & event != 0
new_mask = key.events & ~event
del key.data[event]
if new_mask == 0:
self.selector.unregister(sock)
else:
self.selector.modify(sock, new_mask, key.data)
def _tick(self):
for key, mask in self.selector.select(1.0):
if mask & EVENT_WRITE != 0:
cmd, co = key.data[EVENT_WRITE]
self._unregister(EVENT_WRITE, cmd.sock)
if isinstance(cmd, EventLoopCommand.StreamConnect):
self._continue(co, cmd.sock.connect(cmd.addr))
elif isinstance(cmd, EventLoopCommand.StreamSend):
self._continue(co, cmd.sock.send(cmd.buf))
else:
raise RuntimeError('unknown cmd')
elif mask & EVENT_READ != 0:
cmd, co = key.data[EVENT_READ]
self._unregister(EVENT_READ, cmd.sock)
if isinstance(cmd, EventLoopCommand.StreamAccept):
self._continue(co, cmd.sock.accept())
elif isinstance(cmd, EventLoopCommand.StreamRecv):
self._continue(co, cmd.sock.recv(cmd.buf_len))
else:
raise RuntimeError('unknown cmd')
class Connection(LibConnection):
# Reference to the handling app on the side of the server
TYPE_HANDLER = 'client-handler'
LL_EVENT_ACCEPT = 'accept'
LL_EVENT_READ = 'read'
LL_EVENT_WRITE = 'write'
LL_EVENT_DISCONNECTED = 'disconnected'
_sel: DefaultSelector = None
_perform_select: bool = True
_subscribed_events: dict = None
_t_selecting_events: Thread = None
_t_read: Thread = None
_t_write: Thread = None
_read_lock: Lock = None
_write_lock: Lock = None
def __init__(self, *args, **kwargs):
super(Connection, self).__init__(*args, **kwargs)
self._subscribed_events = {
self.LL_EVENT_ACCEPT: list(),
self.LL_EVENT_READ: list(),
self.LL_EVENT_WRITE: list(),
self.LL_EVENT_DISCONNECTED: list(),
}
def unsubscribe_from_event(self, event_name: str, cb: callable) -> bool:
if event_name not in self._subscribed_events:
return False
if cb in self._subscribed_events[event_name]:
self._subscribed_events[event_name].remove(cb)
return True
def subscribe_to_event(self, event_name: str, cb: callable) -> bool:
"""
Subscribing to low-level communication events (accept, read, write).
:param event_name: One of: 'accept', 'read' or 'write'
:param cb: Callback that should be used for the subscription
:return: Returns True in case of successful subscription or False otherwise (Basically indication if subscribed)
"""
if event_name not in self._subscribed_events:
return False
if cb not in self._subscribed_events[event_name]:
self._subscribed_events[event_name].append(cb)
# Already subscribed
return True
def _after_open_connection(self):
super(Connection, self)._after_open_connection()
self._register_events()
self._t_selecting_events = Thread(target=self._selecting_events,
daemon=True)
def ready_to_operate(self):
"""
This method must be explicitly invoked after reconnection.
Otherwise the reading/writing to socket will work incorrectly
:return:
"""
self._t_selecting_events.start()
def _reading(self):
self._read_lock.acquire()
while self._is_connected:
# print('R')
self._read_lock.acquire()
self._invoke_events_callbacks(self.LL_EVENT_READ)
if self._read_lock.locked():
self._read_lock.release()
def _writing(self):
self._write_lock.acquire()
# print('[W2]', self._is_connected)
while self._is_connected:
# print('[W]')
self._write_lock.acquire()
self._invoke_events_callbacks(self.LL_EVENT_WRITE)
if self._write_lock.locked():
self._write_lock.release()
def _register_events(self):
self._sel = DefaultSelector()
if self._type == self.TYPE_SERVER:
self._sel.register(self._socket, EVENT_READ, self._event_accept)
if self._type in (self.TYPE_HANDLER, self.TYPE_CLIENT):
self._sel.register(self._socket, EVENT_READ | EVENT_WRITE,
self._event_communicate)
self._read_lock = Lock(
) if self._read_lock is None else self._read_lock
self._write_lock = Lock(
) if self._write_lock is None else self._write_lock
self._t_read = Thread(target=self._reading, daemon=True)
self._t_read.start()
self._t_write = Thread(target=self._writing, daemon=True)
self._t_write.start()
def _disconnected(self):
print('Disconnected happened')
self._unregister_events()
def _unregister_events(self):
# self._sel = DefaultSelector()
if self._perform_select:
# print('Unregistering stuff')
self._perform_select = False
self._sel.unregister(self._socket)
if self._read_lock and self._read_lock.locked():
self._read_lock.release()
if self._write_lock and self._write_lock.locked():
self._write_lock.release()
if self._t_read and self._t_read.is_alive():
del self._t_read
if self._t_write and self._t_write.is_alive():
del self._t_write
def _selecting_events(self):
while self._perform_select:
for key, mask in self._sel.select():
key.data(mask)
def _invoke_events_callbacks(self, event_name, *args, **kwargs):
# print(f'LL EVENT Invoked: {event_name}')
if self._subscribed_events[event_name]:
# print('cb: ', self._subscribed_events, event_name)
for cb in self._subscribed_events[event_name]:
cb(*args, **kwargs)
else:
if event_name == self.LL_EVENT_WRITE:
# If there is no write subscriptions, this code
# would decrease unnecessary CPU usage by just a simple timeout.
# Should not be considered as a hack, though if you have
# a better solution - please propose.
# print('No write subscription')
sleep(10)
def _event_accept(self, mask):
if mask & EVENT_READ:
new_connection = self.accept(self.TYPE_HANDLER)
self._invoke_events_callbacks(self.LL_EVENT_ACCEPT, new_connection)
def _event_communicate(self, mask):
if mask & EVENT_READ:
if self._read_lock is not None and self._read_lock.locked():
# print('MREAD')
self._read_lock.release()
# Thread(target=self._invoke_events_callbacks, args=(self.LL_EVENT_READ, ), daemon=True).start()
if mask & EVENT_WRITE:
if self._write_lock is not None and self._write_lock.locked():
# print('MWRITE')
self._write_lock.release()
# self._invoke_events_callbacks(self.LL_EVENT_WRITE)
def collect_frame(self):
frame = None
try:
frame = self._gather_frame_from_socket(SimpleFrame)
if frame is None:
self._invoke_events_callbacks(self.LL_EVENT_DISCONNECTED)
except SSLWantReadError as e:
# todo move to ll
pass
if isinstance(frame, SimpleFrame):
return frame
return False
class MQueue:
def __init__(self, id, host, port):
self.local_node = Node(self, host, port, id=id)
self._sock = ObjectSocket()
self._lock = RLock()
self._msg_cond = Condition(self._lock)
self._nodes_cond = Condition(self._lock)
self._event_thread = Thread(target=self.event_loop, daemon=True)
self._sel = Selector()
self._msgbuf = []
self._message_handlers = {}
self.nodes = {id: self.local_node}
self.size = None
self.timestamp = 0
self.running = True
self.controller = None
def log(self, msg, *args):
# with open(os.path.join(os.path.dirname(sys.argv[0]), 'output-{}.log'.format(self.local_node.id)), 'a') as f:
with open(os.path.join(os.path.dirname(sys.argv[0]), 'debug.log'),
'a') as f:
t = time.time()
date = time.strftime(
'%Y-%m-%d %H:%M:%S',
time.localtime(t)) + '{:.3f}'.format(t - int(t))[1:]
if isinstance(msg, str):
msg = msg.format(*args)
else:
msg = str(msg)
print('{} [{}]: {}'.format(date, self.local_node.id, msg),
file=f,
flush=True)
def event_loop(self):
while True:
# self.log('Waiting for event...')
for key, events in self._sel.select():
try:
# self.log('Event in event_log: {}', key)
if 'handler' in key.data:
key.data['handler'](key.fileobj, key.data)
except EOFError:
self._sel.unregister(key.fileobj)
except SystemExit:
self.log('EXITING')
with self._lock:
self.running = False
self._msg_cond.notify_all()
except:
self.log('Exception in event_loop (key: {}): {}', key,
format_exc())
self._sel.unregister(key.fileobj)
def on_connection(self, sock, data):
c, a = self._sock.accept()
self.log('New connection from {}', a)
self._sel.register(c, EVENT_READ, {'handler': self.on_raw_data})
def _extract_msg(self, sock, data):
if not 'msg' in data:
msg = Message()
data['msg'] = msg
else:
msg = data['msg']
if not msg.iscomplete():
msg.feed(sock)
return msg
def on_raw_data(self, sock, data):
msg = self._extract_msg(sock, data)
if msg.iscomplete():
data = msg.getdata()
self.log('Message received: {} from {}', data, sock.getpeername())
if 'nodes' in data and 'total-node-count' in data:
nodes = data['nodes']
count = data['total-node-count']
self.log('Controller: Nodes: {}, total count: {}', nodes,
count)
with self._lock:
self.size = count
self.log('Setting self.size = {}', self.size)
self._nodes_cond.notify_all()
for id, host, port in nodes:
node = Node(self, host, port, id=id)
node.connect()
self._sel.modify(sock, EVENT_READ,
{'handler': self.on_control})
self.controller = sock
elif 'node-info' in data:
id, host, port = map(lambda k: data['node-info'][k],
['id', 'host', 'port'])
with self._lock:
self.nodes[id] = Node(self, host, port, sock=sock, id=id)
self.log(
'New node (connection from {}): {} (total nodes: {} {})',
sock.getpeername(), id, len(self.nodes), self.nodes)
self._sel.modify(sock, EVENT_READ, {
'handler': self.on_message,
'node': self.nodes[id]
})
self._nodes_cond.notify_all()
def on_message(self, sock, data):
try:
msg = self._extract_msg(sock, data)
except EOFError:
self.log('Node {} disconnected', data['node'])
with self._lock:
del self.nodes[data['node'].id]
raise
except OSError as ose:
self.log('Node {} disconnected due to an error', data['node'])
raise
if msg.iscomplete():
msg = msg.getdata()
del data['msg']
if isinstance(msg, dict) and 'timestamp' in msg and 'data' in msg:
with self._lock:
self.timestamp = max(self.timestamp, msg['timestamp']) + 1
self.log('Message from {}: {} (ts {}). New timestamp: {}',
data['node'].id, msg, msg['timestamp'],
self.timestamp)
msg['node'] = data['node']
if self.controller:
try:
self.controller.sendobj({
'timestamp':
msg['timestamp'],
'from':
msg['node'].id,
'to':
self.local_node.id,
'data':
msg['data'],
'in-channel':
True
})
except:
self.log(
'Exception while sending message {} to controller: {}',
m, format_exc())
for filter, handler in self._message_handlers.items():
if filter(msg):
handler(msg)
return
self._msgbuf.append(msg)
self._msg_cond.notify_all()
def on_control(self, sock, data):
msg = self._extract_msg(sock, data)
if msg.iscomplete():
del data['msg']
msg = msg.getdata()
self.log('Control message: {}', msg)
try:
exec(msg)
except SystemExit:
raise
except:
pass
def register(self, node):
with self._lock:
self.nodes[node.id] = node
self._sel.register(node.sock, EVENT_READ, {
'handler': self.on_message,
'node': node
})
self.log(
'New node (connection to {} using {}): {} (total nodes: {} {})',
node.sock.getpeername(), node.sock.getsockname(), node.id,
len(self.nodes), self.nodes)
self._nodes_cond.notify_all()
def send(self, id, msg):
assert isinstance(msg, dict), 'Message should be dict'
if id != self.local_node.id:
with self._lock:
self.timestamp += 1
self.nodes[id].sock.sendobj({
'timestamp': self.timestamp,
'data': msg
})
def broadcast(self, msg):
with self._lock:
self.timestamp += 1
for n in self.nodes.values():
if n.id != self.local_node.id:
n.sock.sendobj({'timestamp': self.timestamp, 'data': msg})
def recv(self, filter, timeout=None):
with self._lock:
while True:
if not self.running:
self.log('MAIN THREAD TERMINATING')
exit()
for m in self._msgbuf:
if filter(m):
self._msgbuf.remove(m)
if self.controller:
try:
self.controller.sendobj({
'timestamp':
m['timestamp'],
'from':
m['node'].id,
'to':
self.local_node.id,
'data':
m['data'],
'in-channel':
False
})
except:
self.log(
'Exception while sending message {} to controller: {}',
m, format_exc())
return m
if timeout is not None:
t = _time()
if not self._msg_cond.wait(timeout):
raise TimeoutError
if timeout is not None:
timeout -= _time() - t
if timeout <= 0:
raise TimeoutError
def become_coord(self):
with self._lock:
self.timestamp += 1
self._msgbuf.insert(
0, {
'node': self.local_node,
'data': {
'type': MSG_REQUEST,
'change-state': 'coord'
}
})
self._msg_cond.notify_all()
def set_message_handler(self, filter, handler):
with self._lock:
self._message_handlers[filter] = handler
def clear_message_handler(self, filter):
with self._lock:
if filter in self._message_handlers:
del self._message_handlers[filter]
def wait_for_init(self):
with self._lock:
while self.size is None or len(self.nodes) < self.size:
self.log('In wait: self.size: {}, len(self.nodes): {}',
self.size, len(self.nodes))
self._nodes_cond.wait()
def start(self):
# self.log('Starting MQueue...')
self._sock.setsockopt(SOL_SOCKET, SO_REUSEADDR, 1)
self._sock.bind((self.local_node.host, self.local_node.port))
self._sock.listen(5)
self._sel.register(self._sock, EVENT_READ,
{'handler': self.on_connection})
self._event_thread.start()
def loop(self):
try:
self.local_node.loop()
except:
self.mqueue.log('Exception in the main loop: {}', format_exc())
pass
selector.register(self.sock.fileno(),EVENT_WRITE,self.connected)
def connected(self,key,mask):
selector.unregister(key.fd)
get = 'GET {0} HTTP/1.0\r\nHost: www.douban.com\r\n\r\n'.format(self.url)
self.sock.send(get.encode('ascii'))
selector.register(key.fd,EVENT_READ,self.read_response)
def read_response(self,key,mask)
globals stopped
chunk = self.sock.recv(4096)
if chunk:
self.response += chunk
else:
selector.unregister(key.fd)
urls_todo.remove(self.url)
if not urls_todo:
stopped = True
def loop():
while not stopped:
events = selector.select()
for event_key,event_mask in events:
callback = event_key.data
callable(event_key,event_mask)
if __name__=='__main__':
import time
start = time.time()
for url in urls_todo:
class Loop:
def __init__(self):
self._tasks = deque()
self._sleeping = []
self._stop = False
self._selector = DefaultSelector()
def create_task(self, coroutine):
if isinstance(coroutine, Task):
return coroutine
task = Task(coroutine)
self._tasks.append(task)
return task
def remove(self, task):
self.tasks.remove(task)
def _stop_loop_cb(self, task):
self._stop = True
def _wakeup_sleeping_tasks(self):
now = time.monotonic()
while self._sleeping and self._sleeping[0][0] <= now:
_, task = heapq.heappop(self._sleeping)
self._tasks.append(task)
def io_poll(self):
if self._sleeping:
sleep_time = self._sleeping[0][0] - time.monotonic()
else:
sleep_time = None
events = self._selector.select(sleep_time)
for key, mask in events:
task = key.data
self._selector.unregister(key.fileobj)
self._tasks.append(task)
# Wake up time!
self._wakeup_sleeping_tasks()
def run_until_complete(self, coroutine):
task = self.create_task(coroutine)
task.add_done_callback(self._stop_loop_cb)
self.run_forever()
task.remove_done_callback(self._stop_loop_cb)
return task.result
def run_forever(self):
self._stop = False # Just in case we are restarted
while not self._stop:
while not self._tasks:
self.io_poll()
# Let's see if there any task who wants to wake up
self._wakeup_sleeping_tasks()
current = self._tasks.popleft()
try:
# We assume that each Task will yield some method name and
# an argument. Based on that, we decide what to do.
method_name, arg = next(current)
except StopIteration as e:
current.result = e.value
except Exception as e:
current.exception = e
print(f'Warning: Task {current.id} raised {e!r}')
else:
method = getattr(self, method_name, None)
assert method, f'unknown method name {method_name}'
method(arg, current)
finally:
if current.done:
for callback in current.done_callbacks:
callback(current)
# These are the methods which can be "invoked" by Tasks by yielding their
# name, together with an argument.
def sleep_task(self, seconds, task):
# We simply add the task to the self._sleeping priority queue.
# The use of a priority queue here is very convenient: we push
# (time to wake up, task) into the queue. When we get/pop an item from
# the queue, we automatically get the fisrt task to wake up together
# with its wakeup time.
heapq.heappush(self._sleeping, (time.monotonic() + seconds, task))
def read_wait(self, resource, task):
self._selector.register(resource, EVENT_READ, task)
def write_wait(self, resource, task):
self._selector.register(resource, EVENT_WRITE, task)
# Socket stuff
def sock_recv(self, sock, maxsize):
yield 'read_wait', sock
return sock.recv(maxsize)
def sock_sendall(self, sock, data):
while data:
try:
n = sock.send(data)
data = data[n:]
except BlockingIOError:
yield 'write_wait', sock
def sock_accept(self, sock):
yield 'read_wait', sock
return sock.accept()
def sock_connect(self, sock, address):
yield 'write_wait', sock
return sock.connect(address)
class SimpleServer(object):
""" This is a simple server receiving and sending utf-8 plain text lines
over TCP/IP. It should use epoll or other efficient mechanisms if
available to support a large number of clients without cpu hogging.
All data sent to connections will be utf-8 encoded. All received lines
from clients are expected to be utf-8 encoded.
Lines longer than 10KB will be quietly dropped.
"""
def __init__(self, port, callback):
logging.basicConfig(level=logging.INFO)
self.clients_kept_for_sending = []
self.selector = DefaultSelector()
self.callback = callback
self.server_socket = socket.socket(family=socket.AF_INET6,
type=socket.SOCK_STREAM)
self.server_socket.bind(("::0", port))
self.server_socket.listen(5)
self.selector.register(self.server_socket, EVENT_READ, data=None)
self.server_socket.setblocking(False)
def close(self):
self.server_socket.close()
def _remove_socket(self, socket):
try:
self.selector.unregister(socket)
except KeyError:
pass
def _handle_new_client(self, socket, addr):
client = SimpleClient(socket, self, addr)
self.selector.register(socket, EVENT_READ, data=client)
self.callback(client)
def _mark_client_for_sending(self, client, marked):
if marked:
self.selector.modify(client.socket, EVENT_READ | EVENT_WRITE,
data=client)
else:
self.selector.modify(client.socket, EVENT_READ, data=client)
def tick(self, timeout=0):
""" This will check for any server events and process them.
If timeout is greater than zero, it will wait at most that
amount of time in seconds until something happens.
(This is useful if you want to save some cpu time and you have
nothing else to do)
For debugging purposes, it returns the amount of events processed.
"""
# get rid of old connections we kept around for sending:
entry_removed = True
while entry_removed:
entry_removed = False
for entry in self.clients_kept_for_sending:
if entry[1] <= time.time() or len(entry[0].send_buffer) == 0:
try:
entry[0].close()
except:
pass
self.clients_kept_for_sending.remove(entry)
entry_removed = True
# wait for socket events:
events = self.selector.select(timeout=timeout)
events_processed = 0
for key, mask in events:
if mask & EVENT_READ != 0:
events_processed = events_processed + 1
# handle accept for server socket:
if key.data == None:
conn, addr = self.server_socket.accept()
self._handle_new_client(conn, addr)
continue
# handle client socket receive:
key.data._do_recv()
continue
if mask & EVENT_WRITE != 0:
events_processed = events_processed + 1
# handle client socket send:
key.data._do_outstanding_send()
continue
logging.debugging("unknown socket event happening, bug?")
return events_processed
request_callback = lambda sock: sock.send(request.encode())
response_callback = lambda sock: \
print((sock.recv(1000)).decode().split("\n")[-1], 'finish at %s' % str(datetime.now())[:-7])
def get():
sock = socket.socket()
sock.setblocking(False)
try:
sock.connect(('localhost', 3000))
except:
pass
sel.register(sock, EVENT_WRITE, lambda: request_callback(sock))
get_count = 4
print("starts at %s" % str(datetime.now())[:-7])
for i in range(get_count):
get()
while True:
events = sel.select()
for key, _ in events:
callback = key.data
event = key.events
sel.unregister(key.fileobj)
callback()
if event == EVENT_WRITE:
sel.register(key.fileobj, EVENT_READ, lambda: response_callback(key.fileobj))
elif event == EVENT_WRITE:
sel.register(key.fileobj, EVENT_WRITE, lambda: request_callback(key.fileobj))
selector=DefaultSelector()
ctx=SSLContext(PROTOCOL_SSLv23)
ctx.load_cert_chain(certfile=clientcert)
sock=socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.setblocking(False)
selector.register(sock,EVENT_WRITE)
try:
sock.connect((host,443))
except BlockingIOError:
pass
while True:
events=selector.select(2)
if events:
break
selector.unregister(sock)
sc=ctx.wrap_socket(sock,server_side=False,do_handshake_on_connect=False)
selector.register(sc,EVENT_WRITE | EVENT_READ)
while True:
events = selector.select(2)
if events:
try:
sc.do_handshake()
break
except SSLWantReadError:
pass
except SSLWantWriteError:
pass
class Server:
"""
A simple messaging server. Connects to one client at a time.
"""
def __init__(self):
self.host = ''
self.socket = None
self.selector = DefaultSelector()
self.running = False
self.users = {}
self.log = []
def start(self):
"""
Open a socket at this server's host and port and begin listening.
"""
self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
self.socket.bind((self.host, PORT))
self.socket.listen(5)
# Callback self.connect() when a client connects to the server.
self.selector.register(self.socket, EVENT_READ)
self.running = True
def stop(self):
"""
Close this server's socket.
"""
self.running = False
self.socket.close()
def connect(self, server):
"""
Add a new user for an incoming client. Blocks, so call after select.
"""
client, addr = server.accept()
user = User(addr)
self.users[client] = User(addr)
self.selector.register(client, EVENT_READ)
print("Received connection from client at {}.".format(str(user)))
def disconnect(self, s):
"""Remove a user and close their socket."""
name = str(self.users[s])
del self.users[s]
s.close()
self.selector.unregister(s)
print("Client at {} has disconnected.".format(name))
def loop(self):
"""
Infinitely read and write messages, with a small delay to prevent
hogging up cpu time.
"""
while True:
self.select()
self.flush_messages()
sleep(0.1)
def select(self):
"""Select and operate on any ready socket."""
for key, mask in self.selector.select(0.0):
socket = key.fileobj
if socket == self.socket: # server message, client connecting.
self.connect(socket)
elif socket in self.users.keys(): # user message
self.receive_message(socket)
else:
malt.log("Unknown selector event? {}".format(socket))
def flush_messages(self):
"""Send pending messages to all users."""
for socket, user in self.users.items():
if user.pending:
malt.log("Sending messages to {}.".format(user.name))
socket.sendall('\0'.join(user.pending).encode())
user.pending = []
def receive_message(self, socket):
"""Read a socket for messages. Blocks, so only call if selected."""
user = self.users[socket]
# Throws IOError when client disconnects.
message = socket.recv(BUF_SIZE).decode()
if message == '': # Empty message indicates closed socket.
self.disconnect(socket)
else:
try:
head, tail = decode_message(message)
except ValueError:
return
if head == 'name':
# TODO: Add a notification message for name change
malt.log("Renaming {} to {}.".format(str(user), tail))
user.name = tail
elif head == 'message':
self.record(user, tail)
malt.log("{}: '{}'.".format(user.name, tail))
elif head == 'read':
malt.log("Reading messages to {}.".format(user.name))
socket.sendall('\0'.join(user.pending).encode())
user.pending = []
def record(self, user, message):
"""
Format and record a message to both the server log and all other users
currently connected to the server.
"""
text = user.name + ': ' + message
self.log.append(text)
for other in self.users.values():
#if other != user:
# other.pending.append(text)
other.pending.append(text)
class EventLoop:
def __init__(self):
self.tasks = deque()
self.selector = DefaultSelector()
def run_until_complete(self, task):
self.tasks.append(task)
self.run()
def pause(self):
return "pause", None
def schedule(self, target):
return "schedule", target
def sock_accept(self, sock):
yield ("read", sock)
return sock.accept()
def sock_recv(self, sock):
yield ("read", sock)
return sock.recv(1024)
def sock_sendall(self, sock, data):
yield ("write", sock)
return sock.sendall(data)
def start_server(self, handler, host, port, backlog=0):
handler = partial(handler, self)
with socket.socket() as sock:
sock.bind((host, port))
sock.listen(backlog)
print("Listening on {}:{}".format(host, port))
while True:
conn, addr = yield from self.sock_accept(sock)
print("Accepted client from", addr)
yield self.schedule(handler(conn))
def run(self):
while self.tasks or self.selector.get_map():
for _ in range(len(self.tasks)):
try:
task = self.tasks.popleft()
tag, value = next(task)
if tag == "schedule":
self.tasks.append(value)
self.tasks.append(task)
elif tag == "read":
self.selector.register(value, EVENT_READ, data=task)
elif tag == "write":
self.selector.register(value, EVENT_WRITE, data=task)
elif tag == "pause":
self.tasks.append(task)
else:
raise ValueError("Incorrect tag")
except StopIteration:
continue
if self.selector.get_map():
for key, event in self.selector.select():
if event & EVENT_READ or event & EVENT_WRITE:
self.tasks.append(key.data)
self.selector.unregister(key.fileobj)
class Loop:
"""
Событийный цикл, который использует очередь сдвухсторонним доступом
Цикл опрашивает селектор на предмет готовности событий для чтения и записи
в сокет
"""
def __init__(self):
"""
Конструктор событийного цикла, который хранит очередь
:return:
"""
self.ready = deque()
self.selector = DefaultSelector()
async def sock_recv(self, sock, max_bytes):
"""
курутина для чтения данных из сокета в асинхронный способ
:param sock: дескриптор сокета
:param max_bytes: максимальное количество байт, который могут быть
прочитаны за один раз без блокировки
:return: принятые из сокета данные в банарном виде
"""
await read_wait(sock)
return sock.recv(max_bytes)
async def sock_accept(self, sock):
await read_wait(sock)
return sock.accept()
async def sock_sendall(self, sock, data):
while data:
try:
n_sent = sock.send(data)
data = data[n_sent:]
except BlockingIOError:
await write_wait(sock)
def create_task(self, coro):
self.ready.append(coro)
def run_forever(self):
while True:
while not self.ready:
events = self.selector.select()
for key, _ in events:
self.ready.append(key.data)
self.selector.unregister(key.fileobj)
while self.ready:
self.current_task = self.ready.popleft()
try:
# запускаем генератор до появления yield
op, *args = self.current_task.send(None)
getattr(self, op)(*args)
except StopIteration:
pass
def read_wait(self, sock):
self.selector.register(sock, EVENT_READ, self.current_task)
def write_wait(self, sock):
self.selector.register(sock, EVENT_WRITE, self.current_task)
class RtEnvironment(BaseEnvironment):
def __init__(self, initial_time=0, factor=1.0, strict=True):
self.env_start = initial_time
self.real_start = time()
self._factor = factor
self._strict = strict
self._now = initial_time
self._queue = []
self._eid = count()
self._rid = 0
self._active_proc = None
self._prepare()
self._data = {}
def _prepare(self):
self._selector = DefaultSelector()
self._sock = socket()
self._selector.register(self._sock, EVENT_READ, None)
def get_data(self, key):
if key in self._data.keys():
return self._data[key]
else:
return 0
def set_data(self, key, value):
self._data[key] = value
@property
def now(self):
return self._now
@property
def active_process(self):
return self._active_proc
@property
def factor(self):
return self._factor
@property
def strict(self):
return self._strict
def schedule(self, event, priority=NORMAL, delay=0):
heappush(self._queue, (self._now + delay, priority, next(self._eid), event))
def register(self, fileobj, events, callback):
self._rid += 1
self._selector.register(fileobj, events, callback)
def unregister(self, fileobj):
self._rid -= 1
self._selector.unregister(fileobj)
def peek(self):
return self._queue[0][0]
def sync(self):
self.real_start = time()
def step(self):
try:
evt_time = self.peek()
except IndexError:
if self._rid > 0:
evt_time = Infinity
else:
raise EmptySchedule()
real_time = self.real_start + (evt_time - self.env_start) * self.factor
if self.strict and time() - real_time > self.factor:
raise RuntimeError('Simulation too slow for real time (%.3fs).' % (time() - real_time))
while True:
delta = real_time - time()
if delta <= 0:
break
events = self._selector.select(timeout=delta)
if events:
self._now = (time() - self.real_start) / self.factor
for key, mask in events:
callback = key.data
callback()
real_time = self.real_start + (self.peek() - self.env_start) * self.factor
self._now, _, _, event = heappop(self._queue)
callbacks, event.callbacks = event.callbacks, None
for callback in callbacks:
callback(event)
if not event._ok:
exc = type(event._value)(*event._value.args)
exc.__cause__ = event._value
raise exc
class Peer(HandleableMixin, CommandableMixin, LanTopologyMixin,
DefaultAuthenticatorMixin):
"""
Attributes:
server_info (PeerInfo): Peer's peer info.
pkt_handlers (Dict[str, Handler]): All handlers which peer have ability
to process.
peer_pool (Dict[Tuple[str, int], PeerInfo]): All peers currently avai-
lable in net.
"""
@property
def server_info(self):
return self.__server_info
@property
def program_hash(self):
return self.__program_hash
@property
def send_queue(self):
# def packet_queue(self):
return self.__packet_queue
@property
def connectlist(self) -> List: # Remember to remove `List` import.
return self.peer_pool.values()
def __init__(
self,
host: Tuple[str, int],
name: str,
role: str,
cert: Tuple[str, str],
program_hash: str,
ns: str,
auto_register: bool = False,
logger: "logging.Logger" = getLogger(__name__),
) -> None:
"""Init of PeerManager
Args:
host: Binding host.
name: Peer's name in net.
role: Peer's role in net.
cert: Cert file's path.
program_hash: Program self hash to send in packet.
ns: Nameserver address for resolve DNS.
logger: Logger for logging.
"""
super().__init__()
self.logger = getLogger(name)
self.__auto_register = auto_register
self.__selector = DefaultSelector()
self.__packet_queue = {}
self.__cert = cert
self.__program_hash = program_hash
self.__server_info = PeerInfo(host=host, name=name, role=role)
self.__tcp_server = self.__bind_socket(cert=self.__cert)
self.peer_pool = {}
self.pkt_handlers = {}
self.commands = {}
self.logger.info("Program hash: {{{}...{}}}".format(
self.__program_hash[:6], self.__program_hash[-6:]))
self.dns_resolver = DNSResolver(ns="127.0.0.1" if ns is None else ns,
role=role)
self.monitor = Monitor(peer=self, logger=getLogger(name + ".MONITOR"))
if self.__auto_register is False:
self.logger.warning(
("auto_register parameter is set to False,\n You may need to r"
"egister them through _register_command & _register_handler m"
"ethod."))
def _preregister_handler(self) -> None:
self.topology_register_handler()
installing_handlers = [MessageHandler(self)]
for each in installing_handlers:
self.register_handler(handler=each)
def _preregister_command(self) -> None:
installing_commands = [
HelpCmd(self),
JoinCmd(self),
SendCmd(self),
ListCmd(self),
LeaveNetCmd(self),
]
for each in installing_commands:
self.register_command(command=each)
def pend_packet(self, sock: "Socket", pkt: "Packet", **kwargs) -> None:
"""Pending pkt's raw_data to queue's with sepecific sock.
Any exception when wrapping handler to packet whould cause this connec-
tion been close and thread maintaining loop terminate.
Args:
sock: A Socket which wants to pend on its queue.
pkt: A Packet ready to be pend.
**kwargs: Any additional arguments needs by handler object.
Raises:
AssertionError:
If given pkt variable is not in proper Packet type.
"""
assert type(pkt) is Packet
try:
self.__packet_queue[sock].put_nowait(pkt)
except Exception:
self.logger.info(format_exc())
def register_socket(self, sock: "Socket") -> None:
"""Register a new socket with packet queue & selector.
Init a packet queue and put into dict for further handling of packets.
And the given socket will be register in selector for IO process.
Args:
sock: A Socket object which wants to be register.
"""
self.__packet_queue[sock] = Queue()
self.__selector.register(sock, EVENT_READ | EVENT_WRITE,
self.__on_handle)
def unregister_socket(self, sock: "Socket") -> None:
del self.__packet_queue[sock]
self.__selector.unregister(sock)
def _on_packet(self, sock: "Socket", pkt: "Packet",
handler: "Handler") -> None:
"""Method use to process passed packet to higher application layer.
This method will call by AuthenticatorMixin when a packet is passed examination.
This is 3rd layer to process packet to handler. This is last layer to application layer.
"""
handler.on_recv(src=pkt.src, pkt=pkt, sock=sock)
self.monitor.on_recv_pkt(addr=pkt.src, pkt=pkt, conn=sock)
def new_tcp_long_conn(self, dst: Tuple[str, int]) -> "SSLSocket":
"""Create a ssl-wrapped TCP socket with given destination host
Args:
dst: Specified socket destination.
Returns:
A SSLSocket object which connected to destination host with
non-blocking.
Raises:
AssertionError:
If given dst variable is not in proper Tuple[str, int] type.
"""
assert host_valid(dst) is True
unwrap_socket = socket(AF_INET, SOCK_STREAM)
sock = wrap_socket(unwrap_socket,
cert_reqs=CERT_REQUIRED,
ca_certs=self.__cert[0])
sock.connect(dst)
sock.setblocking(False)
return sock
def loop_start(self):
self.logger.info(self.__server_info)
self.__selector.register(self.__tcp_server, EVENT_READ,
self.__on_accept)
if self.__auto_register is True:
self._preregister_handler()
self._preregister_command()
if self.monitor.is_start() is False:
self.monitor.start()
self.logger.info("Peer started.")
def loop(self):
return self._loop()
def _loop(self):
"""Called inside infinite loop from outside inherited class.
It's use to call the method which given event is triggered.
"""
events = self.__selector.select(timeout=0)
for key, mask in events:
if callable(key.data):
key.data(key.fileobj, mask)
def loop_stop(self):
for _, value in self.__packet_queue.items():
if value.empty() is False:
sleep(2)
return self.loop_stop()
self.__selector.unregister(self.__tcp_server)
self.leave_net()
self.monitor.stop()
def loop_stop_post(self):
self.__tcp_server.close()
self.__selector.close()
def __bind_socket(self, cert: Tuple[str, str]) -> "SSLSocket":
unwrap_socket = socket(AF_INET, SOCK_STREAM)
unwrap_socket.setsockopt(SOL_SOCKET, SO_REUSEADDR, 1)
unwrap_socket.setsockopt(SOL_SOCKET, SO_REUSEPORT, 1)
unwrap_socket.bind(self.__server_info.host)
unwrap_socket.listen(5)
unwrap_socket.setblocking(False)
self.logger.info("Peer prepared")
self.logger.info(
"This peer is running with certificate at path {}".format(cert[0]))
self.logger.info("Please make sure other peers have same certicate.")
return wrap_socket(unwrap_socket,
certfile=cert[0],
keyfile=cert[1],
server_side=True)
def __on_accept(self, sock: "Socket", mask, **kwargs):
"""Call when a new socket is connection by waiter socket.
This will accept all sockets from outside, but it doesn't mean every socket's
packet be process by higher layer.
This is the 1st layer to process sockets.
"""
conn, _ = sock.accept()
conn.setblocking(False)
self.register_socket(sock=conn)
def __on_handle(self, sock: "Socket", mask, **kwargs):
"""Decide whether send or recv."""
if mask & EVENT_READ == EVENT_READ:
self.__on_recv(sock=sock, mask=mask, **kwargs)
if mask & EVENT_WRITE == EVENT_WRITE:
self.__on_send(sock=sock, mask=mask, **kwargs)
def __on_recv(self, sock: "Socket", mask, **kwargs):
"""Method use when recieve socket data.
This is 2th layer to process Packets.
"""
try:
raw_data = sock.recv(4096)
if raw_data == b"":
return
pkt = Packet.deserilize(raw_data=raw_data)
return self._authenticate_packet(sock=sock, pkt=pkt)
except SSLWantReadError:
return
except sock_error as sock_err:
if sock_err.errno == ECONNRESET:
peer_info = self.get_peer_info_by_conn(conn=sock)
if peer_info is not None:
peer_info.status.update(status_type=StatusType.NO_RESP)
self.logger.warning("Peer {} Connection Reseted.".format(
peer_info.host))
else:
raise sock_err
except Exception:
self.logger.warning(str(self.server_info) + format_exc())
def __on_send(self, sock: "Socket", mask, **kwargs):
"""Method use when sending data to socket."""
q = self.__packet_queue[sock] if sock in self.__packet_queue else None
while q is not None and q.empty() is False:
try:
pkt = q.get_nowait()
handler = self.select_handler(pkt_type=pkt._type)
handler.pre_send(pkt=pkt)
data = Packet.serilize(obj=pkt)
sock.send(data)
handler.post_send(pkt=pkt, sock=sock)
except sock_error as sock_err:
if sock_err.errno == ECONNRESET:
q.put_nowait(pkt)
self.monitor.peer_status_update_by_host(
host=pkt.src, status_type=StatusType.NO_RESP)
self.logger.warning("Peer {} Connection Reseted.".format(
pkt.src))
else:
raise sock_err
except Exception:
self.logger.warning(format_exc())
class TorReceiver(threading.Thread):
def __init__(self, tor_socket, handler_mgr):
super().__init__(name='RecvLoop_{}'.format(tor_socket.ip_address[0:7]))
self._tor_socket = tor_socket
self._handler_mgr = handler_mgr
self._do_loop = False
# fmt: off
self._regs_funcs_map = {
'reg': {
socket.socket: self.register_socket,
TorStream: self.register_stream
},
'unreg': {
socket.socket: self.unregister_socket,
TorStream: self.unregister_stream
}
}
# fmt: on
self._stream_to_callback = {}
self._selector = DefaultSelector()
self._cntrl_r, self._cntrl_w = socket.socketpair()
self._selector.register(self._cntrl_r, EVENT_READ, self._do_stop)
self._selector.register(self._tor_socket.ssl_socket, EVENT_READ, self._do_recv)
def _cleanup(self):
self._selector.unregister(self._cntrl_r)
self._cntrl_w.close()
self._cntrl_r.close()
self._selector.unregister(self._tor_socket.ssl_socket)
self._selector.close()
def start(self):
self._do_loop = True
super().start()
def stop(self):
logger.debug('Stopping receiver thread...')
self._cntrl_w.send(b'\1')
self.join()
def register(self, sock_or_stream, events, callback):
func = self._regs_funcs_map['reg'].get(type(sock_or_stream))
if not func:
raise Exception('Unknown object for register')
return func(sock_or_stream, events, callback)
def register_socket(self, sock, events, callback):
return self._selector.register(sock, events, callback)
def register_stream(self, stream: TorStream, events, callback):
if events & EVENT_WRITE:
raise Exception('Write event not supported yet')
stream.register(callback)
if stream not in self._stream_to_callback:
self._stream_to_callback[stream] = []
self._stream_to_callback[stream].append(callback)
def unregister(self, sock_or_stream):
func = self._regs_funcs_map['unreg'].get(type(sock_or_stream))
if not func:
raise Exception('Unknown object for unregister')
return func(sock_or_stream)
def unregister_socket(self, sock):
return self._selector.unregister(sock)
def unregister_stream(self, stream):
callbacks = self._stream_to_callback.pop(stream, None)
if not callbacks:
raise Exception('There is no such stream registered')
for callback in callbacks:
stream.unregister(callback)
def _do_stop(self, raw_socket, mask):
self._do_loop = False
def _do_recv(self, raw_socket, mask):
for cell in self._tor_socket.recv_cell_async():
logger.debug('Cell received: %r', cell)
try:
self._handler_mgr.handle(cell)
except BaseException:
logger.exception('Some handle errors')
def run(self):
logger.debug('Starting...')
while self._do_loop:
events = self._selector.select()
for key, mask in events:
callback = key.data
callback(key.fileobj, mask)
self._cleanup()
logger.debug('Stopped...')
class Loop:
def __init__(self):
self.ready = collections.deque()
self.selector = DefaultSelector()
async def sock_accept(self, sock):
'''
Wait for the server socket to become readable, accept a new
client and return her socket, addr pair.
'''
await read_wait(sock)
return sock.accept()
async def sock_recv(self, sock, maxbytes):
'''
Wait for the socket to become readable, read data from the
socket and return data.
'''
await read_wait(sock)
return sock.recv(maxbytes)
async def sock_sendall(self, sock, data):
'''
While we have some data to send, wait for the socket to become
writeable and send some data.
'''
while data:
await write_wait(sock)
nbytes = sock.send(data)
# Modify data according to the number of bytes sent this time
data = data[nbytes:]
def create_task(self, coro):
self.ready.append(coro)
def run_forever(self):
while True:
while not self.ready:
events = self.selector.select()
for key, _ in events:
self.selector.unregister(key.fileobj)
self.ready.append(key.data)
while self.ready:
# I need this task in another method, so I attach it here
self.current_task = self.ready.popleft()
try:
# drive the coroutine to the next yield
op, *args = self.current_task.send(None)
except StopIteration:
pass
else:
# call method on myself
# this methods will register sockets with selector
getattr(self, op)(*args)
def read_wait(self, sock):
# When this socket is ready, drive this task to the next yield
self.selector.register(sock, EVENT_READ, self.current_task)
def write_wait(self, sock):
self.selector.register(sock, EVENT_WRITE, self.current_task)
